From 50677b204be3acc0c381370a5ab9cd393f1c900d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 10:35:33 +0100 Subject: [PATCH 001/243] GEN1: the seat generator refuses a definition whose callers entry is no form of the format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The generator checks each verb's and accepted kind's callers in a definition and a part before reading it, naming the file, the verb and the entry; text is refused as one entry; the contracts it writes carry no caller check. Builds: design 47 G §336, §339, §392 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../buildstep/seatgen/callers.go | 72 +++++++++++++++++++ .../buildstep/seatgen/layout_test.go | 55 ++++++++++++++ .../mesh-build-go/buildstep/seatgen/load.go | 6 ++ 3 files changed, 133 insertions(+) create mode 100644 modules/mesh-build-go/buildstep/seatgen/callers.go diff --git a/modules/mesh-build-go/buildstep/seatgen/callers.go b/modules/mesh-build-go/buildstep/seatgen/callers.go new file mode 100644 index 000000000..a4a40d55c --- /dev/null +++ b/modules/mesh-build-go/buildstep/seatgen/callers.go @@ -0,0 +1,72 @@ +package seatgen + +import ( + "encoding/json" + "fmt" + re "regexp" + "strings" +) + +// callerName is a seat's or a module's name inside a caller entry. +var callerName = re.MustCompile(`^[a-z][a-z0-9]*(-[a-z0-9]+)*$`) + +// callerClasses are the entries that take no name (design 47 G §336, §392). +var callerClasses = map[string]bool{"operator": true, "healers": true, "persons": true, "sessions": true, "invokers": true} + +// callerForm says whether an entry is one of the forms of design 47 G §336 as §392 writes them: `operator`, `healers`, +// `persons`, `sessions`, `invokers`, `seat:`, `module:` and `holder:@same-node`. +func callerForm(entry string) bool { + switch { + case callerClasses[entry]: + return true + case strings.HasPrefix(entry, "seat:"): + return callerName.MatchString(strings.TrimPrefix(entry, "seat:")) + case strings.HasPrefix(entry, "module:"): + return callerName.MatchString(strings.TrimPrefix(entry, "module:")) + case strings.HasPrefix(entry, "holder:") && strings.HasSuffix(entry, "@same-node"): + return callerName.MatchString(strings.TrimSuffix(strings.TrimPrefix(entry, "holder:"), "@same-node")) + } + return false +} + +// checkCallers refuses a definition or part whose verb or accepted kind names a `callers` that is no list of the +// forms of §336, naming the entry (design 47 G §339: the generator reads `callers` and makes nothing of it; the +// grants part reads them from the definition). Text, the old form, is refused as one entry. +func checkCallers(file string, raw []byte) error { + var doc struct { + Verbs []callerHolder `json:"verbs"` + Accepts []callerHolder `json:"accepts"` + } + if err := json.Unmarshal(raw, &doc); err != nil { + return nil // the definition reader names a malformed file + } + for _, group := range []struct { + field string + items []callerHolder + }{{"verbs", doc.Verbs}, {"accepts", doc.Accepts}} { + for _, it := range group.items { + if len(it.Callers) == 0 { + continue + } + var list []string + if err := json.Unmarshal(it.Callers, &list); err != nil { + var text string + if json.Unmarshal(it.Callers, &text) != nil { + return fmt.Errorf("%s: %s %q: callers is neither a list nor text", file, group.field, it.Name) + } + list = []string{text} + } + for _, e := range list { + if !callerForm(e) { + return fmt.Errorf("%s: %s %q: caller %q is no form of the format (operator, healers, persons, sessions, invokers, seat:, module:, holder:@same-node)", file, group.field, it.Name, e) + } + } + } + } + return nil +} + +type callerHolder struct { + Name string `json:"name"` + Callers json.RawMessage `json:"callers"` +} diff --git a/modules/mesh-build-go/buildstep/seatgen/layout_test.go b/modules/mesh-build-go/buildstep/seatgen/layout_test.go index f038c020f..77837dcee 100644 --- a/modules/mesh-build-go/buildstep/seatgen/layout_test.go +++ b/modules/mesh-build-go/buildstep/seatgen/layout_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "go/parser" "go/token" + "os" re "regexp" "strings" "testing" @@ -177,3 +178,57 @@ func TestEachSeatDefinedWholeIsAGoModuleOfItsOwn(t *testing.T) { } } } + +// TestCallersAreTheFormsOfTheFormat is design 47 G §339's check: the generator refuses a definition (or a part) +// whose verb or accepted kind names a caller that is no form of §336, naming the entry, and the contracts it writes +// carry no caller check. +func TestCallersAreTheFormsOfTheFormat(t *testing.T) { + for _, tc := range []struct { + name, callers, refused string + }{ + {"every form", `["operator","healers","persons","sessions","invokers","seat:mesh-wants","module:gitea","holder:node-backup@same-node"]`, ""}, + {"text", `"any caller whose role grants it"`, "any caller whose role grants it"}, + {"a bare seat name", `["mesh-wants"]`, "mesh-wants"}, + {"holder without the node", `["holder:node-backup"]`, "holder:node-backup"}, + {"seat with no name", `["seat:"]`, "seat:"}, + } { + for _, place := range []string{"verbs", "accepts"} { + t.Run(tc.name+"/"+place, func(t *testing.T) { + raw := []byte(`{"format":1,"seat":"x","` + place + `":[{"name":"v","callers":` + tc.callers + `}]}`) + err := checkCallers("seats/x/seat.json", raw) + switch { + case tc.refused == "" && err != nil: + t.Fatalf("refused: %v", err) + case tc.refused != "" && (err == nil || !strings.Contains(err.Error(), tc.refused) || !strings.Contains(err.Error(), "seats/x/seat.json")): + t.Fatalf("error %v, want one naming %q", err, tc.refused) + } + }) + } + } + // Load refuses an unknown entry in a definition before it reads it. + dir := t.TempDir() + if err := os.MkdirAll(dir+"/seats/x", 0o755); err != nil { + t.Fatal(err) + } + def := `{"format":1,"seat":"x","scope":"mesh","grain":"seat-list","verbs":[{"name":"v","callers":["somebody"]}]}` + if err := os.WriteFile(dir+"/seats/x/seat.json", []byte(def), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Load(dir, "", ""); err == nil || !strings.Contains(err.Error(), `"somebody"`) { + t.Fatalf("Load error %v, want one naming the entry", err) + } + // The generated contracts of the testdata hold no caller check. + m, err := Load("testdata/contracts", "example.org/contracts/go", "@example/contracts") + if err != nil { + t.Fatal(err) + } + files, err := Generate(m) + if err != nil { + t.Fatal(err) + } + for p, b := range files { + if strings.Contains(strings.ToLower(string(b)), "callers") && strings.HasSuffix(p, ".go") { + t.Errorf("%s carries a caller check", p) + } + } +} diff --git a/modules/mesh-build-go/buildstep/seatgen/load.go b/modules/mesh-build-go/buildstep/seatgen/load.go index 697dba1c0..62c429672 100644 --- a/modules/mesh-build-go/buildstep/seatgen/load.go +++ b/modules/mesh-build-go/buildstep/seatgen/load.go @@ -50,6 +50,9 @@ func Load(dir, goModule, tsPackage string) (*Module, error) { if err != nil { return nil, err } + if err := checkCallers(rel(dir, p), raw); err != nil { + return nil, err + } part, err := seatdef.ParsePart(raw) if err != nil { return nil, fmt.Errorf("%s: %w", rel(dir, p), err) @@ -66,6 +69,9 @@ func Load(dir, goModule, tsPackage string) (*Module, error) { if err != nil { return nil, err } + if err := checkCallers(rel(dir, p), raw); err != nil { + return nil, err + } d, err := seatdef.Parse(raw) if err != nil { return nil, fmt.Errorf("%s: %w", rel(dir, p), err) -- 2.54.0 From fe8c41928deb261b72e2cf949819d6c25bc4a8eb Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 10:37:07 +0100 Subject: [PATCH 002/243] GEN2: the seat generator emits each TypeScript contract as index.js with its index.d.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A seat's npm package holds index.js, index.d.ts and parts/.js with .d.ts, with no TypeScript source and no tsconfig.json, so a TypeScript module imports it with no build step and no tsc of the contract. Builds: design 47 G §341 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../buildstep/seatgen/layout_test.go | 71 ++++++++- .../mesh-build-go/buildstep/seatgen/states.go | 26 ++-- .../buildstep/seatgen/states_test.go | 22 ++- .../mesh-build-go/buildstep/seatgen/tsgen.go | 137 +++++++++--------- 4 files changed, 167 insertions(+), 89 deletions(-) diff --git a/modules/mesh-build-go/buildstep/seatgen/layout_test.go b/modules/mesh-build-go/buildstep/seatgen/layout_test.go index 77837dcee..4f097012c 100644 --- a/modules/mesh-build-go/buildstep/seatgen/layout_test.go +++ b/modules/mesh-build-go/buildstep/seatgen/layout_test.go @@ -5,6 +5,8 @@ import ( "go/parser" "go/token" "os" + "os/exec" + "path/filepath" re "regexp" "strings" "testing" @@ -79,7 +81,7 @@ func TestEachSeatDefinedWholeIsAGoModuleOfItsOwn(t *testing.T) { !(strings.Contains(src, "\t\"time\"\n") || strings.Contains(src, "import \"time\"")) { t.Errorf("%s does not write a date-time as time.Time:\n%s", seat, src) } - if ts := string(files["ts/seats/"+seat+"/src/parts/registry.ts"]); !strings.Contains(ts, `"pushed": string;`) || !strings.Contains(ts, `"at"?: string;`) { + if ts := string(files["ts/seats/"+seat+"/parts/registry.d.ts"]); !strings.Contains(ts, `"pushed": string;`) || !strings.Contains(ts, `"at"?: string;`) { t.Errorf("%s's TypeScript does not write a date-time as a string:\n%s", seat, ts) } // A scalar typed ["T","null"] is a pointer in Go and `T | null` in TypeScript, so a declared zero survives with its type. @@ -88,7 +90,7 @@ func TestEachSeatDefinedWholeIsAGoModuleOfItsOwn(t *testing.T) { t.Errorf("%s does not write a nullable scalar as a pointer (%s):\n%s", seat, w, src) } } - if ts := string(files["ts/seats/"+seat+"/src/parts/registry.ts"]); !strings.Contains(ts, `"floor"?: number | null;`) || !strings.Contains(ts, `"on"?: boolean | null;`) || !strings.Contains(ts, `"label"?: string | null;`) { + if ts := string(files["ts/seats/"+seat+"/parts/registry.d.ts"]); !strings.Contains(ts, `"floor"?: number | null;`) || !strings.Contains(ts, `"on"?: boolean | null;`) || !strings.Contains(ts, `"label"?: string | null;`) { t.Errorf("%s's TypeScript does not write a nullable scalar as T | null:\n%s", seat, ts) } // A string's enum is a constant per value, so a caller names the definition's value (A3t: the removal reason). @@ -139,8 +141,8 @@ func TestEachSeatDefinedWholeIsAGoModuleOfItsOwn(t *testing.T) { dir = d } } - if dir == "" || !strings.HasPrefix(p, dir+"src/") { - t.Errorf("%s is in no npm package's src", p) + if dir == "" || (p != dir+"index.d.ts" && !strings.HasPrefix(p, dir+"parts/")) { + t.Errorf("%s is in no npm package's files", p) } for _, line := range strings.Split(string(b), "\n") { i := strings.Index(line, ` from "`) @@ -172,8 +174,8 @@ func TestEachSeatDefinedWholeIsAGoModuleOfItsOwn(t *testing.T) { } } for _, seat := range []string{"aregistry", "bregistry"} { - if !strings.Contains(string(files["ts/seats/"+seat+"/src/index.ts"]), "export type PublishedEvent = registry.PublishedEvent") || - !strings.Contains(string(files["ts/seats/"+seat+"/src/parts/registry.ts"]), "export type PublishedEvent") { + if !strings.Contains(string(files["ts/seats/"+seat+"/index.d.ts"]), "export type PublishedEvent = registry.PublishedEvent") || + !strings.Contains(string(files["ts/seats/"+seat+"/parts/registry.d.ts"]), "export type PublishedEvent") { t.Errorf("the npm package of %s does not hold the part's types whole", seat) } } @@ -232,3 +234,60 @@ func TestCallersAreTheFormsOfTheFormat(t *testing.T) { } } } + +// TestATypeScriptContractIsJavaScriptWithItsDeclarations is design 47 G §341's check: each seat's npm package holds the +// contract as index.js with its index.d.ts (and each part the same), no TypeScript source and no tsconfig, so a +// TypeScript module imports it with no build step and no tsc of the contract. +func TestATypeScriptContractIsJavaScriptWithItsDeclarations(t *testing.T) { + m, err := Load("testdata/contracts", "example.org/contracts/go", "@example/contracts") + if err != nil { + t.Fatal(err) + } + files, err := Generate(m) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"index.js", "index.d.ts", "parts/registry.js", "parts/registry.d.ts", "package.json", "package-lock.json"} { + if _, ok := files["ts/seats/aregistry/"+want]; !ok { + t.Errorf("the seat's npm package lacks %s", want) + } + } + for p, b := range files { + if !strings.HasPrefix(p, "ts/") { + continue + } + if (strings.HasSuffix(p, ".ts") && !strings.HasSuffix(p, ".d.ts")) || strings.HasSuffix(p, "tsconfig.json") { + t.Errorf("%s is TypeScript source or a compiler's setting: the contract needs no tsc", p) + } + if strings.HasSuffix(p, ".js") { + for _, typed := range []string{": string", "interface ", " as const", "export type ", "import type", "declare "} { + if strings.Contains(string(b), typed) { + t.Errorf("%s holds TypeScript syntax %q: it is JavaScript", p, typed) + } + } + if node, err := exec.LookPath("node"); err == nil { + path := filepath.Join(t.TempDir(), "check.mjs") + if err := os.WriteFile(path, b, 0o644); err != nil { + t.Fatal(err) + } + if out, err := exec.Command(node, "--check", path).CombinedOutput(); err != nil { + t.Errorf("%s is no JavaScript module: %v\n%s", p, err, out) + } + } + } + } + var pkg struct { + Types string `json:"types"` + Exports map[string]map[string]string `json:"exports"` + Scripts map[string]string `json:"scripts"` + } + if err := json.Unmarshal(files["ts/seats/aregistry/package.json"], &pkg); err != nil { + t.Fatal(err) + } + if pkg.Types != "./index.d.ts" || pkg.Exports["."]["default"] != "./index.js" || pkg.Exports["."]["types"] != "./index.d.ts" || len(pkg.Scripts) != 0 { + t.Errorf("package.json says %+v", pkg) + } + if !strings.Contains(string(files["ts/seats/aregistry/index.d.ts"]), `import type * as registry from "./parts/registry.js";`) { + t.Error("the declarations do not import the part") + } +} diff --git a/modules/mesh-build-go/buildstep/seatgen/states.go b/modules/mesh-build-go/buildstep/seatgen/states.go index 71bb54dd0..d1c40c315 100644 --- a/modules/mesh-build-go/buildstep/seatgen/states.go +++ b/modules/mesh-build-go/buildstep/seatgen/states.go @@ -203,14 +203,14 @@ func goStatesImports(states []seatdef.State) string { return out + ")\n\n" } -// tsStates writes a seat's states for TypeScript: the local names, and for each key form its builder and parser (the -// value's type is declared with the seat's other types). -func tsStates(b *bytes.Buffer, states []seatdef.State) error { +// tsStates writes a seat's states for TypeScript: the local names, and for each key form its builder and parser, as +// JavaScript and as declarations (the value's type is declared with the seat's other types). +func tsStates(o *tsOut, states []seatdef.State) error { names := make([]string, len(states)) for i, s := range states { names[i] = s.Name } - tsNames(b, "states", names) + tsNames(o, "states", names) for _, s := range states { for _, kf := range s.KeyForms { ts, err := keyTokens(kf.Key) @@ -218,11 +218,12 @@ func tsStates(b *bytes.Buffer, states []seatdef.State) error { return fmt.Errorf("state %s: %w", s.Name, err) } fn := tsName(formIdent(s.Name, kf.Key, ts)) - var args, expr, fields, got []string + var args, params, expr, fields, got []string for _, t := range ts { if t.part { p := paramName(t.text) args = append(args, p+": string") + params = append(params, p) expr = append(expr, p) fields = append(fields, p+": string") got = append(got, p) @@ -230,10 +231,17 @@ func tsStates(b *bytes.Buffer, states []seatdef.State) error { expr = append(expr, strconv.Quote(t.text)) } } - fmt.Fprintf(b, "/** The key of one value of the state %s under the form %s. */\nexport function %sKey(%s): string {\n return %s;\n}\n\n", - s.Name, strconv.Quote(kf.Key), fn, strings.Join(args, ", "), strings.Join(expr, " + ")) - fmt.Fprintf(b, "/** The parts of a key of the form %s, or undefined for a key of another form. */\nexport function parse%sKey(_key: string): { %s } | undefined {\n let _rest = _key;\n", - strconv.Quote(kf.Key), ident(fn), strings.Join(fields, "; ")) + doc := fmt.Sprintf("/** The key of one value of the state %s under the form %s. */\n", s.Name, strconv.Quote(kf.Key)) + o.dts.WriteString(doc) + fmt.Fprintf(&o.dts, "export declare function %sKey(%s): string;\n\n", fn, strings.Join(args, ", ")) + o.js.WriteString(doc) + fmt.Fprintf(&o.js, "export function %sKey(%s) {\n return %s;\n}\n\n", fn, strings.Join(params, ", "), strings.Join(expr, " + ")) + doc = fmt.Sprintf("/** The parts of a key of the form %s, or undefined for a key of another form. */\n", strconv.Quote(kf.Key)) + o.dts.WriteString(doc) + fmt.Fprintf(&o.dts, "export declare function parse%sKey(_key: string): { %s } | undefined;\n\n", ident(fn), strings.Join(fields, "; ")) + b := &o.js + b.WriteString(doc) + fmt.Fprintf(b, "export function parse%sKey(_key) {\n let _rest = _key;\n", ident(fn)) for i := 0; i < len(ts); { t := ts[i] switch { diff --git a/modules/mesh-build-go/buildstep/seatgen/states_test.go b/modules/mesh-build-go/buildstep/seatgen/states_test.go index be565d7fd..01c6b3d72 100644 --- a/modules/mesh-build-go/buildstep/seatgen/states_test.go +++ b/modules/mesh-build-go/buildstep/seatgen/states_test.go @@ -41,14 +41,20 @@ func TestAStateHasItsNamesKeyFormsAndStrictValues(t *testing.T) { if strings.Contains(src, "Record") { t.Errorf("the seat's package still names a record:\n%s", src) } - ts := string(files["ts/seats/aregistry/src/index.ts"]) - for _, want := range []string{"export const states = {", "export function asksByAskerIDKey(asker: string, id: string): string", - "export function parseAsksByAskerIDKey(_key: string)", "export interface AsksByAskerIDValue", - // a key form with a part named key: the parser's own parameter and locals are named apart from the parts - "export function parseAsksByNodeModuleKeyKey(_key: string): { node: string; module: string; key: string } | undefined", - "const key = _rest;"} { - if !strings.Contains(ts, want) { - t.Errorf("the seat's npm package lacks %q:\n%s", want, ts) + js := string(files["ts/seats/aregistry/index.js"]) + dts := string(files["ts/seats/aregistry/index.d.ts"]) + for file, wants := range map[string][]string{ + js: {"export const states = {", "export function asksByAskerIDKey(asker, id) {", "export function parseAsksByAskerIDKey(_key) {", + "export function parseAsksByNodeModuleKeyKey(_key) {", "const key = _rest;"}, + dts: {"export declare const states: {", "export declare function asksByAskerIDKey(asker: string, id: string): string;", + "export declare function parseAsksByAskerIDKey(_key: string)", "export interface AsksByAskerIDValue", + // a key form with a part named key: the parser's own parameter and locals are named apart from the parts + "export declare function parseAsksByNodeModuleKeyKey(_key: string): { node: string; module: string; key: string } | undefined;"}, + } { + for _, want := range wants { + if !strings.Contains(file, want) { + t.Errorf("the seat's npm package lacks %q:\n%s", want, file) + } } } diff --git a/modules/mesh-build-go/buildstep/seatgen/tsgen.go b/modules/mesh-build-go/buildstep/seatgen/tsgen.go index 17e58563a..a4476b22c 100644 --- a/modules/mesh-build-go/buildstep/seatgen/tsgen.go +++ b/modules/mesh-build-go/buildstep/seatgen/tsgen.go @@ -19,31 +19,17 @@ func TSSeatDir(seat string) string { return "ts/seats/" + pkgName(seat) } // TSSeatPackage is the npm package name of a seat defined whole, beside the seat set's package root. func TSSeatPackage(root, seat string) string { return root + "-" + seat } -// tsConfig type-checks the package's sources and writes nothing: the package ships its TypeScript, which its importers -// compile or bundle, so it has no build script and needs no compiler of its own (F7: a generated package has no -// dependency, and its lock, which the build module requires of a package artifact, holds none). -const tsConfig = `{ - "compilerOptions": { - "target": "ES2022", - "module": "NodeNext", - "moduleResolution": "NodeNext", - "noEmit": true, - "strict": true, - "skipLibCheck": true, - "forceConsistentCasingInFileNames": true - }, - "include": ["src/**/*.ts"] -} -` - -// tsPackage writes one npm package's package.json, its dependency-free package-lock.json and its tsconfig.json at dir. +// tsPackage writes one npm package's package.json and its dependency-free package-lock.json at dir. The package ships +// its contract as JavaScript with its declarations (design 47 G §341): its importers need no build step of its own and +// nothing runs tsc over it, so it has no tsconfig.json and no script (F7: a generated package has no dependency, and its +// lock, which the build module requires of a package artifact, holds none). func tsPackage(out map[string][]byte, dir, name, description string) { pkg := map[string]any{ "name": name, "version": "0.1.0", "type": "module", "description": description, - "exports": map[string]any{".": "./src/index.ts"}, - "types": "./src/index.ts", - "files": []any{"src"}, + "exports": map[string]any{".": map[string]any{"types": "./index.d.ts", "default": "./index.js"}}, + "types": "./index.d.ts", + "files": []any{"index.js", "index.d.ts", "parts"}, } raw, _ := json.MarshalIndent(pkg, "", " ") out[dir+"/package.json"] = append(raw, '\n') @@ -53,13 +39,18 @@ func tsPackage(out map[string][]byte, dir, name, description string) { } raw, _ = json.MarshalIndent(lock, "", " ") out[dir+"/package-lock.json"] = append(raw, '\n') - out[dir+"/tsconfig.json"] = []byte(tsConfig) } -// tsFiles writes the npm side: one package per seat defined whole at TSSeatDir (ADR 0625 §1), holding the parts it includes whole and importing nothing outside it. +// tsOut is one contract file as its two forms: the JavaScript module a TypeScript module imports and the declarations +// that type it (design 47 G §341). +type tsOut struct{ js, dts bytes.Buffer } + +// tsFiles writes the npm side: one package per seat defined whole at TSSeatDir (ADR 0625 §1), holding the parts it +// includes whole and importing nothing outside it. Each file is a JavaScript module with its declarations beside it +// (`index.js` and `index.d.ts`, `parts/.js` and `parts/.d.ts`; design 47 G §341). func tsFiles(m *Module, out map[string][]byte) error { partTypes := map[string]*typer{} - partFiles := map[string][]byte{} + partFiles := map[string]*tsOut{} for _, name := range sortedKeys(m.Parts) { p := m.Parts[name] d := seatdef.Definition{Grain: seatdef.Whole, Verbs: p.Verbs, Events: p.Events, Defs: p.Defs} @@ -68,13 +59,17 @@ func tsFiles(m *Module, out map[string][]byte) error { return err } partTypes[name] = tp - var b bytes.Buffer - b.WriteString(tsHeader) - fmt.Fprintf(&b, "// The part %q several seats include whole: %s\n\n", p.Part, oneLine(p.Description)) - tsNames(&b, "verbs", seatdef.VerbNames(p.Verbs)) - tsNames(&b, "events", seatdef.MessageNames(p.Events)) - tsTypes(&b, tp) - partFiles[name] = b.Bytes() + o := &tsOut{} + o.js.WriteString(tsHeader) + o.dts.WriteString(tsHeader) + note := fmt.Sprintf("// The part %q several seats include whole: %s\n\n", p.Part, oneLine(p.Description)) + o.js.WriteString(note) + o.dts.WriteString(note) + tsNames(o, "verbs", seatdef.VerbNames(p.Verbs)) + tsNames(o, "events", seatdef.MessageNames(p.Events)) + o.js.WriteString("export {};\n") + tsTypes(&o.dts, tp) + partFiles[name] = o } for _, name := range m.Order { d := m.Seats[name] @@ -89,67 +84,74 @@ func tsFiles(m *Module, out map[string][]byte) error { for _, t := range partTypes[inc].types { fromPart[t.name] = inc } - out[dir+"/src/parts/"+inc+".ts"] = partFiles[inc] + out[dir+"/parts/"+inc+".js"] = partFiles[inc].js.Bytes() + out[dir+"/parts/"+inc+".d.ts"] = partFiles[inc].dts.Bytes() } tp, err := declareAll(d, fromPart) if err != nil { return err } - var b bytes.Buffer - b.WriteString(tsHeader) - fmt.Fprintf(&b, "// The contract of the seat %q, defined whole (seats/%s/seat.json): %s\n\n", d.Seat, d.Seat, oneLine(d.Description)) + o := &tsOut{} + o.js.WriteString(tsHeader) + o.dts.WriteString(tsHeader) + note := fmt.Sprintf("// The contract of the seat %q, defined whole (seats/%s/seat.json): %s\n\n", d.Seat, d.Seat, oneLine(d.Description)) + o.js.WriteString(note) + o.dts.WriteString(note) for _, inc := range raw.Includes { - fmt.Fprintf(&b, "import type * as %s from \"./parts/%s.js\";\n\n", tsName(inc), inc) + fmt.Fprintf(&o.dts, "import type * as %s from \"./parts/%s.js\";\n\n", tsName(inc), inc) } - fmt.Fprintf(&b, "export const seat = %q;\n\n", d.Seat) - tsNames(&b, "verbs", seatdef.VerbNames(d.Verbs)) - tsNames(&b, "accepts", seatdef.MessageNames(d.Accepts)) - tsNames(&b, "events", seatdef.MessageNames(d.Events)) + fmt.Fprintf(&o.js, "export const seat = %q;\n\n", d.Seat) + fmt.Fprintf(&o.dts, "export declare const seat: %q;\n\n", d.Seat) + tsNames(o, "verbs", seatdef.VerbNames(d.Verbs)) + tsNames(o, "accepts", seatdef.MessageNames(d.Accepts)) + tsNames(o, "events", seatdef.MessageNames(d.Events)) if f := d.Family; f != nil { - fmt.Fprintf(&b, "export const family = { name: %q, maxAgeDays: %d", f.Name, f.MaxAgeDays) - b.WriteString(" } as const;\n\n") + fmt.Fprintf(&o.js, "export const family = { name: %q, maxAgeDays: %d };\n\n", f.Name, f.MaxAgeDays) + fmt.Fprintf(&o.dts, "export declare const family: { readonly name: %q; readonly maxAgeDays: %d };\n\n", f.Name, f.MaxAgeDays) } for _, e := range d.Events { - subj := seatdef.EventSubject(d.Seat, e.Name) - if e.ByCaller { - fmt.Fprintf(&b, "export function %sSubject(caller: string): string {\n return %q + caller;\n}\n\n", tsName(e.Name), subj+".") - } else { - fmt.Fprintf(&b, "export const %sSubject = %q;\n\n", tsName(e.Name), subj) - } + tsSubject(o, tsName(e.Name), seatdef.EventSubject(d.Seat, e.Name), e.ByCaller) } for _, a := range d.Accepts { - subj := seatdef.AcceptSubject(d.Seat, a.Name) - if a.ByCaller { - fmt.Fprintf(&b, "export function %sSubject(caller: string): string {\n return %q + caller;\n}\n\n", tsName(a.Name), subj+".") - } else { - fmt.Fprintf(&b, "export const %sSubject = %q;\n\n", tsName(a.Name), subj) - } + tsSubject(o, tsName(a.Name), seatdef.AcceptSubject(d.Seat, a.Name), a.ByCaller) } if len(d.HolderEvents) > 0 { - tsNames(&b, "holderEvents", holderNames(d.HolderEvents)) + tsNames(o, "holderEvents", holderNames(d.HolderEvents)) for _, e := range d.HolderEvents { before, after, _ := strings.Cut(e.Subject, "") - fmt.Fprintf(&b, "export function %sHolderSubject(holder: string): string {\n return %q + holder + %q;\n}\n\n", - tsName(e.Name), before, after) + fmt.Fprintf(&o.js, "export function %sHolderSubject(holder) {\n return %q + holder + %q;\n}\n\n", tsName(e.Name), before, after) + fmt.Fprintf(&o.dts, "export declare function %sHolderSubject(holder: string): string;\n\n", tsName(e.Name)) } } if len(d.States) > 0 { - if err := tsStates(&b, d.States); err != nil { + if err := tsStates(o, d.States); err != nil { return err } } for _, n := range sortedKeys(fromPart) { - fmt.Fprintf(&b, "export type %s = %s.%s;\n", n, tsName(fromPart[n]), n) + fmt.Fprintf(&o.dts, "export type %s = %s.%s;\n", n, tsName(fromPart[n]), n) } if len(fromPart) > 0 { - b.WriteString("\n") + o.dts.WriteString("\n") } - tsTypes(&b, tp) - out[dir+"/src/index.ts"] = b.Bytes() + tsTypes(&o.dts, tp) + out[dir+"/index.js"] = o.js.Bytes() + out[dir+"/index.d.ts"] = o.dts.Bytes() } return nil } +// tsSubject writes the constant, or for a subject ending in the caller's name the function, of one subject. +func tsSubject(o *tsOut, name, subj string, byCaller bool) { + if byCaller { + fmt.Fprintf(&o.js, "export function %sSubject(caller) {\n return %q + caller;\n}\n\n", name, subj+".") + fmt.Fprintf(&o.dts, "export declare function %sSubject(caller: string): string;\n\n", name) + return + } + fmt.Fprintf(&o.js, "export const %sSubject = %q;\n\n", name, subj) + fmt.Fprintf(&o.dts, "export declare const %sSubject: %q;\n\n", name, subj) +} + func holderNames(es []seatdef.HolderEvent) []string { out := make([]string, len(es)) for i, e := range es { @@ -158,15 +160,18 @@ func holderNames(es []seatdef.HolderEvent) []string { return out } -func tsNames(b *bytes.Buffer, what string, names []string) { +func tsNames(o *tsOut, what string, names []string) { if len(names) == 0 { return } - fmt.Fprintf(b, "export const %s = {\n", what) + fmt.Fprintf(&o.js, "export const %s = {\n", what) + fmt.Fprintf(&o.dts, "export declare const %s: {\n", what) for _, n := range names { - fmt.Fprintf(b, " %s: %q,\n", tsName(n), n) + fmt.Fprintf(&o.js, " %s: %q,\n", tsName(n), n) + fmt.Fprintf(&o.dts, " readonly %s: %q;\n", tsName(n), n) } - b.WriteString("} as const;\n\n") + o.js.WriteString("};\n\n") + o.dts.WriteString("};\n\n") } func tsTypes(b *bytes.Buffer, tp *typer) { -- 2.54.0 From c62650e4ff133de7d2e024793de442752df9f1e8 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 10:40:42 +0100 Subject: [PATCH 003/243] GEN3: the seat generator reads acts, subject, keep and callers and writes each event's subject and stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An event whose subject carries parts is published by a function taking each part and heard by a filter with a * per part; an event naming keep gets the stream SEAT__, in Go and in the TypeScript contract. The reader of the sdk's reader release reads acts (every, a language, kind:image), keep as an object and callers as a list. Builds: design 47 G §89, §203, §238, §260, §324, §331, §396, §397 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/buildstep/seatgen/gogen.go | 68 +++++++++++++++++++ .../buildstep/seatgen/layout_test.go | 46 +++++++++++++ .../contracts/seats/a-registry/seat.json | 7 +- .../mesh-build-go/buildstep/seatgen/tsgen.go | 23 +++++++ 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/modules/mesh-build-go/buildstep/seatgen/gogen.go b/modules/mesh-build-go/buildstep/seatgen/gogen.go index 0c231490d..ec0382747 100644 --- a/modules/mesh-build-go/buildstep/seatgen/gogen.go +++ b/modules/mesh-build-go/buildstep/seatgen/gogen.go @@ -180,6 +180,13 @@ func goWhole(m *Module, d seatdef.Definition) ([]byte, []byte, error) { } for _, e := range d.Events { subj := seatdef.EventSubject(d.Seat, e.Name) + if e.Keep != nil { + fmt.Fprintf(&b, "// Stream%s is the stream %s is kept in.\nconst Stream%s = %q\n\n", ident(e.Name), e.Name, ident(e.Name), keptStream(d.Seat, e.Name)) + } + if parts := e.Parts(); len(parts) > 0 { + goPartsSubject(&b, d.Seat, e, parts) + continue + } if e.ByCaller { fmt.Fprintf(&b, "// Subject%s is where %s is published for one caller.\nfunc Subject%s(caller string) string { return %q + caller }\n\n", ident(e.Name), e.Name, ident(e.Name), subj+".") @@ -424,6 +431,9 @@ func fixtureTarget(d seatdef.Definition, kind, name string) (string, string, err } for _, msg := range ms { if msg.Name == name { + if kind == "event" && len(msg.Parts()) > 0 { + return ident(name) + suffix, partsPattern(d.Seat, msg), nil + } p := "^" + regexp(subj) if msg.ByCaller { p += `\.[A-Za-z0-9_-]+` @@ -460,3 +470,61 @@ func regexp(s string) string { } return b.String() } + +// keptStream is the bus's name of the stream an event naming `keep` is kept in: `SEAT__`, which the bus +// takes (no dot) and the grants part makes (design 47 G §260, §396). +func keptStream(seat, event string) string { return "SEAT_" + seat + "_" + event } + +// subjectExpr is the Go or JavaScript string expression of an event's subject with its parts as arguments. +func subjectExpr(seat string, e seatdef.Message) []string { + var expr []string + lit := "mesh.seat." + seat + ".event." + for i, tok := range strings.Split(e.SubjectPattern(), ".") { + if i > 0 { + lit += "." + } + if strings.HasPrefix(tok, "<") { + expr = append(expr, fmt.Sprintf("%q", lit), goParam(tok[1:len(tok)-1])) + lit = "" + continue + } + lit += tok + } + if lit != "" { + expr = append(expr, fmt.Sprintf("%q", lit)) + } + return expr +} + +// goPartsSubject writes the subject of an event whose subject carries parts (design 47 G §260, §397): a function taking +// each part in order, and the filter that hears every instance of it. +func goPartsSubject(b *bytes.Buffer, seat string, e seatdef.Message, parts []string) { + args := make([]string, len(parts)) + for i, p := range parts { + args[i] = goParam(p) + } + fmt.Fprintf(b, "// Subject%s is where %s is published, on %s.\nfunc Subject%s(%s string) string { return %s }\n\n", + ident(e.Name), e.Name, "mesh.seat."+seat+".event."+e.SubjectPattern(), ident(e.Name), strings.Join(args, ", "), + strings.Join(subjectExpr(seat, e), " + ")) + fmt.Fprintf(b, "// Subject%sAll hears every %s.\nconst Subject%sAll = %q\n\n", ident(e.Name), e.Name, ident(e.Name), + "mesh.seat."+seat+".event."+e.Local()) +} + +// goParam is a part's name as a parameter: its words joined, the first lower-case. +func goParam(part string) string { + id := ident(part) + return strings.ToLower(id[:1]) + id[1:] +} + +// partsPattern is the pattern a fixture's subject of an event with parts must match: a part is one token. +func partsPattern(seat string, e seatdef.Message) string { + toks := strings.Split(e.SubjectPattern(), ".") + for i, tok := range toks { + if strings.HasPrefix(tok, "<") { + toks[i] = `[^.*> ]+` + } else { + toks[i] = regexp(tok) + } + } + return "^" + regexp("mesh.seat."+seat+".event.") + strings.Join(toks, `\.`) + "$" +} diff --git a/modules/mesh-build-go/buildstep/seatgen/layout_test.go b/modules/mesh-build-go/buildstep/seatgen/layout_test.go index 4f097012c..6c714b9a2 100644 --- a/modules/mesh-build-go/buildstep/seatgen/layout_test.go +++ b/modules/mesh-build-go/buildstep/seatgen/layout_test.go @@ -291,3 +291,49 @@ func TestATypeScriptContractIsJavaScriptWithItsDeclarations(t *testing.T) { t.Error("the declarations do not import the part") } } + +// TestADefinitionsActsSubjectsKeepAndCallersAreRead is design 47 G §89, §203, §260, §324, §331's check for the +// generator: it reads `acts` (a step on every, a language and `kind: image`), `subject`, `keep` and `callers`, and +// writes each event's subject and the stream `SEAT__` of one that names `keep`. +func TestADefinitionsActsSubjectsKeepAndCallersAreRead(t *testing.T) { + m, err := Load("testdata/contracts", "example.org/contracts/go", "@example/contracts") + if err != nil { + t.Fatal(err) + } + if acts := m.Seats["a-registry"].Acts; len(acts) != 3 || acts[0].On != "kind:image" || acts[1].On != "every" || acts[2].On != "go" { + t.Errorf("acts read as %+v", acts) + } + files, err := Generate(m) + if err != nil { + t.Fatal(err) + } + src := string(files["go/seats/aregistry/contract.go"]) + for _, want := range []string{ + `const StreamMilestoneAsked = "SEAT_a-registry_milestone-asked"`, + `const StreamOwed = "SEAT_a-registry_owed"`, + "func SubjectMilestoneAsked(milestone string) string {\n\treturn \"mesh.seat.a-registry.event.milestone-asked.\" + milestone\n}", + `SubjectMilestoneAskedAll = "mesh.seat.a-registry.event.milestone-asked.*"`, + `func SubjectOwed(repository, number string) string {`, + "func SubjectStepEnded(node string) string { return \"mesh.seat.a-registry.event.step-ended.\" + node }", + } { + if !strings.Contains(src, want) { + t.Errorf("the Go contract lacks %q:\n%s", want, src) + } + } + if strings.Contains(src, "StreamStepEnded") { + t.Error("an event naming no keep has a stream of its own") + } + js := string(files["ts/seats/aregistry/index.js"]) + dts := string(files["ts/seats/aregistry/index.d.ts"]) + for _, want := range []string{`export const milestoneAskedStream = "SEAT_a-registry_milestone-asked";`, + `export function stepEndedSubject(node) {`, `export const milestoneAskedSubjectAll = "mesh.seat.a-registry.event.milestone-asked.*";`} { + if !strings.Contains(js, want) { + t.Errorf("index.js lacks %q:\n%s", want, js) + } + } + for _, want := range []string{`export declare const owedStream: "SEAT_a-registry_owed";`, `export declare function owedSubject(repository: string, number: string): string;`} { + if !strings.Contains(dts, want) { + t.Errorf("index.d.ts lacks %q:\n%s", want, dts) + } + } +} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json index 42259601e..f992622b6 100644 --- a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json @@ -1,5 +1,10 @@ {"format":1,"seat":"a-registry","scope":"mesh","grain":"whole","domain":"Module","decision":"ADR 1","includes":["registry"], - "verbs":[{"name":"hold","input":{"type":"object"},"answer":{"type":"object"}}], + "verbs":[{"name":"hold","class":"read","callers":["invokers","persons","sessions","seat:b-registry","module:gitea","operator","healers","holder:node-backup@same-node"],"input":{"type":"object"},"answer":{"type":"object"}}], + "acts":[{"step":"build","on":"kind:image","bound":"60m"},{"step":"check","on":"every"},{"step":"build","on":"go"}], + "events":[ + {"name":"milestone-asked","subject":"milestone-asked.","body":{"type":"object"},"keep":{"max-age-days":90,"max-mib":16,"last-per-subject":true}}, + {"name":"owed","subject":"owed..","body":{"type":"object"},"keep":{"max-age-days":7,"max-mib":64}}, + {"name":"step-ended","subject":"step-ended.","body":{"type":"object"}}], "family":{"name":"a-registry.events","max-age-days":7}, "states":[ {"name":"asks","template":"state","description":"the asks held, one key per asker and ask", diff --git a/modules/mesh-build-go/buildstep/seatgen/tsgen.go b/modules/mesh-build-go/buildstep/seatgen/tsgen.go index a4476b22c..b1b655b12 100644 --- a/modules/mesh-build-go/buildstep/seatgen/tsgen.go +++ b/modules/mesh-build-go/buildstep/seatgen/tsgen.go @@ -110,6 +110,14 @@ func tsFiles(m *Module, out map[string][]byte) error { fmt.Fprintf(&o.dts, "export declare const family: { readonly name: %q; readonly maxAgeDays: %d };\n\n", f.Name, f.MaxAgeDays) } for _, e := range d.Events { + if e.Keep != nil { + fmt.Fprintf(&o.js, "export const %sStream = %q;\n\n", tsName(e.Name), keptStream(d.Seat, e.Name)) + fmt.Fprintf(&o.dts, "export declare const %sStream: %q;\n\n", tsName(e.Name), keptStream(d.Seat, e.Name)) + } + if parts := e.Parts(); len(parts) > 0 { + tsPartsSubject(o, d.Seat, e, parts) + continue + } tsSubject(o, tsName(e.Name), seatdef.EventSubject(d.Seat, e.Name), e.ByCaller) } for _, a := range d.Accepts { @@ -235,3 +243,18 @@ func tsType(t *typ) string { } return "unknown" } + +// tsPartsSubject writes the subject of an event whose subject carries parts, as goPartsSubject does. +func tsPartsSubject(o *tsOut, seat string, e seatdef.Message, parts []string) { + args := make([]string, len(parts)) + names := make([]string, len(parts)) + for i, p := range parts { + names[i] = goParam(p) + args[i] = names[i] + ": string" + } + all := "mesh.seat." + seat + ".event." + e.Local() + fmt.Fprintf(&o.js, "export function %sSubject(%s) {\n return %s;\n}\n\n", tsName(e.Name), strings.Join(names, ", "), strings.Join(subjectExpr(seat, e), " + ")) + fmt.Fprintf(&o.js, "export const %sSubjectAll = %q;\n\n", tsName(e.Name), all) + fmt.Fprintf(&o.dts, "export declare function %sSubject(%s): string;\n\n", tsName(e.Name), strings.Join(args, ", ")) + fmt.Fprintf(&o.dts, "export declare const %sSubjectAll: %q;\n\n", tsName(e.Name), all) +} -- 2.54.0 From cfad86394f5483422e9ba74e0d6d7110368218bc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 10:54:55 +0100 Subject: [PATCH 004/243] BLD1: the Go build seat serves the verb steps, found by Go's convention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The verb fetches the artifact's tree and answers lint always, unit and integration steps by the packages holding test files (integration by the tag), a level with no test left out; the manifest serves it. Builds: design 47 G §22, §307, §28 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/cmd/mesh-build-go/asks.go | 2 +- .../cmd/mesh-build-go/asks_test.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/steps.go | 198 ++++++++++++++++++ .../cmd/mesh-build-go/steps_test.go | 50 +++++ modules/mesh-build-go/module.json | 2 +- 5 files changed, 251 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/steps.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/steps_test.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 415f217bf..a1fcf324a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -134,7 +134,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // SeatTools are the seat's verbs on this node (ADR 0525 §10): what it runs, and its pause and resume. func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool { - return buildstep.SeatTools(Seat, "Go", p) + return append(buildstep.SeatTools(Seat, "Go", p), m.stepsTool()) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index c866703c9..810569bb7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -118,7 +118,7 @@ func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } // the node's build capacity as a setting, with the builder's default; the seat's verbs are served as its tools. func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { m := readManifest(t) - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { t.Fatalf("claims = %+v", m.Claims) } if !slices.Contains(m.Reads, KeptState) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/steps.go b/modules/mesh-build-go/cmd/mesh-build-go/steps.go new file mode 100644 index 000000000..2aa35e58d --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/steps.go @@ -0,0 +1,198 @@ +package main + +// **The verb `steps`** (design 47 G §22, §307; the contract's verb of the seat `node-build-go`): which pipeline steps an +// artifact can run, found by Go's convention. Lint is always one; unit tests are the packages that hold a test file, +// integration tests those that hold a test file built only with the tag `integration` (the tag the integration level +// runs with, levels.go). A level with no package is not answered: the pipeline has nothing to run there (§22, §328). + +import ( + "context" + "fmt" + "go/build/constraint" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The pipeline steps `steps` answers (the contract's DetectedStep enum). +const ( + StepLint = "lint" + StepUnit = "unit" + StepIntegration = "integration" +) + +// DetectedStep is the contract's. +type DetectedStep = seat.DetectedStep + +// testLevel says, for a test file's header, which level it belongs to: unit when it is built without the tag +// integration, integration when it is built only with it, "" when neither (another platform's file). +func testLevel(header string) string { + base := func(tag string) bool { return tag == "linux" || tag == "amd64" || tag == "unix" || tag == "gc" } + for _, line := range strings.Split(header, "\n") { + line = strings.TrimSpace(line) + if !constraint.IsGoBuild(line) { + if strings.HasPrefix(line, "package ") { + break + } + continue + } + expr, err := constraint.Parse(line) + if err != nil { + return "" + } + switch { + case expr.Eval(base): + return StepUnit + case expr.Eval(func(tag string) bool { return tag == "integration" || base(tag) }): + return StepIntegration + } + return "" + } + return StepUnit +} + +// detectSteps finds the steps of the Go packages under root (the artifact's directory in the fetched tree): the import +// path of each package is its go.mod's module path and its directory below that go.mod. +func detectSteps(root string) ([]DetectedStep, error) { + found := map[string]map[string]bool{StepUnit: {}, StepIntegration: {}} + var walk func(dir, mod, modDir string) error + walk = func(dir, mod, modDir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + if raw, err := os.ReadFile(filepath.Join(dir, "go.mod")); err == nil { + mod, modDir = modulePath(string(raw)), dir + } + for _, e := range entries { + name := e.Name() + if e.IsDir() { + if name == "testdata" || name == "vendor" || strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if err := walk(filepath.Join(dir, name), mod, modDir); err != nil { + return err + } + continue + } + if !strings.HasSuffix(name, "_test.go") || mod == "" { + continue + } + raw, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return err + } + level := testLevel(string(raw[:min(len(raw), 4096)])) + if level == "" { + continue + } + rel, err := filepath.Rel(modDir, dir) + if err != nil { + return err + } + pkg := mod + if rel != "." { + pkg = path.Join(mod, filepath.ToSlash(rel)) + } + found[level][pkg] = true + } + return nil + } + if err := walk(root, "", root); err != nil { + return nil, err + } + steps := []DetectedStep{{Step: StepLint}} + for _, level := range []string{StepUnit, StepIntegration} { + if len(found[level]) == 0 { + continue + } + pkgs := make([]string, 0, len(found[level])) + for p := range found[level] { + pkgs = append(pkgs, p) + } + sort.Strings(pkgs) + steps = append(steps, DetectedStep{Step: level, Packages: pkgs}) + } + return steps, nil +} + +// StepsAnswer is what `steps` answers. +type StepsAnswer struct { + Artifact string `json:"artifact"` + Fingerprint string `json:"fingerprint,omitempty"` + Steps []DetectedStep `json:"steps"` + Kept bool `json:"kept,omitempty"` +} + +// stepsTool is the seat's verb `steps`: it fetches the artifact's commit into a workspace of its own, only the tree (no +// module download: the detection reads files), finds the steps and removes the workspace. +func (m *Module) stepsTool() stdio.Tool { + return stdio.Tool{Name: Seat + "." + seat.VerbSteps, + Description: "Which pipeline steps a Go artifact can run, found by Go's convention: lint always, unit tests where a package holds a test file, " + + "integration tests where one holds a test file built with the tag integration. A level with no test is not answered.", + Input: map[string]any{"type": "object", "required": []string{"module", "artifact", "fetch"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, + "artifact": map[string]any{"type": "string", "description": "the artifact reference, :"}, + "fingerprint": map[string]any{"type": "string"}, + "fetch": map[string]any{"type": "object", "description": "what the fetch tool is asked: the repository, the full commit and the module's directory"}, + "again": map[string]any{"type": "boolean", "description": "find them anew, not from the kept answer"}, + }}, + Run: m.Short(m.steps)} +} + +func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + fetch, _ := args["fetch"].(map[string]any) + req, err := sdkstep.ParseFetchRequest(fetch) + if err != nil { + return nil, err + } + if str("artifact") == "" || str("module") == "" { + return nil, fmt.Errorf("steps: the module and the artifact are asked") + } + steps, err := m.stepsOfTree(ctx, req) + if err != nil { + return nil, err + } + return StepsAnswer{Artifact: str("artifact"), Fingerprint: str("fingerprint"), Steps: steps}, nil +} + +// stepsOfTree fetches the commit into a workspace removed at the end and detects the steps of the module's directory. +func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]DetectedStep, error) { + if err := os.MkdirAll(m.Work, 0o700); err != nil { + return nil, err + } + root, err := os.MkdirTemp(m.Work, "steps-") + if err != nil { + return nil, err + } + defer os.RemoveAll(root) + tree := filepath.Join(root, "tree") + if err := os.MkdirAll(tree, 0o700); err != nil { + return nil, err + } + if err := buildstep.FetchCommit(ctx, m.Run, tree, m.remote(req.Repository), req.Commit, req.Ref, m.fetchOptions(ctx, req.Repository, req.Commit, req.Ref)...); err != nil { + return nil, err + } + if len(req.Paths) > 0 { + if err := buildstep.Narrow(tree, req); err != nil { + return nil, err + } + } + dir := tree + if req.Dir != "" && req.Dir != "." { + dir = filepath.Join(tree, filepath.FromSlash(req.Dir)) + } + if _, err := fs.Stat(os.DirFS(dir), "."); err != nil { + return nil, fmt.Errorf("steps: the module's directory %q is not in the commit: %w", req.Dir, err) + } + return detectSteps(dir) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go b/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go new file mode 100644 index 000000000..c2a118fbf --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + "reflect" + "testing" +) + +// Design 47 G §22, §307 (BLD1): the verb steps answers an artifact's lint, unit and integration steps by Go's +// convention and leaves out a level with no test. +func TestTheStepsVerbAnswersTheStepsOfAFixtureArtifact(t *testing.T) { + f := &fakeStep{files: map[string]string{ + "go.mod": "module example.test/mod\n\ngo 1.26\n", + "a/a_test.go": "package a\n", + "a/b_integration_test.go": "//go:build integration\n\npackage a\n", + "b/b_test.go": "//go:build !windows\n\npackage b\n", + "c/c_test.go": "//go:build integration\n\npackage c\n", + "d/d_test.go": "//go:build windows\n\npackage d\n", + "testdata/x/x_test.go": "package x\n", + "e/e.go": "package e\n", + }} + m, _ := fetchModule(t, f) + var run func(map[string]any) (any, error) + for _, tool := range m.SeatTools(m.Puller(nil, buildstep.NewFakeAsks())) { + if tool.Name == Seat+".steps" { + run = tool.Run + } + } + if run == nil { + t.Fatal("the module serves no verb steps") + } + got, err := run(map[string]any{"module": "mod", "artifact": "mod:mod", "fingerprint": "fp1", + "fetch": map[string]any{"repository": "novox/mod", "commit": commitSHA}}) + if err != nil { + t.Fatal(err) + } + want := []DetectedStep{{Step: "lint"}, + {Step: "unit", Packages: []string{"example.test/mod/a", "example.test/mod/b"}}, + {Step: "integration", Packages: []string{"example.test/mod/a", "example.test/mod/c"}}} + if a := got.(StepsAnswer); !reflect.DeepEqual(a.Steps, want) || a.Artifact != "mod:mod" { + t.Fatalf("steps = %+v, want %+v", a, want) + } + // A module with no test answers lint alone: its unit and integration levels have nothing to run. + f.files = map[string]string{"go.mod": "module example.test/mod\n", "mod/e/e.go": "package e\n"} + got, err = run(map[string]any{"module": "mod", "artifact": "mod:mod", + "fetch": map[string]any{"repository": "novox/mod", "commit": commitSHA}}) + if err != nil || !reflect.DeepEqual(got.(StepsAnswer).Steps, []DetectedStep{{Step: "lint"}}) { + t.Fatalf("a module with no test: %+v, %v", got, err) + } +} diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index b337cd644..fb3655599 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -21,7 +21,7 @@ { "name": "node-build-go", "scope": "node", - "serves": ["current", "drop", "pause", "resume"] + "serves": ["current", "drop", "pause", "resume", "steps"] } ], "consumes": [ -- 2.54.0 From 25bbd5553932ba7ae8fa2eec410096ac7685ad44 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 10:56:55 +0100 Subject: [PATCH 005/243] BLD2: the Go build seat keeps the answer of steps in detected-steps by fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of BLD2 (Go module): the state detected-steps asked by the template state, one key per artifact, answered for the same fingerprint, replaced by a newer one and by again. The drop of a key and the Node and image modules wait (drop: question to the lead; the others follow in BLD7 and BLD8). Builds: design 47 G §326, §414 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-build-go/announce.go | 2 + .../cmd/mesh-build-go/forge_test.go | 6 ++- .../mesh-build-go/cmd/mesh-build-go/main.go | 2 + .../mesh-build-go/cmd/mesh-build-go/module.go | 4 ++ .../mesh-build-go/cmd/mesh-build-go/steps.go | 28 ++++++++++++- .../cmd/mesh-build-go/steps_test.go | 39 +++++++++++++++++++ modules/mesh-build-go/module.json | 1 + 7 files changed, 79 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/announce.go b/modules/mesh-build-go/cmd/mesh-build-go/announce.go index 3550c87d1..4ee60ceae 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/announce.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/announce.go @@ -15,6 +15,8 @@ var Kept = []announce.State{ {Name: "keys", Readable: true}, {Name: "steps", Readable: true}, {Name: "pauses", Readable: true}, + // The steps detected for each artifact, kept by its fingerprint (design 47 G §326, §414). + {Name: "detected-steps", Readable: true}, // The condition keeper raises a first view of the indexer's manifests it could not get (ADR 0615 §6) from this. {Name: "findings", Readable: true}, } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go index adeda47db..bd3fa0e36 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -10,7 +10,7 @@ import ( "encoding/json" "os" "path/filepath" - "slices" + "reflect" "strings" "testing" @@ -147,7 +147,9 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test m := readForgeManifest(t) // node-tools: the module pulls its asks through the tool runner's seat pull (1911). // go-package-registry: the Go seat's read side, the fetch's first source (design 51, CAT-BG3, E2). - if want := []any{"forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !slices.Equal(m.Requires, want) { + // detected-steps: the state of the verb steps (design 47 G §326, §414), asked by the template state. + detected := map[string]any{"name": "detected-steps", "storage": "key-value", "template": "state", "options": map[string]any{"history": float64(1)}} + if want := []any{detected, "forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 15b60c48d..3eae5fca1 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -41,6 +41,8 @@ func main() { Toolchains: stdio.State(seat.StateToolchains).Put, Emit: func(t string, b any) error { return stdio.Emit(t, b) }, Steps: stdio.State(seat.StateSteps).Put, + Detected: stdio.State(seat.StateDetectedSteps).Put, + ReadDetected: readOf(stdio.State(seat.StateDetectedSteps)).Get, ReadSteps: readOf(stdio.State(seat.StateSteps)), Pauses: stdio.State(sdkstep.PausesState).Put, ReadPauses: readOf(stdio.State(sdkstep.PausesState)), diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 5fc8f74a2..1c57a3d28 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -33,6 +33,10 @@ type Module struct { // Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test. Held *buildstep.Puller ReadSteps, ReadToolchainsState StateRead + // Detected and ReadDetected are the Put and the read of the state detected-steps (design 47 G §326, §414): the + // answer of the verb steps, one key per artifact. + Detected buildstep.Put + ReadDetected func(key string) (json.RawMessage, error) // Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back. Pauses buildstep.Put ReadPauses StateRead diff --git a/modules/mesh-build-go/cmd/mesh-build-go/steps.go b/modules/mesh-build-go/cmd/mesh-build-go/steps.go index 2aa35e58d..7745cc695 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/steps.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/steps.go @@ -158,11 +158,35 @@ func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) { if str("artifact") == "" || str("module") == "" { return nil, fmt.Errorf("steps: the module and the artifact are asked") } + artifact, fp := str("artifact"), str("fingerprint") + key := seat.DetectedStepsByModuleArtifactKey(str("module"), artifactName(artifact)) + // The kept answer is the artifact's newest fingerprint's: used for the same fingerprint, replaced by a newer one + // and by again (§326). + if again, _ := args["again"].(bool); !again && fp != "" && m.ReadDetected != nil { + if raw, err := m.ReadDetected(key); err == nil && len(raw) > 0 { + if kept, err := seat.DecodeDetectedStepsByModuleArtifactValue(raw); err == nil && kept.Fingerprint == fp { + return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: kept.Steps, Kept: true}, nil + } + } + } steps, err := m.stepsOfTree(ctx, req) if err != nil { return nil, err } - return StepsAnswer{Artifact: str("artifact"), Fingerprint: str("fingerprint"), Steps: steps}, nil + if fp != "" && m.Detected != nil { + if _, err := m.Detected(key, seat.DetectedSteps{V: 1, Fingerprint: fp, Steps: steps, At: ptr(m.now())}); err != nil { + return nil, fmt.Errorf("steps: keeping the answer: %w", err) + } + } + return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: steps}, nil +} + +// artifactName is the artifact of an artifact reference :. +func artifactName(ref string) string { + if _, name, ok := strings.Cut(ref, ":"); ok { + return name + } + return ref } // stepsOfTree fetches the commit into a workspace removed at the end and detects the steps of the module's directory. @@ -196,3 +220,5 @@ func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]D } return detectSteps(dir) } + +func ptr[T any](v T) *T { return &v } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go b/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go index c2a118fbf..b60021fff 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/steps_test.go @@ -1,6 +1,8 @@ package main import ( + "context" + "encoding/json" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" "reflect" "testing" @@ -48,3 +50,40 @@ func TestTheStepsVerbAnswersTheStepsOfAFixtureArtifact(t *testing.T) { t.Fatalf("a module with no test: %+v, %v", got, err) } } + +// Design 47 G §326 (BLD2): steps answers from the kept key for the same fingerprint, a newer fingerprint and again +// replace it. +func TestTheStepsVerbKeepsItsAnswerByFingerprint(t *testing.T) { + f := &fakeStep{files: map[string]string{"go.mod": "module example.test/mod\n", "a/a_test.go": "package a\n"}} + m, _ := fetchModule(t, f) + kept := map[string]json.RawMessage{} + m.Detected = func(key string, v any) (uint64, error) { + raw, err := json.Marshal(v) + kept[key] = raw + return 1, err + } + m.ReadDetected = func(key string) (json.RawMessage, error) { return kept[key], nil } + call := func(fp string, again bool) StepsAnswer { + t.Helper() + got, err := m.steps(context.Background(), map[string]any{"module": "mod", "artifact": "mod:prog", "fingerprint": fp, "again": again, + "fetch": map[string]any{"repository": "novox/mod", "commit": commitSHA}}) + if err != nil { + t.Fatal(err) + } + return got.(StepsAnswer) + } + if a := call("fp1", false); a.Kept || len(kept) != 1 || kept["mod.prog"] == nil { + t.Fatalf("first ask: %+v, keys %v", a, kept) + } + f.files = map[string]string{"go.mod": "module example.test/mod\n"} // the tree changed, the kept answer still answers fp1 + if a := call("fp1", false); !a.Kept || len(a.Steps) != 2 { + t.Fatalf("same fingerprint: %+v", a) + } + if a := call("fp1", true); a.Kept || len(a.Steps) != 1 { + t.Fatalf("again: %+v", a) + } + f.files = map[string]string{"go.mod": "module example.test/mod\n", "a/a_test.go": "package a\n"} + if a := call("fp2", false); a.Kept || len(a.Steps) != 2 { + t.Fatalf("newer fingerprint: %+v", a) + } +} diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index fb3655599..b447e70c4 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -82,6 +82,7 @@ } ], "requires": [ + {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}}, "forge-account", "go-toolchain", "go-package-registry", -- 2.54.0 From 357967308f9ff6add29e00a690edaa94b094e9a3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:00:52 +0100 Subject: [PATCH 006/243] BLD11: the three build modules take the version from the ask as MESH_VERSION; version.numbered is no longer heard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go compile, the Node tsc/esbuild/npm and the image's buildah bud get the build argument MESH_VERSION from the version the build ask carries. The late publish on version.numbered goes (code, tests, the consumes of the Go and image manifests). The image's label and tag and a program's stamp are open (question to the lead): the design names no key or symbol. Builds: design 47 G §349, §370, §413 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/buildstep/latepublish.go | 91 ------------------- .../mesh-build-go/cmd/mesh-build-go/build.go | 11 ++- .../cmd/mesh-build-go/build_test.go | 31 +++++++ .../cmd/mesh-build-go/latepublish.go | 27 ------ .../mesh-build-go/cmd/mesh-build-go/main.go | 2 - .../cmd/mesh-build-go/manifest_test.go | 6 +- .../cmd/mesh-build-go/publish_test.go | 54 ----------- modules/mesh-build-go/module.json | 3 - .../cmd/mesh-build-image/build.go | 4 + .../cmd/mesh-build-image/build_test.go | 17 ++++ .../cmd/mesh-build-image/compose_test.go | 4 +- .../cmd/mesh-build-image/latepublish.go | 23 ----- .../cmd/mesh-build-image/main.go | 2 - .../cmd/mesh-build-image/publish_test.go | 48 ---------- modules/mesh-build-image/module.json | 1 - .../cmd/mesh-build-node/build.go | 19 +++- .../cmd/mesh-build-node/build_test.go | 26 ++++++ .../cmd/mesh-build-node/package.go | 1 + 18 files changed, 106 insertions(+), 264 deletions(-) delete mode 100644 modules/mesh-build-go/buildstep/latepublish.go delete mode 100644 modules/mesh-build-go/cmd/mesh-build-go/latepublish.go delete mode 100644 modules/mesh-build-image/cmd/mesh-build-image/latepublish.go diff --git a/modules/mesh-build-go/buildstep/latepublish.go b/modules/mesh-build-go/buildstep/latepublish.go deleted file mode 100644 index 46bf19c2f..000000000 --- a/modules/mesh-build-go/buildstep/latepublish.go +++ /dev/null @@ -1,91 +0,0 @@ -package buildstep - -// **The late publish** (novox/hq ADR 0617 §1, design 51 *The versions*, *Publishing*; CAT-BG2, CAT-BI1): a trunk build -// whose number the builder did not know yet publishes its digest unnumbered; the builder says version.numbered once it -// numbers that fingerprint, and the build module publishes the held digest again under that version. Nothing travels: -// the publish names the one part by its digest, which the holder holds, so it answers held and tags the version (the -// holder asks builder_versions first, ADR 0605 §4). The bytes never stayed on the node, so a holder that no longer holds -// the digest is said and the event offered again. Every instance may hear the same event; the publish is the same call -// under the same transfer name, so a second one is answered held. One place for every build module: mesh-build-go's -// bundles and archives on oci-artifact-registry, mesh-build-image's images on container-image-registry. - -import ( - "context" - "errors" - "fmt" - "io" - "log" - "slices" - "time" - - reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" - builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" - "git.novox.be/novox/mesh-delivery/sdk/parts" - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// NumberedPattern is the event type a build module consumes for the late publish, as its manifest's `consumes` names it. -const NumberedPattern = builds.BuilderModule + "." + builds.VersionNumberedEvent - -// errNotHere is a held part's Open: its bytes did not stay on the node. -var errNotHere = errors.New("the built bytes did not stay on the node; the holder must hold the digest") - -// LatePublish publishes a numbered build's held digest under its version on seat, from the builder's version.numbered -// body, when its kind is one of kinds. It answers false, nil for an event that is not the module's to publish: another -// kind, another seat (an event naming none is the artifact seat's, as the builder wrote it before images), or a version -// numbered with no digest. -func LatePublish(ctx context.Context, call SeatCall, body []byte, seat string, kinds ...string) (bool, error) { - n, err := builds.DecodeVersionNumbered(body) - if err != nil { - return false, err - } - on := n.Seat - if on == "" { - on = ArtifactSeat - } - if n.Digest == "" || on != seat || !slices.Contains(kinds, n.Kind) { - return false, nil - } - in := reg.PublishInput{Name: n.Name, Kind: n.Kind, Version: n.Version, Tag: n.Tag, Digest: n.Digest, - Fingerprint: n.Fingerprint, Commit: n.Commit, Transfer: TransferName(n.Name, "v"+n.Version, n.Digest)} - sent, err := OnSeat(ctx, seat, "late publish of "+n.Name+" "+n.Version+" at "+n.Digest, func() (parts.Sent, error) { - return PublishParts(ctx, call, seat, in, []parts.Part{{Digest: n.Digest, - Open: func() (io.ReadCloser, error) { return nil, errNotHere }}}) - }, nil) - if err != nil { - return false, fmt.Errorf("the late publish of %s@%s (%s): %w", n.Name, n.Version, n.Digest, err) - } - if len(sent.Parts) > 0 { - return false, fmt.Errorf("the late publish of %s@%s sent parts it should not have", n.Name, n.Version) - } - return true, nil -} - -// HearNumbered subscribes late to the builder's version.numbered, asking again until the runtime takes it, and says -// each publish and each failure; an error is the event's, which is offered again. -func HearNumbered(ctx context.Context, late func(ctx context.Context, body []byte) (bool, error)) { - handle := func(e stdio.Envelope) error { - did, err := late(ctx, e.Body) - if err != nil { - log.Printf("%v", err) - return err - } - if did { - log.Printf("published a numbered build's held digest under its version (%s)", e.Headers["x-event-id"]) - } - return nil - } - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - err := stdio.Subscribe(NumberedPattern, handle) - if err == nil { - log.Printf("listening for the builder's %s", NumberedPattern) - return - } - log.Printf("not hearing the builder's %s yet (%v); asking again in %s", NumberedPattern, err, wait) - select { - case <-ctx.Done(): - return - case <-time.After(wait): - } - } -} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index 466fae221..6a202317f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -54,7 +54,7 @@ func (m *Module) buildTool() StepTool { "workspace": map[string]any{"type": "string", "description": "the workspace id build_go_fetch answered"}, "module": map[string]any{"type": "string", "description": "the module's directory in the workspace, . for the repository's root"}, "artifact": map[string]any{"type": "string", "description": "the artifact's name in the module's manifest"}, - "version": map[string]any{"type": "string", "description": "the version the builder gave a package at the trunk's claim; absent for a candidate, whose package is not published"}, + "version": map[string]any{"type": "string", "description": "the version the build ask carries for the artifact: a Go package's tag, and the build argument MESH_VERSION of every artifact; absent for a candidate, whose package is not published"}, "seat": map[string]any{"type": "string", "description": "the registry seat the artifact is published through, as the build ask names it"}, "publish": map[string]any{"type": "boolean", "description": "true for a merged trunk commit alone: the build writes the shared Go caches; without it (a candidate) they are read beneath a throwaway layer"}, }, @@ -114,8 +114,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if art == nil { return nil, fmt.Errorf("the module %s has no artifact %q (it has %s)", manifest.Module, name, strings.Join(names, ", ")) } + // The version the build ask carries is the build's argument MESH_VERSION for every artifact (design 47 G §370). + version, _ := args["version"].(string) if art.Kind == catalogue.ArtifactPackage && art.Language == "go" { - version, _ := args["version"].(string) seat, _ := args["seat"].(string) return m.buildGoPackage(ctx, workspace, dir, *art, version, seat) } @@ -153,7 +154,11 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } - if r := run(ctx, m.buildEnv(dir), "go", cmd...); r.Status != 0 || r.Err != "" { + env := m.buildEnv(dir) + if version != "" { + env = append(env, "MESH_VERSION="+version) + } + if r := run(ctx, env, "go", cmd...); r.Status != 0 || r.Err != "" { return nil, fmt.Errorf("go build of %s: %s", art.Name, causeLine(r.Err+"\n"+r.Stderr+"\n"+r.Stdout)) } if err := m.insideModule(workspace, dir, out); err != nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go index 16461569f..29dda3fbc 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build_test.go @@ -255,3 +255,34 @@ func TestAManifestFallingBackToAPredefinedKeyBuilds(t *testing.T) { t.Errorf("result = %+v", res) } } + +// Design 47 G §370 (BLD11): a build ask carrying a version gives the compile the build argument MESH_VERSION, and one +// without gives none. +func TestTheBuildGetsTheVersionTheAskCarriesAsMeshVersion(t *testing.T) { + for _, version := range []string{"1.4.3", ""} { + m, states, calls := buildModule(t, manifestWith(goBundle), "binary") + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + if _, err := tool.Run(map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools", "version": version}); err != nil { + t.Fatal(err) + } + } + } + if st := states.waitEnd(t); st.State != sdkstep.StepPassed { + t.Fatalf("the step ended %s: %s", st.State, st.Reason) + } + got := "" + for _, r := range *calls { + if strings.HasPrefix(r.line, "go ") { + for _, e := range r.env { + if strings.HasPrefix(e, "MESH_VERSION=") { + got = strings.TrimPrefix(e, "MESH_VERSION=") + } + } + } + } + if got != version { + t.Errorf("ask version %q: the compile's MESH_VERSION = %q", version, got) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go b/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go deleted file mode 100644 index 66c6d7123..000000000 --- a/modules/mesh-build-go/cmd/mesh-build-go/latepublish.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -// The late publish (CAT-BG2): a numbered bundle or archive's held digest published under its version, by the build -// modules' one late publish (buildstep/latepublish.go). - -import ( - "context" - - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it. -const NumberedPattern = buildstep.NumberedPattern - -// LatePublish publishes a numbered bundle or archive's held digest under its version on the artifact seat; a Go package -// publishes at its build, an image is mesh-build-image's. -func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) { - call := m.Call - if call == nil { - call = stdio.Ask - } - return buildstep.LatePublish(ctx, call, body, ArtifactSeat, "bundle", "archive") -} - -// hearNumbered hears the builder's version.numbered for the late publish. -func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 3eae5fca1..bda976699 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -88,8 +88,6 @@ func main() { if m.Node != "" { // The seat's asks, pulled once Serve has the runtime's channel; a pull before it is said and tried again. go func() { _ = puller.Pull(ctx) }() - // The builder's version.numbered, for the late publish of a held digest (latepublish.go). - go m.hearNumbered(ctx) } if err := stdio.Serve("", append(m.Tools(steps), m.SeatTools(puller)...)); err != nil { logf("%v", err) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go index 57910c946..e717db4d4 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -186,9 +186,9 @@ func TestTheModuleInvokesTheArtifactSeatsVerbsAndWritesNoStore(t *testing.T) { } } } - // The late publish hears the builder's version.numbered (latepublish.go). - if !slices.Contains(m.Consumes, NumberedPattern) { - t.Errorf("consumes %v lack %s", m.Consumes, NumberedPattern) + // The version comes in the ask: no consume of version.numbered (design 47 G §349, §413). + if slices.Contains(m.Consumes, "mesh-delivery-builder.version.numbered") { + t.Errorf("consumes %v still hear version.numbered", m.Consumes) } for _, r := range m.Resources { if r["id"] == "registry" { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go index 7a37ace2b..0892e16b2 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/publish_test.go @@ -313,57 +313,3 @@ func TestEachGoPackageOfAModuleInAFolderIsPublishedUnderItsOwnModulePath(t *test } } } - -// CAT-BG2 (design 51 *The versions*): the builder's version.numbered for a bundle or an archive publishes the held -// digest under its version, sending nothing; a digest the holder no longer holds fails the event (offered again), -// and what is not this module's to publish is let pass. -func TestTheNumberedEventPublishesAHeldDigestUnderItsVersion(t *testing.T) { - digest := "sha256:" + strings.Repeat("ab", 32) - other := "sha256:" + strings.Repeat("cd", 32) - numbered := func(kind, seat, digest string) []byte { - raw, err := builds.EncodeVersionNumbered(builds.VersionNumbered{Name: "mod/tools", Kind: kind, Version: "0.1.4", - Module: "mod", Fingerprint: "fp", Commit: "c0", Digest: digest, Seat: seat}) - if err != nil { - t.Fatal(err) - } - return raw - } - for _, c := range []struct { - name string - body []byte - did, fail bool - }{ - {"a held bundle is tagged", numbered("bundle", ArtifactSeat, digest), true, false}, - {"an archive with no seat named goes to the artifact seat", numbered("archive", "", digest), true, false}, - {"a digest the holder lacks fails, sending nothing", numbered("bundle", ArtifactSeat, other), false, true}, - {"a Go package is not published late", numbered("package", GoPackageSeat, digest), false, false}, - {"an image is mesh-build-image's", numbered("image", "container-image-registry", digest), false, false}, - {"no digest, nothing to publish", numbered("bundle", ArtifactSeat, ""), false, false}, - {"an unreadable body fails", []byte(`{"name":"mod/tools"}`), false, true}, - } { - t.Run(c.name, func(t *testing.T) { - m := &Module{} - s := withStore(t, m) - s.blobs["mod/tools@"+digest] = []byte("held") - did, err := m.LatePublish(t.Context(), c.body) - if did != c.did || (err != nil) != c.fail { - t.Fatalf("did %v, err %v; want did %v, fail %v", did, err, c.did, c.fail) - } - for _, k := range s.asked { - if strings.HasSuffix(k, ".publish-part") { - t.Errorf("a late publish sent a part: %v", s.asked) - } - } - if !c.did { - return - } - if len(s.published) != 1 { - t.Fatalf("published %v", s.published) - } - p := s.published[0] - if p.Name != "mod/tools" || p.Version != "0.1.4" || p.Digest != digest || p.Fingerprint != "fp" || p.Commit != "c0" { - t.Errorf("published %+v", p) - } - }) - } -} diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index b447e70c4..7d43103f0 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -24,9 +24,6 @@ "serves": ["current", "drop", "pause", "resume", "steps"] } ], - "consumes": [ - "mesh-delivery-builder.version.numbered" - ], "reads": [ "mesh-delivery-checker.kept" ], diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build.go b/modules/mesh-build-image/cmd/mesh-build-image/build.go index 70fa9e083..79b582095 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build.go @@ -170,6 +170,10 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { argv = append(argv, "--target", art.Target) } argv = append(argv, imageArgs(manifest)...) + // The version the build ask carries is the build argument MESH_VERSION (design 47 G §370). + if version != "" { + argv = append(argv, "--build-arg", "MESH_VERSION="+version) + } argv = append(argv, contextDir) if r := m.Run(ctx, m.env(), "buildah", argv...); r.Status != 0 || r.Err != "" { m.sayFailure("buildah bud of "+art.Name+" of "+manifest.Module, r) diff --git a/modules/mesh-build-image/cmd/mesh-build-image/build_test.go b/modules/mesh-build-image/cmd/mesh-build-image/build_test.go index b886de329..7bbb850c4 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/build_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/build_test.go @@ -330,3 +330,20 @@ func TestAManifestFallingBackToAPredefinedKeyBuilds(t *testing.T) { t.Errorf("result = %+v", res) } } + +// Design 47 G §370 (BLD11): a build ask carrying a version gives the image build the build argument MESH_VERSION. +func TestTheImageBuildGetsTheVersionTheAskCarriesAsMeshVersion(t *testing.T) { + m, f, states := setup(t, same) + if _, st := runBuild(t, m, states, map[string]any{"version": "1.4.3"}); st.State != sdkstep.StepPassed { + t.Fatalf("step = %+v", st) + } + bud := f.called("bud") + if len(bud) != 1 || !strings.Contains(bud[0].line, "--build-arg MESH_VERSION=1.4.3") { + t.Fatalf("bud = %+v", bud) + } + m, f, states = setup(t, same) + runBuild(t, m, states, nil) + if bud := f.called("bud"); len(bud) != 1 || strings.Contains(bud[0].line, "MESH_VERSION") { + t.Fatalf("a build with no version: %+v", bud) + } +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go index ff7acedf9..8a09744da 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go @@ -27,8 +27,8 @@ func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) { "seat:container-image-registry.mirror", "seat:mesh-module-assigner.assignments"}; !slices.Equal(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } - // The builder's version.numbered for the late publish (ADR 0617 §1), and the seat's outcomes a copy waits on. - consumes := []string{NumberedPattern} + // The seat's outcomes a copy waits on; the version comes in the ask, so version.numbered is not heard (design 47 G §413). + consumes := []string{} for _, e := range OutcomeEvents { consumes = append(consumes, "container-image-registry."+e) } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go b/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go deleted file mode 100644 index 2e5a03c8b..000000000 --- a/modules/mesh-build-image/cmd/mesh-build-image/latepublish.go +++ /dev/null @@ -1,23 +0,0 @@ -package main - -// The late publish (novox/hq ADR 0617 §1, CAT-BI1): an image whose floor the builder did not know yet is built -// unnumbered and published by its digest; once the builder numbers it, its version.numbered publishes the held digest -// under that version through the image seat, sending nothing (buildstep/latepublish.go, the build modules' one). - -import ( - "context" - - "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" -) - -// NumberedPattern is the event type this module consumes for the late publish, as its manifest's `consumes` names it. -const NumberedPattern = buildstep.NumberedPattern - -// LatePublish publishes a numbered image's held digest under its version on the image seat; a bundle or an archive is -// mesh-build-go's. -func (m *Module) LatePublish(ctx context.Context, body []byte) (bool, error) { - return buildstep.LatePublish(ctx, m.call(), body, buildstep.ImageSeat, KindImage) -} - -// hearNumbered hears the builder's version.numbered for the late publish. -func (m *Module) hearNumbered(ctx context.Context) { buildstep.HearNumbered(ctx, m.LatePublish) } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/main.go b/modules/mesh-build-image/cmd/mesh-build-image/main.go index 82be8871b..13224f1b2 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/main.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/main.go @@ -76,8 +76,6 @@ func main() { // The image seat's outcomes, which a copy waits on (the manifest's consumes), and the copies' waits in // mirror-waits, where another instance writes an outcome it heard. go hearOutcomes(ctx, m.Outcomes) - // The builder's version.numbered, for the late publish of an image built unnumbered (ADR 0617 §1). - go m.hearNumbered(ctx) go waits.Follow("", m.Outcomes.Changed) } if m.Node != "" { diff --git a/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go b/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go index c7e51a66a..554635c50 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/publish_test.go @@ -9,7 +9,6 @@ import ( "testing" reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" - builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" ) @@ -123,50 +122,3 @@ func TestATrunkBuildPublishesThroughTheImageSeatOnlyWhatTheHolderLacks(t *testin t.Errorf("a tag that is not one: step = %+v", st) } } - -// ADR 0617 §1: the builder's version.numbered for an image publishes the held digest under its version through the image -// seat, sending nothing; a digest the holder lacks fails the event (offered again); what is not this module's passes. -func TestTheNumberedEventPublishesAHeldImageUnderItsVersion(t *testing.T) { - digest := "sha256:" + strings.Repeat("ab", 32) - numbered := func(kind, seat, digest string) []byte { - raw, err := builds.EncodeVersionNumbered(builds.VersionNumbered{Name: "fixture-image/server", Kind: kind, Version: "1.4.0", - Module: "fixture-image", Fingerprint: "fp", Commit: "c0", Digest: digest, Seat: seat}) - if err != nil { - t.Fatal(err) - } - return raw - } - for name, c := range map[string]struct { - body []byte - did, fail bool - }{ - "a held image is numbered": {body: numbered("image", reg.Seat, digest), did: true}, - "a digest the holder lacks fails": {body: numbered("image", reg.Seat, "sha256:"+strings.Repeat("cd", 32)), fail: true}, - "a bundle is mesh-build-go's": {body: numbered("bundle", "oci-artifact-registry", digest)}, - "an event naming no seat is a bundle's": {body: numbered("image", "", digest)}, - "no digest, nothing to publish": {body: numbered("image", reg.Seat, "")}, - } { - t.Run(name, func(t *testing.T) { - m, f, _ := setup(t, same) - f.held[digest] = true - did, err := m.LatePublish(t.Context(), c.body) - if did != c.did || (err != nil) != c.fail { - t.Fatalf("did %v, err %v; want did %v, fail %v", did, err, c.did, c.fail) - } - if len(f.sent) != 0 { - t.Errorf("a late publish sent parts: %v", f.sent) - } - if !c.did { - return - } - if len(f.published) != 1 { - t.Fatalf("published %+v", f.published) - } - p := f.published[0] - if p.Name != "fixture-image/server" || p.Kind != KindImage || p.Version != "1.4.0" || p.Digest != digest || - p.Fingerprint != "fp" || p.Commit != "c0" || len(p.Parts) != 1 || p.Parts[0].Digest != digest { - t.Errorf("published %+v", p) - } - }) - } -} diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index ac256f530..ec0de87d9 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -68,7 +68,6 @@ "seat:mesh-module-assigner.assignments" ], "consumes": [ - "mesh-delivery-builder.version.numbered", "container-image-registry.published", "container-image-registry.refused" ], diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build.go b/modules/mesh-build-node/cmd/mesh-build-node/build.go index 4c7bb7da0..4147c76c5 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build.go @@ -110,8 +110,9 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if err != nil { return nil, err } + // The version the build ask carries is the build argument MESH_VERSION of every artifact (design 47 G §370). + version, _ := args["version"].(string) if a.Kind == "package" { - version, _ := args["version"].(string) seat, _ := args["seat"].(string) at := a.packageDir(dir) return m.buildPackage(ctx, workspace, module, root, filepath.Join(dir, filepath.FromSlash(at)), at, a, publish, version, seat) @@ -145,14 +146,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { tsc := []string{m.tsc(), "--module", "NodeNext", "--moduleResolution", "NodeNext", "--target", "ES2022", "--rootDir", ".", "--outDir", out} tsc = append(tsc, sourcesFor(a.Entrypoints, out, ".ts")...) - if why := ranFailed(m.runIn(ctx, root, publish, dir, nil, "node", tsc...)); why != "" { + if why := ranFailed(m.runIn(ctx, root, publish, dir, versionEnv(version), "node", tsc...)); why != "" { return nil, fmt.Errorf("tsc for %s: %s", a.Name, why) } launchers, err := writeLaunchers(filepath.Join(dir, out), a) if err != nil { return nil, err } - if err := m.bundle(ctx, root, publish, dir, a, launchers); err != nil { + if err := m.bundle(ctx, root, publish, dir, a, launchers, version); err != nil { return nil, err } bundleDir := filepath.Join(dir, final) @@ -185,7 +186,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { // bundle makes every entrypoint and every launcher ONE file, with esbuild (builder.go:1971-2039): an entrypoint is // bundled in place under its own name, a launcher keeps its name with the extension .mjs. -func (m *Module) bundle(ctx context.Context, root string, publish bool, dir string, a artifact, launchers map[string]string) error { +func (m *Module) bundle(ctx context.Context, root string, publish bool, dir string, a artifact, launchers map[string]string, version string) error { out, final := Out(a.Name), Out(a.Name)+".bundled" if err := os.RemoveAll(filepath.Join(dir, final)); err != nil { return err @@ -212,7 +213,7 @@ func (m *Module) bundle(ctx context.Context, root string, publish bool, dir stri return nil } args := append(append(append([]string{}, entries...), common...), extra...) - if why := ranFailed(m.runIn(ctx, root, publish, dir, nil, m.esbuild(), args...)); why != "" { + if why := ranFailed(m.runIn(ctx, root, publish, dir, versionEnv(version), m.esbuild(), args...)); why != "" { return fmt.Errorf("esbuild for %s: %s", a.Name, why) } return nil @@ -319,3 +320,11 @@ func readArtifact(dir, name string) (artifact, error) { } return artifact{}, fmt.Errorf("module.json declares no artifact %q", name) } + +// versionEnv is the build argument MESH_VERSION for the version the build ask carries, none when it carries none. +func versionEnv(version string) []string { + if version == "" { + return nil + } + return []string{"MESH_VERSION=" + version} +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/build_test.go b/modules/mesh-build-node/cmd/mesh-build-node/build_test.go index 43fe83cc0..469482168 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/build_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/build_test.go @@ -302,3 +302,29 @@ func TestRanFailedKeepsTheError(t *testing.T) { } } } + +// Design 47 G §370 (BLD11): a build ask carrying a version gives the compiler and the bundler the build argument +// MESH_VERSION; one without gives none. +func TestTheBundleBuildGetsTheVersionTheAskCarriesAsMeshVersion(t *testing.T) { + for _, version := range []string{"1.4.3", ""} { + m, f, _ := builtMachine(t, "") + args := buildArgs() + if version != "" { + args["version"] = version + } + if _, err := m.build(context.Background(), args); err != nil { + t.Fatal(err) + } + for _, c := range f.calls { + got := "" + for _, e := range c.env { + if v, ok := strings.CutPrefix(e, "MESH_VERSION="); ok { + got = v + } + } + if got != version { + t.Errorf("ask version %q: %s ran with MESH_VERSION=%q", version, c.name, got) + } + } + } +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/package.go b/modules/mesh-build-node/cmd/mesh-build-node/package.go index 9da43efab..4a994da7e 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/package.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/package.go @@ -201,6 +201,7 @@ func (m *Module) buildAndPack(ctx context.Context, root, dir, rel string, write return packed{}, err } env := []string{"PATH=" + shimDir(root, rel) + ":" + pathOrDefault(), "npm_config_cache=" + m.npmCache()} + env = append(env, versionEnv(version)...) if p.Scripts["build"] != "" { if why := ranFailed(m.runIn(ctx, root, write, dir, env, "npm", "run", "build")); why != "" { return packed{}, fmt.Errorf("the build of the package %s: %s", p.Name, why) -- 2.54.0 From dfb682fa33794691a466916efcac8a016a9cd5fd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:02:03 +0100 Subject: [PATCH 007/243] BLD14: a run's input is asked of the verb the asker's ask names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Go build module asks the input verb in the ask (the checker's, the indexer's mesh-catalog input or the assigner's mesh-wants input) and its manifest invokes the two seats' verbs. The bus refuses the other accounts (grants of the contract's callers). The step key keeps the contract's form .: no asker part is named there (question to the lead). Builds: design 47 G §125, §206, §338, §354 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/cmd/mesh-build-go/run.go | 6 ++- .../cmd/mesh-build-go/run_test.go | 42 +++++++++++++++++++ modules/mesh-build-go/module.json | 2 + 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run.go b/modules/mesh-build-go/cmd/mesh-build-go/run.go index fc11b7998..06aa9979c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run.go @@ -63,8 +63,10 @@ func (m *Module) inputOf(in sdkstep.RunInput) ([]byte, error) { } // A verb that does not answer fails the step; an answer that is not the input named refuses the ask. var askErr error - raw, err := checkerclient.ReadInput(func(key string, body any) (json.RawMessage, error) { - out, err := m.AskInput(key, body) + // The verb is the one the asker's ask names: the runner's, the indexer's (mesh-catalog's input) or the assigner's + // (mesh-wants' input) own, each serving the input by its digest alike (design 47 G §125, §206, §338, §354). + raw, err := checkerclient.ReadInput(func(_ string, body any) (json.RawMessage, error) { + out, err := m.AskInput(in.Verb, body) askErr = err return out, err }, in.Digest) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go index a5ca2dd0c..09c98d6b7 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/run_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/run_test.go @@ -10,6 +10,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "testing" "time" @@ -154,3 +155,44 @@ func TestAComposeAskKeepsItsSeats(t *testing.T) { t.Fatalf("the composer did not receive the seats: %+v %v", v, err) } } + +// Design 47 G §125, §206, §338, §354 (BLD14): a run's input is asked of the verb the ask names, so the indexer's and the +// assigner's runs read their own seat's input, and the manifest invokes both. +func TestARunsInputIsAskedOfTheVerbTheAskNames(t *testing.T) { + for _, verb := range []string{"seat:mesh-catalog.input", "seat:mesh-wants.input", runInputVerb} { + m := composeModule(t) + body := `{"v":1}` + var asked []string + m.AskInput = func(key string, b any) (json.RawMessage, error) { + asked = append(asked, key) + return answersInputOn(verb, body)(key, b) + } + a := runAskOf(t, body) + in, err := sdkstep.RunInputOf(verb, []byte(body)) + if err != nil { + t.Fatal(err) + } + a.Run.Input = in + got, err := m.inputOf(a.Run.Input) + if err != nil || string(got) != body || len(asked) != 1 || asked[0] != verb { + t.Errorf("%s: input %q, err %v, asked %v", verb, got, err, asked) + } + } + doc := readManifest(t) + for _, verb := range []string{"seat:mesh-catalog.input", "seat:mesh-wants.input"} { + if !slices.Contains(doc.Invokes, verb) { + t.Errorf("the manifest does not invoke %s: %v", verb, doc.Invokes) + } + } +} + +// answersInputOn is answersInput for the seat verb wanted. +func answersInputOn(wanted, body string) func(key string, b any) (json.RawMessage, error) { + return func(key string, b any) (json.RawMessage, error) { + ask, _ := b.(checker.InputInput) + if sum := sha256.Sum256([]byte(body)); key != wanted || ask.Digest != hex.EncodeToString(sum[:]) { + return nil, fmt.Errorf("%s keeps no input %s", key, ask.Digest) + } + return json.Marshal(checkerclient.InputAnswerOf([]byte(body))) + } +} diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 7d43103f0..ee50a37dd 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -34,6 +34,8 @@ "seat:go-package-registry.publish", "seat:go-package-registry.publish-part", "seat:mesh-delivery-checker.input", + "seat:mesh-catalog.input", + "seat:mesh-wants.input", "seat:mesh-module-assigner.assignments" ], "config": { -- 2.54.0 From b7dab47e88f41009f5d6a8c8040c5c111ef9c85c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:03:11 +0100 Subject: [PATCH 008/243] BLD15: the Go build module declares its finding stream and hears the keeper's ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Part of BLD15: the manifest's stream mesh-build-go-findings (the three finding events, 7 days, 16 MiB) and the consume of mesh-conditions.findings-asked. The module's writing of findings through mesh-sdk's finding writer waits for the writer (the writer's API is an open design question, D-16). Builds: design 47 G §383, §384, §387 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/cmd/mesh-build-go/emit_test.go | 2 +- .../cmd/mesh-build-go/manifest_test.go | 13 +++++++++++++ modules/mesh-build-go/module.json | 9 +++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go b/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go index 71fb11d53..46112afd6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/emit_test.go @@ -49,7 +49,7 @@ func TestTheManifestDeclaresTheToolchainsStream(t *testing.T) { Events []string `json:"events"` } `json:"streams"` } - if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) != 1 || m.Streams[0].Name != "toolchains-go" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) { + if err := json.Unmarshal(raw, &m); err != nil || len(m.Streams) < 1 || m.Streams[0].Name != "toolchains-go" || !slices.Equal(m.Streams[0].Events, []string{"toolchain.**"}) { t.Errorf("streams = %+v (%v)", m.Streams, err) } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go index e717db4d4..3e89059cf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/manifest_test.go @@ -190,6 +190,19 @@ func TestTheModuleInvokesTheArtifactSeatsVerbsAndWritesNoStore(t *testing.T) { if slices.Contains(m.Consumes, "mesh-delivery-builder.version.numbered") { t.Errorf("consumes %v still hear version.numbered", m.Consumes) } + // An observer says its findings on a stream of its own and hears the keeper's ask (design 47 G §383, §384, §387). + if !slices.Contains(m.Consumes, "mesh-conditions.findings-asked") { + t.Errorf("consumes %v lack mesh-conditions.findings-asked", m.Consumes) + } + found := false + for _, st := range m.Streams { + if st.Name == "mesh-build-go-findings" { + found = slices.Equal(st.Events, []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"}) && st.MaxAgeDays == 7 && st.MaxMiB == 16 + } + } + if !found { + t.Errorf("streams %+v lack mesh-build-go-findings (7 days, 16 MiB)", m.Streams) + } for _, r := range m.Resources { if r["id"] == "registry" { t.Errorf("the store's address is still placed: %v", r) diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index ee50a37dd..c513d1438 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -24,6 +24,9 @@ "serves": ["current", "drop", "pause", "resume", "steps"] } ], + "consumes": [ + "mesh-conditions.findings-asked" + ], "reads": [ "mesh-delivery-checker.kept" ], @@ -78,6 +81,12 @@ { "name": "toolchains-go", "events": ["toolchain.**"] + }, + { + "name": "mesh-build-go-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 } ], "requires": [ -- 2.54.0 From 4ad5f408177ec64263755b08953733b0795d78d0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:06:40 +0100 Subject: [PATCH 009/243] BLD7, BLD8: the Node and image build seats serve the verb steps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node answers the type-check as lint always, unit where a package has a test script and integration where it has test:integration, kept in detected-steps by fingerprint as the Go module keeps it; the image answers none. The manifests serve it. The step ask's end (step-ended) of BLD7 and BLD8 waits for BLD3 and the build-step package of mesh-delivery-sdk. Builds: design 47 G §22, §307, §326, §328 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-build-image/asks.go | 2 +- .../cmd/mesh-build-image/manifest_test.go | 2 +- .../cmd/mesh-build-image/steps.go | 41 +++++ .../cmd/mesh-build-image/steps_test.go | 24 +++ modules/mesh-build-image/module.json | 2 +- .../cmd/mesh-build-node/announce.go | 1 + .../cmd/mesh-build-node/asks.go | 2 +- .../cmd/mesh-build-node/main.go | 2 + .../cmd/mesh-build-node/manifest_test.go | 2 +- .../cmd/mesh-build-node/module.go | 4 + .../cmd/mesh-build-node/steps.go | 159 ++++++++++++++++++ .../cmd/mesh-build-node/steps_test.go | 53 ++++++ modules/mesh-build-node/module.json | 3 +- 13 files changed, 291 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-build-image/cmd/mesh-build-image/steps.go create mode 100644 modules/mesh-build-image/cmd/mesh-build-image/steps_test.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/steps.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/steps_test.go diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index 015c25ab1..ca33045ad 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -77,7 +77,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // SeatTools are the seat's verbs on this node (ADR 0525 §10). func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool { - return buildstep.SeatTools(Seat, "image", p) + return append(buildstep.SeatTools(Seat, "image", p), stepsTool()) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go index f3c2cd6e2..8419a7a08 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go @@ -125,7 +125,7 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if want := []string{"keys", MirrorWaitsState, "pauses", "steps"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) diff --git a/modules/mesh-build-image/cmd/mesh-build-image/steps.go b/modules/mesh-build-image/cmd/mesh-build-image/steps.go new file mode 100644 index 000000000..e62ff9364 --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/steps.go @@ -0,0 +1,41 @@ +package main + +// **The verb `steps`** (design 47 G §22, §307, §328): an image has no lint, unit or integration test step, so `steps` +// answers none for it and the seed has those steps `bypass` (*A module's registration*). The answer is the same for +// every fingerprint, so nothing is kept for it. + +import ( + "fmt" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// StepsAnswer is what `steps` answers. +type StepsAnswer struct { + Artifact string `json:"artifact"` + Fingerprint string `json:"fingerprint,omitempty"` + Steps []seat.DetectedStep `json:"steps"` +} + +// stepsTool is the seat's verb `steps`. +func stepsTool() stdio.Tool { + return stdio.Tool{Name: Seat + "." + seat.VerbSteps, + Description: "Which pipeline steps an image artifact can run: none (an image has no lint, unit or integration test step).", + Input: map[string]any{"type": "object", "required": []string{"module", "artifact"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, + "artifact": map[string]any{"type": "string", "description": "the artifact reference, :"}, + "fingerprint": map[string]any{"type": "string"}, + "fetch": map[string]any{"type": "object", "description": "not read: an image's steps need no tree"}, + "again": map[string]any{"type": "boolean", "description": "not read: the answer is the same anew"}, + }}, + Run: steps} +} + +func steps(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + if str("artifact") == "" || str("module") == "" { + return nil, fmt.Errorf("steps: the module and the artifact are asked") + } + return StepsAnswer{Artifact: str("artifact"), Fingerprint: str("fingerprint"), Steps: []seat.DetectedStep{}}, nil +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/steps_test.go b/modules/mesh-build-image/cmd/mesh-build-image/steps_test.go new file mode 100644 index 000000000..b24fea3e4 --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/steps_test.go @@ -0,0 +1,24 @@ +package main + +import "testing" + +// Design 47 G §22, §328 (BLD8): the verb steps is served and answers none for an image. +func TestTheStepsVerbAnswersNoStepForAnImage(t *testing.T) { + served := false + for _, tool := range (&Module{}).SeatTools(nil) { + if tool.Name == Seat+".steps" { + served = true + got, err := tool.Run(map[string]any{"module": "m", "artifact": "m:server", "fingerprint": "fp"}) + a, ok := got.(StepsAnswer) + if err != nil || !ok || a.Artifact != "m:server" || a.Steps == nil || len(a.Steps) != 0 { + t.Errorf("steps = %+v (%v)", got, err) + } + if _, err := tool.Run(map[string]any{"module": "m"}); err == nil { + t.Error("an ask with no artifact was answered") + } + } + } + if !served { + t.Fatal("the module serves no verb steps") + } +} diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index ec0de87d9..be406b135 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -21,7 +21,7 @@ { "name": "node-build-image", "scope": "node", - "serves": ["current", "drop", "pause", "resume"] + "serves": ["current", "drop", "pause", "resume", "steps"] } ], "config": { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/announce.go b/modules/mesh-build-node/cmd/mesh-build-node/announce.go index 2163bca16..642d81767 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/announce.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/announce.go @@ -15,6 +15,7 @@ var Kept = []announce.State{ {Name: "keys", Readable: true}, {Name: "steps", Readable: true}, {Name: "pauses", Readable: true}, + {Name: "detected-steps", Readable: true}, } // keepStates announces Kept as this module's (stdio.Keeps): main calls it before it serves. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index 93147db9a..5161802e3 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -87,7 +87,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // SeatTools are the seat's verbs on this node (ADR 0525 §10). func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool { - return buildstep.SeatTools(Seat, "Node", p) + return append(buildstep.SeatTools(Seat, "Node", p), m.stepsTool()) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/main.go b/modules/mesh-build-node/cmd/mesh-build-node/main.go index 772abe1cb..ad83cc12f 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/main.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/main.go @@ -48,6 +48,8 @@ func main() { Emit: func(t string, b any) error { return stdio.Emit(t, b) }, Steps: stdio.State(seat.StateSteps).Put, ReadSteps: readOf(stdio.State(seat.StateSteps)), + Detected: stdio.State(seat.StateDetectedSteps).Put, + ReadDetected: readOf(stdio.State(seat.StateDetectedSteps)).Get, Pauses: stdio.State(sdkstep.PausesState).Put, ReadPauses: readOf(stdio.State(sdkstep.PausesState)), ForgetPause: stdio.State(sdkstep.PausesState).Delete, diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 06b8939d9..131d1c15c 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -117,7 +117,7 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if want := []string{"keys", "pauses", "steps", "toolchains"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } if !slices.Contains(m.Tools, HealthTool) { diff --git a/modules/mesh-build-node/cmd/mesh-build-node/module.go b/modules/mesh-build-node/cmd/mesh-build-node/module.go index a98ca9209..f87c17b98 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/module.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/module.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "fmt" stdio "git.novox.be/novox/mesh-sdk/go" "net/http" @@ -47,6 +48,9 @@ type Module struct { // Held is the puller whose running asks a step-end tool will not end (issue 1890); nil in a test. Held *buildstep.Puller ReadSteps, ReadToolchainsState StateRead + // Detected and ReadDetected are the Put and the read of the state detected-steps (design 47 G §326, §414). + Detected buildstep.Put + ReadDetected func(key string) (json.RawMessage, error) // Pauses is the Put of the pauses state, which keeps the node's pause key without an age; ReadPauses reads it back. Pauses buildstep.Put ReadPauses StateRead diff --git a/modules/mesh-build-node/cmd/mesh-build-node/steps.go b/modules/mesh-build-node/cmd/mesh-build-node/steps.go new file mode 100644 index 000000000..0f109953f --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/steps.go @@ -0,0 +1,159 @@ +package main + +// **The verb `steps`** (design 47 G §22, §307, §326, §328): which pipeline steps a TypeScript artifact can run, by the +// convention check.go tests by. The type-check is the lint step (§328) and is always one; the unit step is the packages +// holding a package.json script `test`, the integration step those holding `test:integration`. A level with none is not +// answered. The answer is kept in the state detected-steps by the artifact's fingerprint, replaced by a newer one and by +// again, as the Go build module keeps it. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// The pipeline steps `steps` answers (the contract's DetectedStep enum). +const ( + StepLint = "lint" + StepUnit = "unit" + StepIntegration = "integration" +) + +// DetectedStep is the contract's. +type DetectedStep = seat.DetectedStep + +// StepsAnswer is what `steps` answers. +type StepsAnswer struct { + Artifact string `json:"artifact"` + Fingerprint string `json:"fingerprint,omitempty"` + Steps []DetectedStep `json:"steps"` + Kept bool `json:"kept,omitempty"` +} + +// detectSteps finds the steps of the package directories under dir. +func detectSteps(dir string) ([]DetectedStep, error) { + targets, err := testTargets(dir, ".") + if err != nil { + return nil, err + } + unit, integration := []string{}, []string{} + for _, tg := range targets { + scripts, err := packageScripts(tg.dir) + if err != nil { + return nil, err + } + if scripts["test"] != "" { + unit = append(unit, tg.where) + } + if scripts["test:integration"] != "" { + integration = append(integration, tg.where) + } + } + steps := []DetectedStep{{Step: StepLint}} + for _, l := range []struct { + step string + pkgs []string + }{{StepUnit, unit}, {StepIntegration, integration}} { + if len(l.pkgs) > 0 { + sort.Strings(l.pkgs) + steps = append(steps, DetectedStep{Step: l.step, Packages: l.pkgs}) + } + } + return steps, nil +} + +// stepsTool is the seat's verb `steps`. +func (m *Module) stepsTool() stdio.Tool { + return stdio.Tool{Name: Seat + "." + seat.VerbSteps, + Description: "Which pipeline steps a TypeScript artifact can run: the type-check as lint, always; unit tests where a package has a test script; " + + "integration tests where one has a test:integration script. A level with no test is not answered.", + Input: map[string]any{"type": "object", "required": []string{"module", "artifact", "fetch"}, "properties": map[string]any{ + "module": map[string]any{"type": "string"}, + "artifact": map[string]any{"type": "string", "description": "the artifact reference, :"}, + "fingerprint": map[string]any{"type": "string"}, + "fetch": map[string]any{"type": "object", "description": "what the fetch tool is asked: the repository, the full commit and the module's directory"}, + "again": map[string]any{"type": "boolean", "description": "find them anew, not from the kept answer"}, + }}, + Run: m.Short(m.steps)} +} + +func (m *Module) steps(ctx context.Context, args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + fetch, _ := args["fetch"].(map[string]any) + req, err := sdkstep.ParseFetchRequest(fetch) + if err != nil { + return nil, err + } + if str("artifact") == "" || str("module") == "" { + return nil, fmt.Errorf("steps: the module and the artifact are asked") + } + artifact, fp := str("artifact"), str("fingerprint") + name := artifact + if _, n, ok := strings.Cut(artifact, ":"); ok { + name = n + } + key := seat.DetectedStepsByModuleArtifactKey(str("module"), name) + if again, _ := args["again"].(bool); !again && fp != "" && m.ReadDetected != nil { + if raw, err := m.ReadDetected(key); err == nil && len(raw) > 0 { + if kept, err := seat.DecodeDetectedStepsByModuleArtifactValue(raw); err == nil && kept.Fingerprint == fp { + return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: kept.Steps, Kept: true}, nil + } + } + } + steps, err := m.stepsOfTree(ctx, req) + if err != nil { + return nil, err + } + if fp != "" && m.Detected != nil { + at := time.Now() + if m.Clock != nil { + at = m.Clock() + } + if _, err := m.Detected(key, seat.DetectedSteps{V: 1, Fingerprint: fp, Steps: steps, At: &at}); err != nil { + return nil, fmt.Errorf("steps: keeping the answer: %w", err) + } + } + return StepsAnswer{Artifact: artifact, Fingerprint: fp, Steps: steps}, nil +} + +// stepsOfTree fetches the commit alone (nothing installed) into a workspace removed at the end. +func (m *Module) stepsOfTree(ctx context.Context, req sdkstep.FetchRequest) ([]DetectedStep, error) { + if err := os.MkdirAll(m.Work, 0o700); err != nil { + return nil, err + } + root, err := os.MkdirTemp(m.Work, "steps-") + if err != nil { + return nil, err + } + defer os.RemoveAll(root) + tree := filepath.Join(root, "tree") + if err := os.MkdirAll(tree, 0o700); err != nil { + return nil, err + } + gitRun := func(ctx context.Context, env []string, name string, a ...string) forgekey.Ran { + return m.Run(ctx, append(m.baseEnv(), env...), name, a...) + } + if err := buildstep.FetchCommit(ctx, gitRun, tree, m.remote(req.Repository), req.Commit, req.Ref, m.FetchOptions...); err != nil { + return nil, err + } + if len(req.Paths) > 0 { + if err := buildstep.Narrow(tree, req); err != nil { + return nil, err + } + } + dir := tree + if req.Dir != "" && req.Dir != "." { + dir = filepath.Join(tree, filepath.FromSlash(req.Dir)) + } + return detectSteps(dir) +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/steps_test.go b/modules/mesh-build-node/cmd/mesh-build-node/steps_test.go new file mode 100644 index 000000000..0e7accaaa --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/steps_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "reflect" + "testing" +) + +// Design 47 G §22, §307, §328 (BLD7): steps answers the type-check as lint always, the packages with a test script as +// unit, those with test:integration as integration, and leaves out a level with none. +func TestTheStepsOfATypeScriptArtifactAreFoundByItsScripts(t *testing.T) { + put := func(dir, name, body string) { + if err := os.MkdirAll(filepath.Join(dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + for name, c := range map[string]struct { + pkg string + want []DetectedStep + }{ + "both levels": {`{"scripts":{"test":"node t.js","test:integration":"node i.js"}}`, + []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}, {Step: "integration", Packages: []string{"."}}}}, + "unit only": {`{"scripts":{"test":"node t.js"}}`, []DetectedStep{{Step: "lint"}, {Step: "unit", Packages: []string{"."}}}}, + "no script": {`{"scripts":{}}`, []DetectedStep{{Step: "lint"}}}, + } { + dir := t.TempDir() + put(dir, "package.json", c.pkg) + got, err := detectSteps(dir) + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Errorf("%s: steps = %+v (%v), want %+v", name, got, err, c.want) + } + } +} + +// The verb is served beside the seat's others and refuses an ask that names no artifact. +func TestTheStepsVerbIsServedAndRefusesAnAskWithoutAnArtifact(t *testing.T) { + m, _, _ := newMachine(t) + served := false + for _, tool := range m.SeatTools(m.Puller(nil, nil)) { + served = served || tool.Name == Seat+".steps" + } + if !served { + t.Fatal("the module serves no verb steps") + } + if _, err := m.steps(context.Background(), map[string]any{"module": "m", "fetch": map[string]any{"repository": "novox/m", "commit": "0123456789abcdef0123456789abcdef01234567"}}); err == nil { + t.Error("an ask with no artifact was answered") + } +} diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 0e84060bf..4d9a5936b 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -21,7 +21,7 @@ { "name": "node-build-node", "scope": "node", - "serves": ["current", "drop", "pause", "resume"] + "serves": ["current", "drop", "pause", "resume", "steps"] } ], "config": { @@ -65,6 +65,7 @@ } ], "requires": [ + {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}}, "forge-account", "npm-package-registry", "node-toolchain", -- 2.54.0 From b5aebbbe912802ef58575ab4af93b1b2dd562816 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:27:17 +0100 Subject: [PATCH 010/243] GEN4: the seat generator emits the types of bodies, resolves #/bodies/ refs and names an accepted kind by its stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G-9 §452, §454, §455 (§391: no verb is refused for a missing class) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/buildstep/seatgen/gogen.go | 35 +++++++++++++--- .../buildstep/seatgen/layout_test.go | 40 +++++++++++++++++++ .../contracts/seats/a-registry/seat.json | 8 +++- .../mesh-build-go/buildstep/seatgen/tsgen.go | 4 +- .../mesh-build-go/buildstep/seatgen/types.go | 6 ++- 5 files changed, 83 insertions(+), 10 deletions(-) diff --git a/modules/mesh-build-go/buildstep/seatgen/gogen.go b/modules/mesh-build-go/buildstep/seatgen/gogen.go index ec0382747..a1179943f 100644 --- a/modules/mesh-build-go/buildstep/seatgen/gogen.go +++ b/modules/mesh-build-go/buildstep/seatgen/gogen.go @@ -79,6 +79,12 @@ func declareAll(d seatdef.Definition, fromPart map[string]string) (*typer, error return nil, err } } + // A body of `bodies` yields its type alone: the sending is mesh-sdk's (design 47 G §452, §454). + for _, n := range sortedKeys(d.Bodies) { + if err := decl(n, "", d.Bodies[n]); err != nil { + return nil, err + } + } for _, v := range d.Verbs { if err := decl(ident(v.Name)+"Input", "is what "+v.Name+" takes.", v.Input); err != nil { return nil, err @@ -88,7 +94,7 @@ func declareAll(d seatdef.Definition, fromPart map[string]string) (*typer, error } } for _, a := range d.Accepts { - if err := decl(ident(a.Name)+"Accept", "is the body of "+a.Name+".", a.Body); err != nil { + if err := decl(ident(acceptName(a))+"Accept", "is the body of "+acceptName(a)+".", a.Body); err != nil { return nil, err } } @@ -171,7 +177,7 @@ func goWhole(m *Module, d seatdef.Definition) ([]byte, []byte, error) { } fmt.Fprintf(&b, "// Seat is the seat's name.\nconst Seat = %q\n\n", d.Seat) writeNames(&b, "Verb", seatdef.VerbNames(d.Verbs)) - writeNames(&b, "Accept", seatdef.MessageNames(d.Accepts)) + writeNames(&b, "Accept", acceptNames(d.Accepts)) writeNames(&b, "Event", seatdef.MessageNames(d.Events)) writeList(&b, "Verbs", "Verb", "every verb the seat serves, in its definition's order", seatdef.VerbNames(d.Verbs)) writeList(&b, "Events", "Event", "every event published on the seat's subject", seatdef.MessageNames(d.Events)) @@ -195,12 +201,12 @@ func goWhole(m *Module, d seatdef.Definition) ([]byte, []byte, error) { } } for _, a := range d.Accepts { - subj := seatdef.AcceptSubject(d.Seat, a.Name) + subj := seatdef.AcceptSubject(d.Seat, acceptName(a)) if a.ByCaller { fmt.Fprintf(&b, "// AcceptSubject%s is where one caller sends %s.\nfunc AcceptSubject%s(caller string) string { return %q + caller }\n\n", - ident(a.Name), a.Name, ident(a.Name), subj+".") + ident(acceptName(a)), acceptName(a), ident(acceptName(a)), subj+".") } else { - fmt.Fprintf(&b, "// AcceptSubject%s is where %s is sent.\nconst AcceptSubject%s = %q\n\n", ident(a.Name), a.Name, ident(a.Name), subj) + fmt.Fprintf(&b, "// AcceptSubject%s is where %s is sent.\nconst AcceptSubject%s = %q\n\n", ident(acceptName(a)), acceptName(a), ident(acceptName(a)), subj) } } if len(d.HolderEvents) > 0 { @@ -430,7 +436,7 @@ func fixtureTarget(d seatdef.Definition, kind, name string) (string, string, err ms, subj, suffix = d.Events, seatdef.EventSubject(d.Seat, name), "Event" } for _, msg := range ms { - if msg.Name == name { + if (kind == "event" && msg.Name == name) || (kind == "accept" && acceptName(msg) == name) { if kind == "event" && len(msg.Parts()) > 0 { return ident(name) + suffix, partsPattern(d.Seat, msg), nil } @@ -460,6 +466,23 @@ func fixtureTarget(d seatdef.Definition, kind, name string) (string, string, err return "", "", fmt.Errorf("the definition has no %s %q", kind, name) } +// acceptName is an accepted kind's name as the generated names and its subject carry it: the plain kind, or for a +// kind on a stream `.` (design 47 G §455), so `step` on `check` and on `delivery` stay two. +func acceptName(m seatdef.Message) string { + if m.Stream != "" { + return m.Stream + "." + m.Name + } + return m.Name +} + +func acceptNames(ms []seatdef.Message) []string { + out := make([]string, len(ms)) + for i, m := range ms { + out[i] = acceptName(m) + } + return out +} + func regexp(s string) string { var b strings.Builder for _, r := range s { diff --git a/modules/mesh-build-go/buildstep/seatgen/layout_test.go b/modules/mesh-build-go/buildstep/seatgen/layout_test.go index 6c714b9a2..ebef2b555 100644 --- a/modules/mesh-build-go/buildstep/seatgen/layout_test.go +++ b/modules/mesh-build-go/buildstep/seatgen/layout_test.go @@ -337,3 +337,43 @@ func TestADefinitionsActsSubjectsKeepAndCallersAreRead(t *testing.T) { } } } + +// TestBodiesAreTypesAndAcceptedKindsRideStreams is design 47 G §452, §454, §455 and §391's check for the generator: a +// definition's `bodies` yield their types in both languages and no sending code, a reference into `bodies` resolves +// as one into `defs`, an accepted kind on a stream has the subject `.` (`step` on both streams stays +// two), and a verb naming no class is refused by nothing. +func TestBodiesAreTypesAndAcceptedKindsRideStreams(t *testing.T) { + m, err := Load("testdata/contracts", "example.org/contracts/go", "@example/contracts") + if err != nil { + t.Fatal(err) + } + files, err := Generate(m) + if err != nil { + t.Fatal(err) + } + src := string(files["go/seats/aregistry/contract.go"]) + for _, want := range []string{ + "type Finding struct", "Related []Finding", "type PeekAnswer = Finding", + `"check.step"`, `"delivery.step"`, `"check.run"`, + `const AcceptSubjectCheckStep = "mesh.seat.a-registry.accept.check.step"`, + `const AcceptSubjectDeliveryStep = "mesh.seat.a-registry.accept.delivery.step"`, + "type CheckStepAccept struct", "type DeliveryStepAccept", + } { + if !strings.Contains(src, want) { + t.Errorf("the Go contract lacks %q:\n%s", want, src) + } + } + if strings.Contains(src, "SendFinding") || strings.Contains(src, "SubjectFinding") { + t.Error("a body yields more than its type") + } + dts := string(files["ts/seats/aregistry/index.d.ts"]) + js := string(files["ts/seats/aregistry/index.js"]) + for _, want := range []string{"export interface Finding", "checkStepSubject", "deliveryStepSubject"} { + if !strings.Contains(dts, want) { + t.Errorf("index.d.ts lacks %q:\n%s", want, dts) + } + } + if !strings.Contains(js, `"mesh.seat.a-registry.accept.delivery.step"`) { + t.Errorf("index.js lacks the delivery stream's subject:\n%s", js) + } +} diff --git a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json index f992622b6..dbcd3a8a7 100644 --- a/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json +++ b/modules/mesh-build-go/buildstep/seatgen/testdata/contracts/seats/a-registry/seat.json @@ -1,5 +1,11 @@ {"format":1,"seat":"a-registry","scope":"mesh","grain":"whole","domain":"Module","decision":"ADR 1","includes":["registry"], - "verbs":[{"name":"hold","class":"read","callers":["invokers","persons","sessions","seat:b-registry","module:gitea","operator","healers","holder:node-backup@same-node"],"input":{"type":"object"},"answer":{"type":"object"}}], + "verbs":[{"name":"hold","class":"read","callers":["invokers","persons","sessions","seat:b-registry","module:gitea","operator","healers","holder:node-backup@same-node"],"input":{"type":"object"},"answer":{"type":"object"}}, + {"name":"peek","input":{"type":"object"},"answer":{"$ref":"#/bodies/Finding"}}], + "accepts":[ + {"name":"step","stream":"check","body":{"type":"object","properties":{"id":{"type":"string"}}}}, + {"name":"step","stream":"delivery","body":{"type":"object"}}, + {"name":"run","stream":"check","body":{"type":"object"}}], + "bodies":{"Finding":{"type":"object","properties":{"id":{"type":"string"},"owner":{"type":"string"},"related":{"type":"array","items":{"$ref":"#/bodies/Finding"}}},"required":["id"]}}, "acts":[{"step":"build","on":"kind:image","bound":"60m"},{"step":"check","on":"every"},{"step":"build","on":"go"}], "events":[ {"name":"milestone-asked","subject":"milestone-asked.","body":{"type":"object"},"keep":{"max-age-days":90,"max-mib":16,"last-per-subject":true}}, diff --git a/modules/mesh-build-go/buildstep/seatgen/tsgen.go b/modules/mesh-build-go/buildstep/seatgen/tsgen.go index b1b655b12..352c2cb53 100644 --- a/modules/mesh-build-go/buildstep/seatgen/tsgen.go +++ b/modules/mesh-build-go/buildstep/seatgen/tsgen.go @@ -103,7 +103,7 @@ func tsFiles(m *Module, out map[string][]byte) error { fmt.Fprintf(&o.js, "export const seat = %q;\n\n", d.Seat) fmt.Fprintf(&o.dts, "export declare const seat: %q;\n\n", d.Seat) tsNames(o, "verbs", seatdef.VerbNames(d.Verbs)) - tsNames(o, "accepts", seatdef.MessageNames(d.Accepts)) + tsNames(o, "accepts", acceptNames(d.Accepts)) tsNames(o, "events", seatdef.MessageNames(d.Events)) if f := d.Family; f != nil { fmt.Fprintf(&o.js, "export const family = { name: %q, maxAgeDays: %d };\n\n", f.Name, f.MaxAgeDays) @@ -121,7 +121,7 @@ func tsFiles(m *Module, out map[string][]byte) error { tsSubject(o, tsName(e.Name), seatdef.EventSubject(d.Seat, e.Name), e.ByCaller) } for _, a := range d.Accepts { - tsSubject(o, tsName(a.Name), seatdef.AcceptSubject(d.Seat, a.Name), a.ByCaller) + tsSubject(o, tsName(acceptName(a)), seatdef.AcceptSubject(d.Seat, acceptName(a)), a.ByCaller) } if len(d.HolderEvents) > 0 { tsNames(o, "holderEvents", holderNames(d.HolderEvents)) diff --git a/modules/mesh-build-go/buildstep/seatgen/types.go b/modules/mesh-build-go/buildstep/seatgen/types.go index 4b96bae42..c44cb57ed 100644 --- a/modules/mesh-build-go/buildstep/seatgen/types.go +++ b/modules/mesh-build-go/buildstep/seatgen/types.go @@ -62,9 +62,13 @@ func (tp *typer) read(where string, s map[string]any, top bool) (*typ, error) { return &typ{kind: "any"}, nil } if r, ok := s["$ref"].(string); ok { + // A reference into `bodies` is read as one into `defs` (design 47 G §454). n, found := strings.CutPrefix(r, "#/defs/") + if !found { + n, found = strings.CutPrefix(r, "#/bodies/") + } if !found || n == "" { - return nil, fmt.Errorf("$ref %q is not #/defs/", r) + return nil, fmt.Errorf("$ref %q is not #/defs/ or #/bodies/", r) } return &typ{kind: "ref", ref: n}, nil } -- 2.54.0 From 32655b95f6eda0b1891dcf22a304c04b804c2382 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:31:24 +0100 Subject: [PATCH 011/243] GIT1, GIT2, GIT3, GIT10: the forge module serves the seat git's verbs and says its events, retires its acting tools, follows the catalog half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The four share files (index.ts, module.json, guard.test.ts), so they land as one commit. GIT1: seat.ts serves pull, open-pulls, tags, trees, file, contains, statuses, status, comment, view, note, merge-pull as tools on the seat, each answering the contract's shape and an error of kind refused, not-found or failed; a comment, view or note whose act id is held is answered held (note matched by id). merge-pull merges the head the caller names (its from-G judgements are GIT6, GIT7, GIT9). GIT2: seatevents.ts says pull-updated, pull-merged, pull-closed, forge-pull, forge-status, forge-delete and repo-writable on mesh.seat.git.event.; the module's emits drop them. GIT3: merge.ts, check.ts, the acting tools, the owed watch, the gate's source read and check-gates-merge are gone. GIT10: catalog.ts follows the seat mesh-catalog (revision, catalog-revision); no write is granted before the first view. Builds: design 47 G §310, §311, §312, §313, §314, §163, §317, §278, §321, §196 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/README.md | 4 +- modules/gitea/announce.ts | 21 +- modules/gitea/catalog.ts | 145 ++++ modules/gitea/check.ts | 225 ------- modules/gitea/client.ts | 15 + modules/gitea/cmd/forge/commit.go | 3 +- modules/gitea/cmd/forge/main.go | 4 +- .../cmd/package-registry/retention_test.go | 4 +- modules/gitea/delivery.ts | 91 +-- modules/gitea/gate.ts | 8 +- modules/gitea/guard.ts | 7 +- modules/gitea/index.ts | 99 ++- modules/gitea/internal/packageseats/holder.go | 90 ++- .../internal/packageseats/holder_test.go | 26 + modules/gitea/kept.ts | 1 + modules/gitea/mainrules.ts | 13 - modules/gitea/merge.ts | 279 -------- modules/gitea/module.json | 46 +- modules/gitea/package.json | 4 +- modules/gitea/protectfull.ts | 42 +- modules/gitea/seat.ts | 323 +++++++++ modules/gitea/seatevents.ts | 96 +++ modules/gitea/test/announce.test.ts | 46 +- modules/gitea/test/bundle-import.test.ts | 84 +-- modules/gitea/test/check-owed.test.ts | 207 ------ modules/gitea/test/delivery.test.ts | 33 +- modules/gitea/test/gate.test.ts | 29 +- modules/gitea/test/guard.test.ts | 624 +++++------------- modules/gitea/test/kept.test.ts | 4 +- modules/gitea/test/protection-full.test.ts | 31 +- modules/gitea/test/token.test.ts | 5 +- modules/gitea/test/tools-account.test.ts | 20 +- modules/gitea/test/views.test.ts | 143 ++-- modules/gitea/tools/index.ts | 75 +-- modules/gitea/views.ts | 241 ------- modules/gitea/writes.ts | 69 +- 36 files changed, 1172 insertions(+), 1985 deletions(-) create mode 100644 modules/gitea/catalog.ts delete mode 100644 modules/gitea/check.ts delete mode 100644 modules/gitea/merge.ts create mode 100644 modules/gitea/seat.ts create mode 100644 modules/gitea/seatevents.ts delete mode 100644 modules/gitea/test/check-owed.test.ts delete mode 100644 modules/gitea/views.ts diff --git a/modules/gitea/README.md b/modules/gitea/README.md index de98b90ed..119e7c92d 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -67,8 +67,8 @@ refuses the commit that would make a branch a rule guards. A rule covers a branc forge itself decides it (Gitea 1.27, novox/hq issue 539): a plain name covers that branch in any case; a glob (`*`, `**`, `?`, `[abc]`, `[!abc]`, `{a,b}`, `\` escaping) covers case-sensitively with `/` as separator; a glob the forge cannot compile covers its own text. `cmd/forge/rules.go` and `guard.ts` hold this one rule. The rules are read before anything is sent; rules that cannot be read refuse the -commit. Nothing about merging changes: a merge still needs an independent review at the head and both -required checks green, through `gitea_merge_pull_request`. +commit. Nothing about merging changes: a merge is the seat git's `merge-pull`, at the head the operator's +Approve names. **A patch too large to type** (novox/hq issue 852: a feature's diff is megabytes) goes as `patch_file`, with `patch_sha256`. The caller writes the patch into the module's spool directory `forge-patches` diff --git a/modules/gitea/announce.ts b/modules/gitea/announce.ts index 19293af54..42fe46fd1 100644 --- a/modules/gitea/announce.ts +++ b/modules/gitea/announce.ts @@ -5,13 +5,12 @@ // the bus's message id (mesh-tools node-tools/internal/bus/bus.go, `nats.MsgId`), and the bus stores a message it has // seen the id of, within its de-duplication window, once. // -// The SDK's `emit` takes no id, so this publishes through the broker with the headers `emit` stamps and the id given. -// It returns to `emit` once the SDK takes an id. +// They are the seat git's events (design 47 *The seat git*), said through the runtime's seat publish with that id. // // The ids: a merge by its merge commit, a head by the head and the trunk commit it was announced against (the check asked // again against a new trunk commit is another announcement), a close by its head. -import { broker } from "@novox/mesh-sdk/messaging"; +import { sayOnSeat } from "./seatevents.js"; import { state as keptState } from "@novox/mesh-sdk/state"; type Said = { owner?: string; repo?: string; number?: number; head_sha?: string; base_sha?: string; merge_commit_sha?: string }; @@ -32,23 +31,11 @@ export function announcementId(type: string, body: unknown): string | undefined } } -/** Announce a merge, a head or a close: published with its message id. */ +/** Announce a merge, a head or a close: said on the seat git (seatevents.ts) with its message id. */ export async function announce(type: string, body: unknown): Promise { const id = announcementId(type, body); if (!id) throw new Error(`${type} is not an announcement with a message id`); - const node = process.env.MESH_NODE ?? "unknown"; - await broker().publish({ - key: type, - node, - body, - headers: { - "x-event-id": id, - "x-source": process.env.MESH_MODULE ?? "unknown", - "x-node": node, - "x-time": new Date().toISOString(), - "content-type": "application/json", - }, - }); + if (!(await sayOnSeat(type, (body ?? {}) as Record, id))) throw new Error(`the seat git says no ${type}`); } // --- the watermark: the one value of the old files that is not the bus's to remember -------------------------------- diff --git a/modules/gitea/catalog.ts b/modules/gitea/catalog.ts new file mode 100644 index 000000000..a45517c5e --- /dev/null +++ b/modules/gitea/catalog.ts @@ -0,0 +1,145 @@ +// **The forge module's view of the catalog half** (novox/hq design 47 *Every part takes a first view, then follows the +// events*, ADR 0656 §6): which repositories build a registered module is the catalog half's word, asked of the seat +// mesh-catalog's verb `revision` through the seat's generated TypeScript contract and then kept from its event +// `catalog-revision`, never the holder's state nor a module named. The picture is mesh-sdk's views Follower, so the +// module's first-view tools (statusTools) read and steer it as every part's. +// +// The readers' rule: a candidate's revision moves nothing; a trunk revision heard that the view does not hold yet is +// waited for (said in the status) and asked, and past mesh-sdk's WAITS_BOUND_MS the module writes view-waits; until the +// first view is held the module acts on nothing (writes.ts grants no write). + +import { Follower, type Answer, type Finding, type FollowerOptions, type Status } from "@novox/mesh-sdk/views"; +import { + seat as CATALOG, verbs as CATALOG_VERBS, events as CATALOG_EVENTS, states as CATALOG_STATES, + type CatalogEntry, type CatalogRevisionEvent, type RevisionAnswer, type RevisionInput, +} from "@novox/mesh-delivery-contracts-mesh-catalog"; + +/** The seat verb the view asks, as the manifest's `invokes` names it. */ +export const REVISION_VERB = `seat:${CATALOG}.${CATALOG_VERBS.revision}`; +/** The seat event the view hears, as the manifest's `consumes` names it. */ +export const CATALOG_REVISION_EVENT = `${CATALOG}.${CATALOG_EVENTS.catalogRevision}`; +/** A trunk revision; a candidate's moves no running part (ADR 0656 §6). */ +export const TRUNK = "trunk"; + +/** A revision view is never made again: one fixed creation time lets the follower tell a revision heard from one held. */ +const CREATED = new Date(0).toISOString(); + +/** Asks one seat verb over the bus and answers what the holder said. */ +export type Ask = (verb: string, input: RevisionInput) => Promise; + +/** The hooks a part hands the view: everything of mesh-sdk's FollowerOptions but what the view sets itself. */ +export type CatalogHooks = Omit; + +/** The revision's answer, or the holder's refusal as an error: never an empty view taken for the trunk. */ +export function revisionAnswer(raw: unknown): RevisionAnswer { + let a: unknown = raw; + if (typeof raw === "string") { + try { + a = JSON.parse(raw); + } catch { + throw new Error(`${REVISION_VERB} answered: ${raw.slice(0, 200)}`); + } + } + const r = a as Partial & { error?: unknown }; + if (!r || typeof r !== "object" || typeof r.revision?.revision !== "number" || r.revision.revision <= 0 || !Array.isArray(r.entries)) { + const said = r && typeof r === "object" && typeof r.error === "string" ? r.error : JSON.stringify(a ?? null).slice(0, 200); + throw new Error(`${REVISION_VERB} answered no revision: ${said}`); + } + return r as RevisionAnswer; +} + +export class CatalogView { + readonly follower: Follower; + /** The trunk revision whose pages are being read; 0 between asks. */ + private paging = 0; + /** The trunk revision the view holds whole; 0 before the first view. */ + private heldRevision = 0; + + private readonly ask: Ask; + + constructor(ask: Ask, hooks: CatalogHooks) { + this.ask = ask; + this.follower = new Follower({ ...hooks, owner: CATALOG, tool: REVISION_VERB, state: CATALOG_STATES.catalogRevisions, + ask: (after) => this.page(after) }); + } + + /** The ask the last event began, not waited on by hear: a test awaits it. */ + asked: Promise = Promise.resolve(); + + /** The trunk revision the view holds, 0 before the first view. */ + get held(): number { + return this.heldRevision; + } + + /** One module's entry in the view, as the catalog half says it. */ + entry(module: string): CatalogEntry | undefined { + return this.follower.get(module) as CatalogEntry | undefined; + } + + /** The Follower's ask: the trunk's newest revision, then the next pages of the same one. */ + private async page(after: string): Promise { + const input: RevisionInput = after && this.paging ? { after, revision: this.paging } : {}; + const a = revisionAnswer(await this.ask(REVISION_VERB, input)); + const rev = a.revision.revision; + if (a.revision.kind !== TRUNK) throw new Error(`${REVISION_VERB} answered a ${a.revision.kind} revision for the trunk`); + if (after && this.paging && rev !== this.paging) { + const was = this.paging; + this.paging = 0; + throw new Error(`${REVISION_VERB} answered revision ${rev} for a page of revision ${was}`); + } + this.paging = rev; + const values: Record = {}; + for (const e of a.entries) values[e.module] = e; + if (!a.next) { + this.paging = 0; + this.heldRevision = rev; + const waits = Number(this.follower.status()["waits-for"] ?? 0); + if (waits <= rev) await this.follower.holds(String(rev)); + } + return { v: 1, owner: CATALOG, state: CATALOG_STATES.catalogRevisions, revision: rev, created: CREATED, values, + ...(a.next ? { next: a.next } : {}), at: new Date().toISOString() }; + } + + /** + * One `catalog-revision` event. A candidate's moves nothing; a trunk revision the view holds, or an older one, is passed + * over; a newer one is waited for and asked, not waited on here. Data that is no such event is an error. + */ + async hear(ev: CatalogRevisionEvent | undefined): Promise { + if (!ev || typeof ev.revision !== "number" || ev.revision <= 0) { + throw new Error(`not a ${CATALOG_REVISION_EVENT} event: ${JSON.stringify(ev ?? null).slice(0, 200)}`); + } + if (ev.kind !== TRUNK || ev.revision <= this.heldRevision) return; + const held = this.heldRevision; + // The follower keeps the newest revision heard, so an ask running when it was heard is asked once more. + await this.follower.apply({ v: 1, owner: CATALOG, state: CATALOG_STATES.catalogRevisions, key: CATALOG_REVISION_EVENT, + revision: ev.revision, created: CREATED, gap: true }); + await this.follower.waitFor(String(ev.revision)); + this.asked = this.follower.gap(`catalog revision ${ev.revision} said, the view holds ${held}`); + } +} + +/** The kind of a finding of a first view unanswered past its bound (ADR 0615 §6), as mesh-sdk's Go views writes it. */ +export const UNANSWERED_KIND = "first-view-unanswered"; + +/** The key of a finding about a module: scope, the id's dotted parts, and the kind, each part made safe (mesh-sdk health.Key). */ +export function findingKey(scope: string, id: string, kind: string): string { + return [scope, ...id.split("."), kind].map((p) => p.trim().replace(/[^A-Za-z0-9_=/-]/g, "_") || "_").join("."); +} + +/** A finding as the condition keeper reads it from the module's state `findings`: its key and its value. */ +export function findingValue(f: Finding): { key: string; value: Record } { + const key = findingKey("module", f.id, f.kind); + return { key, value: { + v: 1, key, kind: f.kind, subject: { scope: "module", id: f.id }, severity: "warning", summary: f.summary, headline: f.headline, + explanation: f.explanation, resolved: "", evidence: [{ at: f.at, said: f.summary }], source: `${f.observer}.${f.kind}`, + observer: f.observer, at: f.at, + } }; +} + +/** A part's finding of a first view unanswered past its bound, from the follower's status. */ +export function unansweredFinding(part: string, s: Status, at: Date): Finding { + const summary = `${part} has no first view of ${s.owner}'s ${s.state}: ${s.tool} has not answered since ${s.asked ?? "its start"} (${(s.error ?? "").trim()})`; + return { observer: part, kind: UNANSWERED_KIND, id: `${part}.${s.owner}.${s.state}`, summary, headline: `${part} waits for ${s.owner}`, + explanation: `${part} reads ${s.owner}'s ${s.state} only through ${s.tool}, and keeps asking it; it never reads the state itself. The answer ` + + `clears this. Look at ${s.owner}'s holder: is it running and connected?`, at: at.toISOString() }; +} diff --git a/modules/gitea/check.ts b/modules/gitea/check.ts deleted file mode 100644 index 3af490492..000000000 --- a/modules/gitea/check.ts +++ /dev/null @@ -1,225 +0,0 @@ -// **mesh/check is written from the checker's owed state, read by a watch** (ADR 0521; R4e round 2 finding R2-1; finding K1 -// before it; novox/hq issue 1301; ADR 0447 §1, ADR 0503 §3: main requires mesh/check alone). -// -// No tool writes mesh/check: a tool is told no caller, and the console every agent session speaks through (node-tools) -// invokes every tool, so a tool that wrote it could be called by any session to merge past the checker (the fault issue -// 293 closed for the two statuses before it). Nor does an event: nothing in the delivery plan orders this module before the -// forge bridge, and a durable consumer whose filters change keeps its deliver policy, so an event said before this module -// consumed it was lost. The checker keeps what it owes the forge in its state `owed` (ADR 0484 §2), which it announces -// readable (ADR 0495 §2-§3), and only the checker writes it. This module watches it: the watch hands over the latest value of -// every key when it starts, whenever that is, and then every change. Each key whose act is the status mesh/check is -// written on its head, once per revision; a revision older than one already written on the key is dropped. -// -// A read refused (the grant not yet composed) is said once and asked again on a bound until it holds. A forge that refuses -// a status is asked again on a bound, until it writes or a newer revision of the key comes. - -import type { CommitStatus } from "./client.js"; -import { bindUntilHeld, bindWait } from "./bind.js"; -import { checkName } from "./guard.js"; -import { CHECK_STATUS } from "./pulls.js"; - -/** The checker's state this module reads, and the act of its keys this module writes. */ -export const CHECKER_OWED = "mesh-delivery-checker.owed"; -export const CHECK_ACT = `status=${CHECK_STATUS}`; - -const states = new Set(["pending", "success", "error", "failure", "warning"]); - -/** What writing the check needs of the forge. */ -export interface CheckForge { - setCommitStatus(owner: string, repo: string, sha: string, status: CommitStatus): Promise; -} - -/** One change of the checker's owed, as the SDK's state watch hands it. */ -export interface OwedChange { - key: string; - op: "put" | "delete"; - value?: unknown; - revision: number; - current?: boolean; -} - -/** A key part as mesh-sdk go/delivery EscapeName writes it, read back: `=` and two lowercase hex digits per byte. */ -function unescapeName(part: string): string { - if (!/^([-/_a-zA-Z0-9]|=[0-9a-f]{2})+$/.test(part)) throw new Error(`${JSON.stringify(part)} is not an escaped name`); - return Buffer.from(part.replace(/=([0-9a-f]{2})/g, (_, h) => String.fromCharCode(parseInt(h, 16))), "latin1").toString("utf8"); -} - -/** The pull request and head of the checker's owed key (mesh-sdk go/delivery CheckerOwedKey): - * `/...status=mesh/check`, owner and name escaped. */ -export function owedKeyOf(key: string): { owner: string; repo: string; number: number; sha: string } { - const parts = String(key ?? "").split("."); - if (parts.length !== 4) throw new Error(`${JSON.stringify(key)} is not an owed key, ...`); - const [repository, number, sha, act] = parts; - if (act !== CHECK_ACT) throw new Error(`${JSON.stringify(key)} owes ${JSON.stringify(act)}, not ${CHECK_ACT}`); - const m = /^([^/]+)\/([^/]+)$/.exec(repository); - if (!m) throw new Error(`${JSON.stringify(key)} names no /`); - if (!/^(0|[1-9][0-9]*)$/.test(number)) throw new Error(`${JSON.stringify(key)} names no pull request number`); - if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha)) throw new Error(`${JSON.stringify(key)} names no head's whole sha`); - return { owner: checkName("owner", unescapeName(m[1])), repo: checkName("name", unescapeName(m[2])), number: Number(number), sha }; -} - -/** The status an owed change asks for, checked as mesh-sdk go/delivery DecodeCheckerOwed checks it; or why it is not one. */ -export function checkStatusOf(change: OwedChange): { owner: string; repo: string; sha: string; revision: number; status: CommitStatus } { - if (change.op !== "put") throw new Error(`a ${change.op} owes nothing`); - const { owner, repo, sha } = owedKeyOf(change.key); - const revision = change.revision; - if (typeof revision !== "number" || !Number.isSafeInteger(revision) || revision < 0) { - throw new Error(`${JSON.stringify(revision)} is not a revision`); - } - const v = change.value; - if (!v || typeof v !== "object") throw new Error("the value is no owed act"); - const o = v as Record; - if (o.v !== 1) throw new Error(`the owed act's version ${JSON.stringify(o.v)} is not one this module reads`); - if (o.act !== "status" || o.context !== CHECK_STATUS) { - throw new Error(`the owed act is ${JSON.stringify(o.act)} ${JSON.stringify(o.context)}, not the status ${CHECK_STATUS}`); - } - const state = String(o.state ?? ""); - if (!states.has(state)) throw new Error(`${JSON.stringify(o.state)} is not a status the forge keeps`); - let description = String(o.text ?? "").replace(/\s+/g, " ").trim(); - if (description.length > 140) description = description.slice(0, 139) + "…"; - const target = typeof o.target === "string" && /^https?:\/\//.test(o.target) ? o.target : undefined; - const status: CommitStatus = { state: state as CommitStatus["state"], context: CHECK_STATUS, description, - ...(target ? { target_url: target } : {}) }; - return { owner, repo, sha, revision, status }; -} - -/** Whether the forge's refusal will not change by asking again: it forbids the write (403) or knows no such repository - * or commit (404), as the client words it ("Gitea API : ..."). */ -export function refusedForGood(err: unknown): boolean { - return /Gitea API \S+: (403|404) /.test(err instanceof Error ? err.message : String(err)); -} - -/** How many mesh/check writes this module has at the forge at once. */ -export const WRITES_AT_ONCE = 4; - -/** The one writer of mesh/check: each owed revision of the checker's, in revision order per key. */ -export class CheckStatuses { - /** The newest revision written of each key, and the newest seen; the oldest keys go past what any forge holds open. */ - private written = new Map(); - private seen = new Map(); - /** The turn each key's writes wait on, and the asks again still running. */ - private turns = new Map>(); - private again = new Set>(); - /** Writes at the forge at once, at most WRITES_AT_ONCE: a watch's start hands over every key together (issue 1726). */ - private writing = 0; - private queued: (() => void)[] = []; - private forge: CheckForge; - private say: (line: string) => void; - private sleep: (ms: number) => Promise; - - constructor(forge: CheckForge, say: (line: string) => void = (l) => console.error(l), - sleep: (ms: number) => Promise = (ms) => new Promise((r) => setTimeout(r, ms))) { - this.forge = forge; - this.say = say; - this.sleep = sleep; - } - - /** Write the change's status: "written", or "dropped" for one that owes no mesh/check or is older than one written. A - * forge that refuses throws. */ - async apply(change: OwedChange): Promise<"written" | "dropped"> { - let asked; - try { - asked = checkStatusOf(change); - } catch (err) { - // A deleted key is owed no more: its revision is seen, so a write of it still asked again ends (askAgain). - if (change?.op === "delete" && typeof change.key === "string" && Number.isInteger(change.revision)) { - const seen = this.seen.get(change.key); - if (seen === undefined || change.revision > seen) this.remember(this.seen, change.key, change.revision); - return "dropped"; - } - this.say(`[gitea] ${CHECKER_OWED} ${change?.key} at revision ${change?.revision} writes nothing: ${err instanceof Error ? err.message : err}`); - return "dropped"; - } - const key = change.key; - const seen = this.seen.get(key); - if (seen === undefined || asked.revision > seen) this.remember(this.seen, key, asked.revision); - // One key's writes take turns, so two at once are weighed one after the other. - const before = this.turns.get(key) ?? Promise.resolve(); - const turn = before.then(async (): Promise<"written" | "dropped"> => { - const newest = this.written.get(key); - if (newest !== undefined && asked.revision < newest) { - this.say(`[gitea] ${asked.owner}/${asked.repo}@${asked.sha.slice(0, 12)}: mesh/check at revision ${asked.revision} is older than ${newest}, already written, and is dropped`); - return "dropped"; - } - await this.atTheForge(() => this.forge.setCommitStatus(asked.owner, asked.repo, asked.sha, asked.status)); - this.remember(this.written, key, asked.revision); - return "written"; - }); - const settled = turn.catch(() => {}); - this.turns.set(key, settled); - void settled.then(() => { if (this.turns.get(key) === settled) this.turns.delete(key); }); - return turn; - } - - /** What the watch hands over: taken at once and written in the background, or when the forge refuses, asked again on a - * bound until it is written or a newer revision of the key was seen. Never throws: a watch's handler that throws loses - * the change. It never waits on the forge (novox/hq issue 1726): the runtime hands a watch's current values one at a - * time, waits for each to be taken and gives the whole set 10 s; a forge write per key made a large owed outlast that - * bound at every attempt, so the watch never held and no verdict reached mesh/check. Each key's writes still take - * turns in revision order (apply). */ - async take(change: OwedChange): Promise { - const run = this.apply(change).then(() => {}, (err) => this.askAgain(change, err)); - this.again.add(run); - void run.finally(() => this.again.delete(run)); - } - - /** Runs one forge write when fewer than WRITES_AT_ONCE are running, else after one ends. */ - private async atTheForge(write: () => Promise): Promise { - if (this.writing >= WRITES_AT_ONCE) await new Promise((r) => this.queued.push(r)); - else this.writing++; - try { - return await write(); - } finally { - const next = this.queued.shift(); - if (next) next(); - else this.writing--; - } - } - - /** Resolves when no write or ask again is running. */ - async settled(): Promise { - while (this.again.size) await Promise.all([...this.again]); - } - - private async askAgain(change: OwedChange, first: unknown): Promise { - let err = first; - for (let attempt = 1; ; attempt++) { - if (refusedForGood(err)) { - // A repository the forge forbids or does not know answers the same at every ask (the sweep of 2026-10-30, issue - // 774: hal and novadock, 119 asks each): said once, never asked again; a newer revision of the key asks anew. - this.say(`[gitea] ${change.key}: mesh/check at revision ${change.revision} is refused for good (${err instanceof Error ? err.message : err}); not asked again`); - return; - } - const wait = bindWait(attempt); - this.say(`[gitea] ${change.key}: mesh/check at revision ${change.revision} could not be written (${err instanceof Error ? err.message : err}); asked again in ${wait / 1000}s`); - await this.sleep(wait); - const newest = this.seen.get(change.key); - if (newest !== undefined && newest > change.revision) return; - try { - await this.apply(change); - return; - } catch (e) { - err = e; - } - } - } - - private remember(map: Map, key: string, revision: number): void { - map.delete(key); - map.set(key, revision); - if (map.size > 4000) map.delete(map.keys().next().value as string); - } -} - -/** - * Watch the checker's owed with `watch` (the SDK's `state(CHECKER_OWED).watch`), handing each change to `checks`. A watch - * refused — the read not granted yet — is said once and asked again on a bound until it holds (bind.ts). Never rejects. - */ -export async function watchOwed(watch: (handler: (change: OwedChange) => Promise) => Promise, checks: CheckStatuses, - sleep?: (ms: number) => Promise, say: (line: string) => void = (l) => console.error(l)): Promise { - let refusals = 0; - await bindUntilHeld(CHECKER_OWED, () => watch((change) => checks.take(change)), sleep, (line) => { - if (/could not be bound/.test(line) && refusals++ > 0) return; - say(line); - }); -} diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 8a8079a89..42898d71c 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -481,6 +481,21 @@ export class GiteaClient { return GiteaClient.mapPull(await this.request(`${repoPath(owner, repo)}/pulls/${index}`)); } + /** The pull request a commit was merged by (the forge's own lookup of a merge commit): what the seat git's `pull` + * answers by merge commit. A commit no pull request merged is the forge's 404. */ + async pullOfCommit(owner: string, repo: string, sha: string): Promise { + if (!/^[0-9a-f]{40}$/.test(sha)) throw new Error(`${sha} is not a commit's full 40-hex sha`); + return GiteaClient.mapPull(await this.request(`${repoPath(owner, repo)}/commits/${sha}/pull`)); + } + + /** One page of a repository's tags, newest first, by name: what the seat git's `tags` answers. */ + async tags(owner: string, repo: string, page: number, limit: number): Promise { + if (!Number.isInteger(page) || page < 1) throw new Error(`${page} is not a page, from 1`); + const list = await this.request(`${repoPath(owner, repo)}/tags?page=${page}&limit=${limit}`); + if (!Array.isArray(list)) throw new Error(`the forge answered no list of the tags of ${owner}/${repo}`); + return list.map((t) => String(t?.name ?? "")).filter((n) => n !== ""); + } + /** A commit's parents, by sha, in order: what a merge commit says it joined (novox/hq issue 381). An answer * that names no parents is no answer, never an empty list read as a root commit. */ async commitParents(owner: string, repo: string, sha: string): Promise { diff --git a/modules/gitea/cmd/forge/commit.go b/modules/gitea/cmd/forge/commit.go index a2c0245e2..bb911820a 100644 --- a/modules/gitea/cmd/forge/commit.go +++ b/modules/gitea/cmd/forge/commit.go @@ -16,8 +16,7 @@ import ( // gitea_commit (novox/hq issue 375): one commit on a branch of its own, so an agent follows the branch / pull // request / approval rule without anyone pushing by hand. By the rule gitea_api already enforces for a file // written (novox/hq ADR 0266): never the repository's default branch, never a branch a protection rule names. -// Nothing about merging changes — that still needs a review at the head and both required checks green -// (gitea_merge_pull_request). +// Nothing about merging changes — a merge is the seat git's merge-pull, at the head the operator's Approve names. // FileChange is one file of a commit: its new content, as text or base64, or its deletion. type FileChange struct { diff --git a/modules/gitea/cmd/forge/main.go b/modules/gitea/cmd/forge/main.go index 89900a64e..fadedecdd 100644 --- a/modules/gitea/cmd/forge/main.go +++ b/modules/gitea/cmd/forge/main.go @@ -49,8 +49,8 @@ func Tools(forge func() (*Forge, error)) []stdio.Tool { Name: "gitea_commit", Description: "Put one commit on a branch of its own in a forge repository, creating the branch from base " + "when it does not exist. Refused on the repository's default branch and on any branch a protection " + - "rule names (novox/hq ADR 0266): a change reaches those only by a pull request whose head an " + - "independent review approved and whose required checks passed (gitea_merge_pull_request). Carries " + + "rule names (novox/hq ADR 0266): a change reaches those only by a pull request merged at the head the " + + "operator's Approve names (the seat git's merge-pull). Carries " + "either files — each {path, content (text) | content_base64 | delete: true | copy_from {owner, repo, ref, path}: a file or directory of another repository, copied by the module}, created or updated as " + "the branch holds them — or patch, a unified diff as git diff writes it, which the forge applies. " + "At most 200 files, 4 MiB each and 16 MiB together, or a 4 MiB patch. Answers the commit it made and " + diff --git a/modules/gitea/cmd/package-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go index 7145bdddf..93d77ace9 100644 --- a/modules/gitea/cmd/package-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -487,8 +487,8 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { if strings.Contains(string(req), `"name":"transfers","options"`) { t.Errorf("transfers has options (a lifetime): %s", req) } - // The retention asks the builder's versions; the forge's other bundles ask the indexer's modules and record a re-read. - if strings.Join(m.Invokes, ",") != "mesh-delivery-indexer.indexer_read_modules,seat:mesh-hand-acts.hand-act,mesh-delivery-builder.builder_versions" { + // The retention asks the builder's versions; the forge's other bundles ask the catalog half and record a re-read. + if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-hand-acts.hand-act,mesh-delivery-builder.builder_versions" { t.Errorf("the manifest invokes %v", m.Invokes) } if len(m.Tools) != 0 { diff --git a/modules/gitea/delivery.ts b/modules/gitea/delivery.ts index 050f60678..ed65da6f4 100644 --- a/modules/gitea/delivery.ts +++ b/modules/gitea/delivery.ts @@ -1,7 +1,6 @@ // What the forge's holder does for a delivery (novox/hq ADR 0239): the commit's note under -// refs/notes/mesh-plan, the delivery's view on its pull request, and its statuses — asked by mesh-delivery, -// the delivery's owner, through this module's tools. The forge is this module's; mesh-delivery never writes -// to it itself. +// refs/notes/mesh-plan and the delivery's view on its pull request, asked through the seat git's verbs `note` and +// `view` (seat.ts). The forge is this module's; no delivery part writes to it itself. // // **The note is written in the forge's own repository, as the forge's own user.** The forge's API reads a // note and writes none, and a push needs a clone and a credential nobody else should hold; the forge's @@ -13,13 +12,6 @@ import { execFile } from "node:child_process"; import { promisify } from "node:util"; -import type { ToolDefinition } from "@novox/mesh-sdk/tools"; -import type { CommitStatus } from "./client.js"; -import { checkName } from "./guard.js"; -import { CHECK_STATUS } from "./pulls.js"; - -export { CHECK_STATUS }; - /** How a command is run: docker on the forge's machine, or a test's. */ export type Runner = (file: string, args: string[]) => Promise<{ stdout: string; code: number }>; @@ -64,12 +56,13 @@ export function noteArgs(container: string, owner: string, repo: string, commit: /** Append a line to a commit's note, unless the note already holds it. Answers whether it was added. */ export async function appendNote(runner: Runner, container: string, owner: string, repo: string, commit: string, - notesRef: string, line: string): Promise<{ added: boolean; lines: number }> { + notesRef: string, line: string, heldId?: string): Promise<{ added: boolean; lines: number }> { const wanted = noteLine(line); const shown = await runner("docker", noteArgs(container, owner, repo, commit, notesRef)); // No note yet is git's exit 1 with nothing on stdout; anything else unreadable is said. const lines = shown.code === 0 ? shown.stdout.split("\n").filter((l) => l.trim() !== "") : []; - if (lines.includes(wanted)) return { added: false, lines: lines.length }; + // A line that carries the act's id is the act already done, whatever else it says. + if (lines.includes(wanted) || (heldId && lines.some((l) => l.includes(`[${heldId}]`)))) return { added: false, lines: lines.length }; const appended = await runner("docker", noteArgs(container, owner, repo, commit, notesRef, wanted)); if (appended.code !== 0) throw new Error(`the note on ${commit.slice(0, 8)} could not be appended (git exited ${appended.code})`); return { added: true, lines: lines.length + 1 }; @@ -89,77 +82,3 @@ export function viewComment(comments: T[ return comments.find((c) => c.body.includes(VIEW_MARKER)); } -const states = new Set(["pending", "success", "error", "failure", "warning"]); - -/** The merge check's contexts (pulls.ts): a branch's protection may require them. */ -const mergeCheckContexts = new Set(["mesh/merge-gate", "mesh/repo-check"]); - - -/** A status mesh-delivery asks for, checked: one of the forge's states, a context of the mesh's own, a - * bounded description. */ -export function deliveryStatus(context: string, state: string, description: string, target?: string) { - if (!/^mesh\/[a-z-]+$/.test(context)) throw new Error(`${context} is not a status of the mesh's own`); - if (!states.has(state)) throw new Error(`${state} is not a status the forge keeps`); - // mesh/check is the checker's verdict and the one status main requires (ADR 0447 §1): no hand sets it in any - // state, as issue 293 closed for the two before it. No tool writes it, since a tool is told no caller (R4e finding K1): - // check.ts writes it from the checker's owed state (ADR 0521). - if (context === CHECK_STATUS) { - throw new Error(`${CHECK_STATUS} is the checker's status: it is written from mesh-delivery-checker.owed alone, never by a tool`); - } - // The merge check's statuses are its verdict's, set when the controller says it, and a branch may require - // them: never set to a verdict by hand, so nothing passes — or fails — a merge past the check (novox/hq - // issue 293). Pending is no verdict: it is what this holder says on every new head until the verdict comes, - // and what a recheck of the same head says, so the old verdict does not let it merge while the fresh check - // runs. - if (mergeCheckContexts.has(context) && state !== "pending") { - throw new Error(`${context} is the merge check's status, set by its verdict only (pending, for a check asked again, is the one state it takes by hand)`); - } - let d = String(description ?? "").replace(/\s+/g, " ").trim(); - if (d.length > 140) d = d.slice(0, 139) + "…"; - return { state: state as "pending" | "success" | "error" | "failure" | "warning", context, description: d, - ...(target && /^https?:\/\//.test(target) ? { target_url: target } : {}) }; -} - -/** What the status tool writes with: the forge, and the gate writer the merge check's statuses go through. */ -export interface StatusWriters { - setCommitStatus(owner: string, repo: string, sha: string, status: CommitStatus): Promise; - gates: { rechecked(owner: string, repo: string, sha: string, status: CommitStatus): Promise }; -} - -/** - * The `gitea_commit_status` tool. **The merge check's pending goes through the module's one writer** (novox/hq issue - * 456): a recheck's pending on mesh/merge-gate or mesh/repo-check takes its turn on the head's chain (gate.ts), so an - * old run's verdict in flight lands before it and a redelivered one is dropped. Served from the watcher's bundle - * (index.ts), the process that holds that writer: each bundle the runtime serves is its own process. The mesh's other - * statuses (mesh/delivery, mesh/delivery-group) are written as they were. - */ -export function commitStatusTool(writers: StatusWriters): ToolDefinition { - return { - name: "gitea_commit_status", - description: "Set one of the mesh's statuses on a commit (mesh/delivery, mesh/delivery-group): pending, success, error, failure or warning, a short description, and the page it links to. mesh/check, the status main requires, is refused: no tool writes it: it is written from the checker's own state. hq's mesh/repo-check (and mesh/merge-gate, kept for a revert) take pending only — a recheck putting them back until the fresh verdict sets them — on the head's whole sha, in their turn after the check's earlier writes.", - input: { - owner: { type: "string", description: "the repository owner" }, - repo: { type: "string", description: "the repository name" }, - sha: { type: "string", description: "the commit" }, - context: { type: "string", description: "the status's name, mesh/…" }, - state: { type: "string", description: "pending | success | error | failure | warning" }, - description: { type: "string", description: "one line" }, - target_url: { type: "string", description: "the page it links to (the pull request)" }, - }, - run: async (args) => { - const owner = checkName("owner", args.owner), repo = checkName("name", args.repo), sha = String(args.sha ?? ""); - const status = deliveryStatus(String(args.context), String(args.state), String(args.description ?? ""), - args.target_url ? String(args.target_url) : undefined); - if (mergeCheckContexts.has(status.context)) { - // The writer keeps a head by its whole sha, as the forge announced it: a short one would be another head. - if (!/^([0-9a-f]{40}|[0-9a-f]{64})$/i.test(sha)) { - throw new Error(`${JSON.stringify(args.sha)} is not a head's whole sha: the merge check's status names the head whole. Nothing was sent`); - } - await writers.gates.rechecked(owner, repo, sha, status); - } else { - await writers.setCommitStatus(owner, repo, sha, status); - } - return { set: status }; - }, - }; -} diff --git a/modules/gitea/gate.ts b/modules/gitea/gate.ts index e1c36bb56..e738bc70e 100644 --- a/modules/gitea/gate.ts +++ b/modules/gitea/gate.ts @@ -73,11 +73,11 @@ export class GateWriter { } /** - * A recheck's pending, asked through the gitea_commit_status tool (novox/hq issue 456): it takes its turn after every - * write already asked for, so it lands over them. Any other status of the merge check takes its turn on the same - * chain. A write that fails fails the caller. + * A status the seat git's `status` verb sets (seat.ts), a recheck's pending among them (novox/hq issue 456): it takes + * its turn after every write of the head already asked for, so it lands over them, and a later write lands over it. A + * write that fails fails the caller. */ - rechecked(owner: string, repo: string, sha: string, status: CommitStatus): Promise { + status(owner: string, repo: string, sha: string, status: CommitStatus): Promise { return this.turn(this.head(owner, repo, sha), () => this.forge.setCommitStatus(owner, repo, sha, status)); } diff --git a/modules/gitea/guard.ts b/modules/gitea/guard.ts index e285ab907..372f112ad 100644 --- a/modules/gitea/guard.ts +++ b/modules/gitea/guard.ts @@ -14,9 +14,8 @@ // the tools' account the only one that merges; it requires no status, as no rule of the mesh does (ADR 0534 §5); // - `gitea_default_branch_set` and `gitea_pull_base_set` are the two settings gitea_api refuses, as named tools: a branch that // exists (and, for a default, that a rule guards), a stated reason, the tools' account, the value before and after (issue 1422); -// - `gitea_merge_pull_request` merges only into a branch that refuses its push, only a head whose mesh/check passed -// against the trunk's head, approved by the operator where the branch requires no status (ADR 0534 §4), and only -// that head; +// - the seat git's `merge-pull` (seat.ts) merges only into a branch that refuses its push, and only the head the +// operator's Approve names (ADR 0661); // - `gitea_delete_repo` refuses a repository any protection rule guards. // // Pure, so it is tested without a forge; the tools do the asking (tools/index.ts). @@ -165,7 +164,7 @@ export function judgeApiCall(method: string, rawPath: string, body: unknown): Ap `writes — issues, comments, labels, pull requests (never their merge), branches, and files on a branch ` + `that is not the default branch and that no protection rule names. The forge's administration, a ` + `branch's protection, a repository's settings, collaborators, teams, hooks, keys, tokens and statuses ` + - `are the operator's at the forge, or have their own tool (gitea_merge_pull_request, gitea_commit_status, ` + + `are the operator's at the forge, or have their own tool (the seat git's merge-pull and status, ` + `gitea_branch_protection_set, which never lets a person push). Nothing was sent`, }; } diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index bb15ebc64..2f927b0c4 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -13,9 +13,8 @@ // statuses, `mesh/merge-gate` and `mesh/repo-check`, are written for no base that main requires (issue 1301); main // requires `mesh/check`, the checker's, everywhere, hq included since ADR 0527 §2. // -// Reads mesh-delivery-checker.owed — what the checker owes the forge, mesh/check on each head it judged — by a watch, and -// writes it (check.ts; ADR 0521, R4e finding R2-1). No tool writes mesh/check (finding K1), and no event: the watch hands -// over the latest value of every key whenever this module starts. +// It reads no other module's state: mesh/check, and every act of a delivery part on the forge, arrive as verbs of the seat +// git (seat.ts), asked by the forge bridge (novox/hq design 47 *The seat git*). // // This module is the one writer of mesh/merge-gate. It hears no build seat's event: since R4g a check's steps are asks // on the build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525 §9; issue 1887). @@ -25,9 +24,6 @@ // request into it go back to pending with the reason. Nothing is run for it. A verdict that lands judged against a trunk // commit that is no longer the trunk's head is said out of date in place of its pass or failure. // -// Serves gitea_commit_status and gitea_merge_pull_request from this bundle, not the tools', so a recheck's pending and -// the merge tool's ask of a check reach that one writer (novox/hq issue 456, ADR 0323). -// // Every pull request the forge holds is announced, whatever its repository: the controller holds the // module graph and decides what is checked — a repository is never asked to opt in. // @@ -42,14 +38,16 @@ import { emit } from "@novox/mesh-sdk/events"; import { registerModuleTools } from "@novox/mesh-sdk/tools"; import { statesAnnouncement } from "./kept.js"; -import { commitStatusTool } from "./delivery.js"; -import { CHECKER_OWED, CheckStatuses, watchOwed, type OwedChange } from "./check.js"; import { state } from "@novox/mesh-sdk/state"; import { on } from "@novox/mesh-sdk/events"; import { broker } from "@novox/mesh-sdk/messaging"; -import { ViewFollower, unansweredFinding, viewTools, type ViewAnswer, type ViewEvent } from "./views.js"; +import { STATUS_STATE, keepStatus, statusTools, viewWaitsFinding, type Follower, type Status } from "@novox/mesh-sdk/views"; +import type { CatalogRevisionEvent, RevisionInput } from "@novox/mesh-delivery-contracts-mesh-catalog"; +import { CATALOG_REVISION_EVENT, CatalogView, findingValue, unansweredFinding } from "./catalog.js"; import { announce, keepWatermark, readWatermark } from "./announce.js"; -import { mergeTool } from "./merge.js"; +import { SEAT_ONLY, sayOnSeat } from "./seatevents.js"; +import { FORGE_CONTAINER, seatTools } from "./seat.js"; +import { run } from "./delivery.js"; import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { keepMainRules, mainRulesStep } from "./mainrules.js"; import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, headKey, requiredOf, writesGate, type Announced } from "./pulls.js"; @@ -60,7 +58,7 @@ import { keepSiblings, readSiblings, siblingsFile, weighSiblings } from "./sibli import { bindInBackground, bindUntilHeld } from "./bind.js"; import { announcementOf, weighTrunks } from "./trunk.js"; import { ReadAll, readAllTools } from "./readall.js"; -import { DeliveredWrites, INDEXER_MODULES_EVENTS, INDEXER_MODULES_TOOL, INDEXER_OWNER, writeTools, type ModulesChange } from "./writes.js"; +import { DeliveredWrites, writeTools } from "./writes.js"; import { TOOLS_ACCOUNT, TOOLS_ORG, meshAccounts } from "./tools-account.js"; import { PACKAGES_TOKEN_REFUSED, keepPackagesToken, packagesToken, renewOnRefusal } from "./packages-account.js"; @@ -363,14 +361,6 @@ const gates = new GateWriter({ headChecks: (owner, repo, sha) => (gitea as GiteaClient).headChecks(owner, repo, sha), }); -// The one writer of mesh/check, from the checker's owed (check.ts; ADR 0521). -const checks = new CheckStatuses({ - setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), -}); - -// **The status tool is served here, beside the writer** (novox/hq issue 456): the runtime starts each bundle as its own -// process, so a recheck's pending written from the tools' bundle bypassed this writer's chain. Without a forge to -// reach, nothing is served, as the tools' bundle serves nothing then. /** How fast the polls look: 30 s until the forge's events are heard, five minutes while they are; and what * gitea_events says of them (webhook.ts). */ const pace = new Pace(); @@ -391,19 +381,22 @@ let readAllOnce: ReadAll | null = null; const readAll = (): ReadAll => (readAllOnce ??= new ReadAll(GiteaAdmin.fromEnv(), meshAccounts(process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT))); // **The merging account writes every repository that builds a registered module** (writes.ts; the operator, -// 2026-10-26; issue 1945): which those are is the indexer's state, watched below. Made on first use, as readAll. +// 2026-10-26; issue 1945): which those are is the catalog half's word, followed below. Made on first use, as readAll. let writesOnce: DeliveredWrites | null = null; const writes = (): DeliveredWrites => (writesOnce ??= new DeliveredWrites(GiteaAdmin.fromEnv(), - process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT, TOOLS_ORG, (l) => console.log(`[gitea] ${l}`), (type, body) => emit(type, body))); + process.env.MESH_GITEA_TOOLS_USER || TOOLS_ACCOUNT, TOOLS_ORG, (l) => console.log(`[gitea] ${l}`), async (type, body) => { await sayOnSeat(type, body as Record); })); -// The first view of the indexer's modules, once the module follows it (below); gitea_first_views says where it stands. -let modulesView: { follower: ViewFollower; owner: string; tool: string; state: string } | undefined; +// The first view of the catalog half (catalog.ts), made once the module follows it (below). Its status is kept in the +// module's state first-views, which gitea_first_views reads; gitea_first_view_again asks it again by hand (mesh-sdk views). +const firstViews = state(STATUS_STATE); +const followers: Follower[] = []; -// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). -registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...viewTools(() => modulesView, (act) => broker().request("seat:mesh-hand-acts.hand-act", act)), ...(gitea ? [commitStatusTool({ - setCommitStatus: (owner, repo, sha, status) => (gitea as GiteaClient).setCommitStatus(owner, repo, sha, status), - gates, -}), mergeTool(gitea, { gates })] : [])]); +// gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). The seat git's verbs are +// served here, beside the one writer of a head's statuses: the seat's status takes its turn on the head's chain, after the +// announcement's pending and a trunk's move, so a recheck's pending never lands over a later verdict (issue 456). +registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...statusTools("gitea", "gitea_first_views", "gitea_first_view_again", followers, firstViews, + async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), + ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, statuses: gates }) : [])]); /** What the forge's event asks of the announcer at once (issue 729, 1422): see watch.ts. */ const actOnEvent = (client: GiteaClient, event: ForgeEvent): Promise => actOn(event, { @@ -434,7 +427,11 @@ async function listenForForge(client: GiteaClient): Promise { const host = process.env.MESH_GITEA_HOOK_HOST ?? "0.0.0.0"; const server = receiver({ secret, port, heard: (kind, delivery) => pace.heard(kind, delivery), refused: (reason, from) => pace.refused(reason, from), - publish: (event) => emit(event.name, event.body), + // The seat git's forge events are said on the seat (seatevents.ts); the forge's others stay the module's own. + publish: async (event) => { + if (!SEAT_ONLY.has(event.name)) await emit(event.name, event.body); + await sayOnSeat(event.name, event.body as Record); + }, act: async (event) => { // A repository made, or one not yet seen readable, is granted read before anything acts on it (readall.ts). await Promise.resolve().then(() => readAll().onEvent(event)).catch((err) => @@ -493,7 +490,7 @@ if (gitea) { // polls slow to every five minutes and announce only what no event said (Pace); otherwise they keep 30 s. // **Not awaited** (novox/hq issue 1726): the receiver's set-up asks the forge's admin API, and the runtime kills a bundle // whose import has not ended in its 10 s handshake and never starts it again; a forge slow to answer while every bundle - // of the node starts left gitea_commit_status, gitea_merge_pull_request and gitea_events served nowhere. The pass below + // of the node starts left gitea_events served nowhere. The pass below // runs again once it is listening, so nothing said before is missed; listenForForge says its own failure. const listenOk = listenForForge(client); // The start's one pass over every repository (readall.ts), not awaited for the same reason. A forge not answering yet @@ -513,26 +510,29 @@ if (gitea) { } catch (err) { console.error(`[gitea] the mesh's accounts are not given read on every repository: no site administrator's login — ${err instanceof Error ? err.message : err}`); } - // The indexer's modules, watched: the watch hands every key's latest value at its start and then each change, so a - // repository that builds a registered module is made writable at the start and when its first module is registered - // (writes.ts). A read not granted yet is said and asked again until it holds. + // The catalog half, followed: its first view hands every registered module's entry at the start and each newer trunk + // revision what changed, so a repository that builds a registered module is made writable at the start and when its + // first module is registered (writes.ts); a candidate's revision moves nothing, and nothing is granted before the first view. try { const w = writes(); - // ADR 0615: asked once of the indexer's read tool, then kept by its events; the indexer's state itself is not read. - const modules = new ViewFollower({ owner: INDEXER_OWNER, state: "modules", tool: INDEXER_MODULES_TOOL, - ask: (after) => broker().request(INDEXER_MODULES_TOOL, after ? { values: true, after } : { values: true }) as Promise, - take: (change) => w.take(change as ModulesChange), - // Unanswered past its bound is a finding of the module's own, which the condition keeper raises (ADR 0615 §6). - raise: async (asked, error) => { - const f = unansweredFinding("gitea", INDEXER_OWNER, "modules", INDEXER_MODULES_TOOL, asked, error, new Date()); - await state("findings").put(f.key, f.value); - }, - clear: async () => { - await state("findings").delete(unansweredFinding("gitea", INDEXER_OWNER, "modules", INDEXER_MODULES_TOOL, new Date(), "", new Date()).key); - } }); - void bindUntilHeld(INDEXER_MODULES_EVENTS, () => on(INDEXER_MODULES_EVENTS, (event) => modules.hear(event.body))); - modulesView = { follower: modules, owner: INDEXER_OWNER, tool: INDEXER_MODULES_TOOL, state: "modules" }; - void modules.start(); + const findings = state("findings"); + const put = async (f: { key: string; value: unknown }) => { await findings.put(f.key, f.value); }; + const view = new CatalogView((verb, input: RevisionInput) => broker().request(verb, input), { + said: keepStatus(firstViews), + changed: (key, value, deleted) => w.take({ key, op: deleted ? "delete" : "put", value }), + // Unanswered past its bound, or a revision waited for past mesh-sdk's WAITS_BOUND_MS, is a finding of the module's + // own, which the condition keeper raises (ADR 0615 §6; design 47 *A part with no view says it waits*). + raise: (s) => put(findingValue(unansweredFinding("gitea", s, new Date()))), + clear: async (s) => { await findings.delete(findingValue(unansweredFinding("gitea", s, new Date())).key); }, + raiseWaits: (s) => put(findingValue(viewWaitsFinding("gitea", s, new Date()))), + clearWaits: async (s) => { await findings.delete(findingValue(viewWaitsFinding("gitea", s, new Date())).key); }, + log: (line) => console.log(`[gitea] ${line}`), + }); + followers.push(view.follower); + void bindUntilHeld(CATALOG_REVISION_EVENT, () => on(CATALOG_REVISION_EVENT, (event) => + // What is no catalog-revision event is said and taken: asked again, it would be no more readable. + view.hear(event.body).catch((err) => console.error(`[gitea] ${err instanceof Error ? err.message : err}`)))); + void view.follower.start().then(() => w.viewed()); } catch (err) { console.error(`[gitea] the merging account is not given write on the delivered repositories: no site administrator's login — ${err instanceof Error ? err.message : err}`); } @@ -570,7 +570,4 @@ if (gitea) { // No consumer is bound since ADR 0571 clause 13 took the controller's checked out of this module; a consumer added later // is bound again until it holds, and not awaited (bind.ts, issue 1793). bindInBackground([], () => console.log("[gitea] watching for new repositories and merged pull requests")); - // The checker's mesh/check, from its owed state (check.ts; ADR 0521): a read not granted yet is said once and asked again, - // apart from the binds above, so neither waits for the other. - void watchOwed((handler) => state(CHECKER_OWED).watch((change) => handler(change)), checks); } diff --git a/modules/gitea/internal/packageseats/holder.go b/modules/gitea/internal/packageseats/holder.go index ea56b67f0..70c98c080 100644 --- a/modules/gitea/internal/packageseats/holder.go +++ b/modules/gitea/internal/packageseats/holder.go @@ -7,9 +7,11 @@ package packageseats import ( + "encoding/json" "errors" "fmt" "strings" + "sync" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -77,13 +79,91 @@ func DefaultCallers() Callers { VerbVersions: {stdio.AnyCaller}} } -// configured holds the config value `callers`: asked of the runtime once, by a watch, and replaced on each change of -// the value, so a call never asks (issue 1966). -var configured = stdio.HoldVerbCallers("callers", Verbs, DefaultCallers()) - -// ConfiguredCallers is the config value `callers` as mesh-config resolved it, held from its watch; unset is DefaultCallers. +// ConfiguredCallers is the config value `callers` as mesh-config resolved it, its lines of the package seats' verbs; unset, +// or with none of them, is DefaultCallers. It is held from a watch of the value and replaced on each change, so a call +// never asks the runtime (issue 1966). The same value carries the lines of the other seat the module holds (git, served +// by its TypeScript side), so each holder takes its own verbs' lines and neither refuses the other's. func ConfiguredCallers() (Callers, error) { return configured() } +var ( + watchMu sync.Mutex + heldMu sync.Mutex + heldList Callers + heldBad error +) + +// configured reads the held list; the first call starts the watch, and a change that cannot be read keeps the list last +// read, said and never widened. +func configured() (Callers, error) { + if l, err := read(); l != nil || err != nil { + return l, err + } + watchMu.Lock() + defer watchMu.Unlock() + if l, err := read(); l != nil || err != nil { + return l, err + } + if err := stdio.WatchConfig("callers", func(c stdio.ProvisionChange) error { + heldMu.Lock() + defer heldMu.Unlock() + if c.Refused != "" { + heldBad = fmt.Errorf("the config value callers is not answered: %s", c.Refused) + return nil + } + l, err := OwnCallers(c.Value) + if err != nil { + heldBad = fmt.Errorf("the config value callers cannot be read: %w", err) + return nil + } + heldList, heldBad = l, nil + return nil + }); err != nil { + return nil, fmt.Errorf("the config value callers cannot be read: %w", err) + } + l, err := read() + if l == nil && err == nil { + err = errors.New("the config value callers cannot be read: the runtime handed over no value") + } + return l, err +} + +func read() (Callers, error) { + heldMu.Lock() + defer heldMu.Unlock() + if heldList != nil { + return heldList, nil + } + return nil, heldBad +} + +// OwnCallers reads the package seats' lines of the config value as the runtime answered it: a JSON object of verb to +// callers, or that object written as a string. A value that cannot be read is an error, never the defaults. +func OwnCallers(raw json.RawMessage) (Callers, error) { + text := strings.TrimSpace(string(raw)) + var quoted string + if json.Unmarshal(raw, "ed) == nil { + text = strings.TrimSpace(quoted) + } + if text == "" || text == "null" { + return DefaultCallers(), nil + } + var all map[string]json.RawMessage + if err := json.Unmarshal([]byte(text), &all); err != nil { + return nil, fmt.Errorf("callers is not a JSON object of verb to callers: %w", err) + } + own := map[string]json.RawMessage{} + for _, verb := range Verbs { + if line, ok := all[verb]; ok { + own[verb] = line + } + } + if len(own) == 0 { + return DefaultCallers(), nil + } + b, _ := json.Marshal(own) + return stdio.ParseVerbCallers(string(b), Verbs, DefaultCallers()) +} + // Publish is a publish as the seats' contract names it (mesh-contracts npmpackageregistry and gopackageregistry share // one shape): what the tools bundle reads from either seat's PublishInput. type Publish struct { diff --git a/modules/gitea/internal/packageseats/holder_test.go b/modules/gitea/internal/packageseats/holder_test.go index e8e9bdc14..91e062cdc 100644 --- a/modules/gitea/internal/packageseats/holder_test.go +++ b/modules/gitea/internal/packageseats/holder_test.go @@ -303,4 +303,30 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) { t.Errorf("removed %v, said %+v", b.forge.removed, b.said) } }) + + t.Run("the callers value: its own verbs' lines, the other seat's left to it, unset or none of its own the defaults", func(t *testing.T) { + manifest := `"{\"publish\":[\"mesh-build-go\"],\"remove\":[\"mesh-delivery-builder\"],\"pull\":[\"mesh-delivery-gate\"]}"` + for _, row := range []struct { + raw string + publish []string + fails bool + }{ + {raw: manifest, publish: []string{"mesh-build-go"}}, + {raw: `{"pull":["mesh-delivery-gate"]}`, publish: DefaultCallers()[VerbPublish]}, + {raw: `""`, publish: DefaultCallers()[VerbPublish]}, + {raw: `{"publish":"mesh-build-go"}`, fails: true}, + {raw: `"not json"`, fails: true}, + } { + got, err := OwnCallers([]byte(row.raw)) + if row.fails != (err != nil) { + t.Fatalf("%s: err %v", row.raw, err) + } + if !row.fails && strings.Join(got[VerbPublish], ",") != strings.Join(row.publish, ",") { + t.Errorf("%s: publish %v, want %v", row.raw, got[VerbPublish], row.publish) + } + if _, theirs := got["pull"]; theirs { + t.Errorf("%s: the seat git's line is not the package seats'", row.raw) + } + } + }) } diff --git a/modules/gitea/kept.ts b/modules/gitea/kept.ts index a591b88f0..eb170dfa7 100644 --- a/modules/gitea/kept.ts +++ b/modules/gitea/kept.ts @@ -7,6 +7,7 @@ export const KEPT: readonly { readonly name: string; readonly readable?: boolean { name: "announced" }, { name: "bound", readable: true }, { name: "findings", readable: true }, + { name: "first-views" }, ]; /** The announcement of these states, in the layout mesh-sdk's go/announce reads (novox/hq ADR 0495 clause 2): version 1, diff --git a/modules/gitea/mainrules.ts b/modules/gitea/mainrules.ts index 6137559e4..e8ac249f7 100644 --- a/modules/gitea/mainrules.ts +++ b/modules/gitea/mainrules.ts @@ -15,19 +15,6 @@ export function keepMainRules(env: NodeJS.ProcessEnv = process.env): boolean { } catch { return false; } } -/** The setting check-gates-merge (the operator, 2026-10-25 22:56: "run the check locally only, but not as a gate, it loses - * us too much time"): whether the merge tool judges mesh/check before it merges. Absent, unreadable or anything but false - * is true; false lets a head the operator's Approve names merge with mesh/check pending, failed, missing or judged on an - * older trunk (mesh-delivery-gate's setting of the same name stops waiting for it before the Approve). */ -export function checkGatesMerge(env: NodeJS.ProcessEnv = process.env): boolean { - const file = env.MESH_GITEA_CONFIG_FILE; - if (!file) return true; - try { - const v = (JSON.parse(readFileSync(file, "utf8")) as Record)["check-gates-merge"]; - return !(v === false || v === "false"); - } catch { return true; } -} - /** The 10-minute step: puts the rules right only when kept; else says once that they are left and writes nothing. */ export function mainRulesStep(keep: boolean, put: () => Promise, say: (line: string) => void): () => Promise { let said = false; diff --git a/modules/gitea/merge.ts b/modules/gitea/merge.ts deleted file mode 100644 index 96f6d69a2..000000000 --- a/modules/gitea/merge.ts +++ /dev/null @@ -1,279 +0,0 @@ -// The forge's merge tool, `gitea_merge_pull_request` (novox/hq ADR 0266, issue 381, ADR 0323). -// -// **Served from the watcher's bundle, beside the merge check's one writer** (index.ts, gate.ts; novox/hq issue 456): -// when a verdict is judged against an earlier trunk commit than the trunk's head, the tool asks the head's check and -// puts its statuses back to pending through that writer, so where the check runs is said and no second check is asked. -// Each bundle the runtime serves is its own process; from the tools' bundle the pending bypassed the writer's chain. -// -// **It judges mesh/check whether or not the base requires it** (issue 1301; ADR 0447 §1, §4; ADR 0534 §4): the checker's -// verdict, whose line names the trunk it judged ("pass on "), must be a pass against the trunk's head. -// The forge requires no status to merge (ADR 0534 §1); where a base still requires one of the controller's old statuses, -// until the forge's holder takes it off, that status judges, as before. A mesh/check of another trunk is refused and -// nothing is asked: the checker sets it out of date on the trunk's move and checks it again. Where the base requires no -// status, only a head the operator's Approve names (approved_heads) merges (ADR 0534 §4, ADR 0421 §9). - -import { announce } from "./announce.js"; -import type { ToolDefinition } from "@novox/mesh-sdk/tools"; -import type { CommitStatus, GiteaClient } from "./client.js"; -import { mergeableUnder, newestByContext } from "./checks.js"; -import { MERGE_METHODS, Refused, checkName } from "./guard.js"; -import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT, maintained, requiredOf } from "./pulls.js"; -import { approvedHeadsOf, groupRefusal, memberKey, namesSiblings, notApproved, readSiblings, siblingOutOfDate, siblingsFile } from "./siblings.js"; -import { checkGatesMerge } from "./mainrules.js"; -import { announcementOf, askedWords, outOfDateWords, sameCommit, twelve, verdictAtTrunk, waitsForComposed } from "./trunk.js"; - -/** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */ -const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/; - -/** What the merge tool reaches beside the forge: the merge check's one writer, how an event is said, the module's - * state directory, and the clock. */ -export interface MergeDeps { - gates: { rechecked(owner: string, repo: string, sha: string, status: CommitStatus): Promise }; - emit?: (type: string, body: unknown) => Promise; - stateDir?: string; - now?: () => Date; - /** Whether mesh/check gates the merge: the setting check-gates-merge, read at each merge (mainrules.ts). */ - checkGates?: () => boolean; -} - -export function mergeTool(gitea: GiteaClient, given: MergeDeps): ToolDefinition { - const deps = { ...given, emit: given.emit ?? announce, stateDir: given.stateDir ?? process.env.MESH_GITEA_STATE_DIR, - now: given.now ?? (() => new Date()), checkGates: given.checkGates ?? (() => checkGatesMerge()) }; - return { - name: "gitea_merge_pull_request", - description: "Merge a pull request at the head you name — the commit you reviewed — when that is still its head, " + - "its base refuses this account's push, its mesh/check passed judged against the trunk's head as it is now, and, " + - "where the base requires no status (novox/hq ADR 0534), the operator's Approve names it in approved_heads; " + - "optionally deleting the source branch afterwards. Refused " + - "otherwise, with nothing merged: no tool merges past a check, and none merges a head that moved after it was " + - "reviewed. A verdict judged against an earlier trunk commit is refused, naming both, and the head's check against " + - "the trunk's head is asked once: ask again at the same head " + - "once it passes — the tool never merges by itself.", - input: { - owner: { type: "string", description: "the repository owner" }, - repo: { type: "string", description: "the repository name" }, - number: { type: "number", description: "the PR number" }, - head: { type: "string", description: "the commit you reviewed, as its sha or at least its first 12 characters: " + - "refused unless it is the pull request's head" }, - method: { type: "string", description: "merge (default) | fast-forward-only | squash | rebase | rebase-merge. " + - "Only merge and fast-forward-only can be read back to prove the head you named is what landed" }, - delete_branch: { type: "boolean", description: "delete the source branch after merge (default true)" }, - approved_heads: { type: "array", items: { type: "string" }, description: "the heads the operator's Approve covers, " + - "each owner/repo@, one per member of the group (novox/hq ADR 0421 §9): refused unless this pull " + - "request's repository is named at the head it has now. Required when the pull request's verdict read siblings, " + - "where each sibling's head must be named too; there the Approve takes the place of an approving review" }, - }, - run: async (args) => { - const owner = checkName("owner", args.owner); - const repo = checkName("name", args.repo); - const number = Number(args.number); - const method = args.method ? String(args.method) : "merge"; - if (!MERGE_METHODS.has(method)) throw new Refused(`${method} is not a merge method: ${[...MERGE_METHODS].join(", ")}`); - const deleteBranch = args.delete_branch === undefined ? true : Boolean(args.delete_branch); - // **Only a head whose required statuses all succeeded** (novox/hq ADR 0266): read here as well as by the - // forge, which holds the tools' own account to it (novox/hq ADR 0365 §2). The protection is the one the forge - // applies to the base, read from the branch: the rules' own list is a repository administrator's. A rule that - // cannot be read is no rule passed. - // **Only the head that was reviewed** (novox/hq issue 381): mesh-controller#197 was reviewed at 065cfa0d, - // force-pushed to 3ced96f, checked green there, and merged at 3ced96f, because this tool merged whatever the - // head was and nothing told it which head the review had approved. The caller names it; a head that moved - // since is refused, and the forge is handed the same head, so a push between this read and the merge is - // refused by the forge too. - const named = args.head === undefined ? "" : String(args.head).trim().toLowerCase(); - if (!named) { - throw new Refused(`${owner}/${repo}#${number} is not merged: name the head you reviewed (head), so that only ` + - `that commit can land (novox/hq issue 381). Nothing was merged`); - } - // Twelve characters, not seven: an author who force-pushes can make a commit whose first seven match the - // reviewed one in seconds, and the review would then merge it. Twelve cannot be matched that way. - if (!REVIEWED_HEAD.test(named)) { - throw new Refused(`${owner}/${repo}#${number} is not merged: head ${named} does not name one commit — give its ` + - `sha, or at least its first 12 characters. Nothing was merged`); - } - const pull = await gitea.getPullRequest(owner, repo, number); - if (pull.merged) throw new Refused(`${owner}/${repo}#${number} is already merged`); - if (!pull.head_sha || !pull.base) throw new Refused(`${owner}/${repo}#${number} names no head commit or base branch`); - if (!pull.head_sha.toLowerCase().startsWith(named)) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its head is ${pull.head_sha.slice(0, 12)}, not the ` + - `${named} you named. If you reviewed an earlier head, review the one it has now, then name it. Nothing was merged`); - } - // **Only a head the operator's Approve covers** (novox/hq ADR 0421 §9): mesh-delivery hands the heads its Approve - // named; a pull request whose head differs is refused, so a push between the Approve and this call merges nothing. - const approved = approvedHeadsOf(args.approved_heads); - if (approved) { - const why = notApproved(approved, `${owner}/${repo}`, number, String(pull.head_sha)); - if (why) throw new Refused(`${owner}/${repo}#${number} is not merged: ${why}. A merge lands the heads the operator ` + - `approved (novox/hq ADR 0421). Nothing was merged`); - } - const rule = await gitea.appliedProtection(owner, repo, pull.base); - // Only into a branch whose protection refuses this account's push: a pull request whose base was moved to a - // branch nothing guards would land where anyone pushes (novox/hq ADR 0266, ADR 0534 §4). - if (!rule || rule.enable_push !== false) { - throw new Refused(`${owner}/${repo}#${number} merges into ${pull.base}, which no protection rule refusing ` + - `a push guards: a tool merges only into such a branch (novox/hq ADR 0266, ADR 0534). Nothing was merged`); - } - // **The verdict must be judged against the trunk's head** (novox/hq ADR 0323 §5, issue 341): a check judges the - // head merged into a named trunk commit, and a merge of another trunk is not the one that check judged. Compared - // here at the moment the merge is asked, not left to the statuses the trunk's move resets a pass later. When they - // differ nothing is merged, and the head's check against the trunk's head is asked once — the announcement a new - // head makes — never again while one is on its way. The caller asks again at the reviewed head once it passes. - const trunkHead = await gitea.branchHead(owner, repo, pull.base); - const checks = await gitea.headChecks(owner, repo, pull.head_sha); - const required = requiredOf(rule) ?? []; - // **The forge requires no status; the operator's Approve and mesh/check do** (ADR 0534 §4): where the base requires - // no status, only a head the Approve names merges, and mesh/check judges it as where it was required. - if (required.length === 0 && !approved) { - throw new Refused(`${owner}/${repo}#${number} is not merged: ${pull.base} requires no status, so a merge into it ` + - `lands only a head the operator's Approve names: give approved_heads with ${owner}/${repo}@ ` + - `(novox/hq ADR 0534, ADR 0421 §9). Nothing was merged`); - } - const kept = maintained(required); - const byCheck = required.includes(CHECK_STATUS) || kept.length === 0; - // **check-gates-merge false** (the operator, 2026-10-25): mesh/check is information, not a gate. A head the operator's - // Approve names merges whatever its mesh/check says, pending, failed, missing or judged on an older trunk; without - // approved_heads the check still judges. - if (byCheck && (deps.checkGates() || !approved)) { - // **mesh/check is the checker's** (ADR 0447 §4): it is judged here against the trunk's head, and refused when it - // is not current; nothing is written or asked — the checker checks a head again when its trunk moves. - const check = checks[CHECK_STATUS]; - const at = verdictAtTrunk(checks, trunkHead, CHECK_STATUS); - if (!check || check.state === "pending") { - throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} is ` + - `${check ? `pending (${check.description ?? ""})` : "not set yet"}; the checker answers it. Ask again at ` + - `${twelve(pull.head_sha)} once it passes against ${pull.base}'s head ${twelve(trunkHead)}. Nothing was merged`); - } - if (check.state !== "success") { - throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} is ${check.state} ` + - `(${check.description ?? ""}). A pull request merges once its check passes (novox/hq ADR 0534). Nothing was merged`); - } - if (!at.current) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its ${CHECK_STATUS} ` + - `${at.judged ? `was judged on trunk ${twelve(at.judged)}` : "names no trunk commit it was judged on"}, and ` + - `${pull.base} is now ${twelve(trunkHead)}. The checker checks it again against the trunk's head (novox/hq ` + - `ADR 0447); this tool asks nothing. Ask again at ${twelve(pull.head_sha)} once it passes. Nothing was merged`); - } - } - // Where the base still requires the controller's status (mesh/merge-gate, only on a - // revert), the first it requires judges, as before (ADR 0323 §5). - const judge = kept[0] ?? CHECK_CONTEXT; - const at = kept.length ? verdictAtTrunk(checks, trunkHead, judge) : { current: true, onItsWay: false }; - if (!at.current) { - // A verdict put out of date by a sibling's move says so in its own words (ADR 0382 §5); it is asked again as a - // trunk's move is, and a failed ask puts those words back, not the trunk's. - const siblingWords = siblingOutOfDate(checks[judge]?.description) ? checks[judge]?.description : undefined; - const judged = siblingWords ? `was put out of date by a sibling's move (${siblingWords})` - : at.judged ? `was judged against trunk ${twelve(at.judged)}` : "names no trunk commit it was judged against"; - const head = twelve(pull.head_sha), now = twelve(trunkHead); - // **A member that waits for its group's composed check is not announced again** (novox/hq issue 1106): its own - // check can only say "composed" once more, so an ask here brought a verdict naming no trunk and the next merge was - // refused the same way for ever. The composed check is mesh-delivery's to ask; this says whose verdict is awaited. - if (waitsForComposed(checks[judge])) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its merge gate ${checks[judge]?.description}; the ` + - `verdict that names its trunk commit is its group's composed check's, and mesh-delivery asks it, this tool asks no check ` + - `(main is now ${now}). Ask again once the group's composed check has passed. Nothing was merged`); - } - if (at.onItsWay) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` + - `${now}. Its check is already asked and not yet answered (${checks[judge]?.description}), so it is not ` + - `asked again. Ask again at ${head} once both statuses pass: this tool never merges by itself ` + - `(novox/hq ADR 0323). A check that never answers is asked again with mesh-delivery.recheck. Nothing was merged`); - } - // Pending first, so a second ask finds the check on its way; then the head is announced as a new head is. An - // announcement that cannot be made puts the statuses back to out of date, so the next ask asks again rather - // than wait for ever on a check nobody asked. - // The statuses the base requires of the controller's (issue 1301): the one that judges, and the repository's own - // check beside it where a revert has the base require both. - const contexts = kept.filter((c) => c === judge || (c === REPO_CHECK_CONTEXT && checks[REPO_CHECK_CONTEXT])); - // Through the module's one writer of the merge check (gate.ts, novox/hq issue 456): the pending opens a new - // check of the head, so where it runs is said ("building on …") and its verdict is not weighed as older. - const say = async (description: string) => { - for (const context of contexts) { - await deps.gates.rechecked(owner, repo, String(pull.head_sha), { state: "pending", context, description, - ...(pull.html_url ? { target_url: pull.html_url } : {}) }); - } - }; - await say(askedWords(deps.now(), trunkHead)); - try { - const cloneURL = (await gitea.getRepo(owner, repo)).clone_url; - await deps.emit("pull.updated", await announcementOf(gitea, owner, repo, cloneURL, pull)); - } catch (err) { - await say(siblingWords ?? outOfDateWords(at.judged, trunkHead)).catch(() => {}); - throw new Error(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` + - `${now}, and its check could not be asked: ${err instanceof Error ? err.message : err}. Nothing was merged`); - } - throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict ${judged}, and ${pull.base} is now ` + - `${now}. The check of head ${head} merged into trunk ${now} was asked. Ask again at ${head} once both ` + - `statuses pass: this tool never merges by itself (novox/hq ADR 0323). In a busy repository another merge may ` + - `move the trunk again first; this is then refused and asked once more. A check that never answers is asked ` + - `again with mesh-delivery.recheck. Nothing was merged`); - } - const statuses = newestByContext((await gitea.combinedStatus(owner, repo, pull.head_sha)).statuses); - const verdict = mergeableUnder(pull.base, rule, statuses); - if (verdict.mergeable !== true) { - throw new Refused(`${owner}/${repo}#${number} is not merged: ${verdict.says}. A pull request merges once ` + - `its checks succeed; no tool merges past them (novox/hq ADR 0266)`); - } - // **A member judged beside its siblings merges only with its group passed and reviewed** (novox/hq ADR 0382 §6): the - // siblings its verdict names are the holder's record (siblings.ts); a status that says it read siblings with no record - // of them is no verdict to merge on, never read as none. - // Under mesh/check (issue 1301) the group is the checker's one verdict on every member's head (ADR 0447 §2) and the - // Approve's heads above; a record the controller's verdicts left before the switch judges nothing there. - const heard = byCheck ? undefined : readSiblings(siblingsFile(deps.stateDir))[memberKey(owner, repo, number)]; - const siblings = heard && sameCommit(heard.head, pull.head_sha) ? heard.siblings : []; - if (!siblings.length && !byCheck && - namesSiblings(statuses.find((s) => s.context === CHECK_CONTEXT)?.description)) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict read siblings, and which ones is not held ` + - `here, so the group cannot be weighed. Ask its check again at ${twelve(pull.head_sha)}. Nothing was merged`); - } - if (siblings.length) { - const missing = await groupRefusal({ headChecks: (o, r, sha) => gitea.headChecks(o, r, sha) }, - { owner, repo, number, head: String(pull.head_sha) }, siblings, approved); - if (missing) { - throw new Refused(`${owner}/${repo}#${number} is not merged: its verdict was judged beside ${siblings.length} ` + - `sibling(s), and ${missing}. A member judged beside its siblings merges once its group passed at the heads that ` + - `verdict read and the operator's Approve covers each of them (novox/hq ADR 0382, 0421). Nothing was merged`); - } - } - await gitea.mergePullRequest(owner, repo, number, method, deleteBranch, pull.head_sha); - // The merge commit only exists now; answered so the caller can follow what is built from it. - // pull.merged is the events entrypoint's to announce (index.ts), once, within its poll. - // From here the merge has happened. Nothing below may read as if it had not: a read-back that fails or cannot - // be done is said beside merged: true, and only a proven other head is an error — one that says the merge happened. - const answer: Record = { merged: true, number, method, head: pull.head_sha, deleted_branch: deleteBranch }; - let landed: string | undefined; - try { - const merged = await gitea.getPullRequest(owner, repo, number); - answer.merge_commit_sha = merged.merge_commit_sha; - // What landed, read back from the commit the merge made rather than from the forge's record of the pull - // request, which is the very thing a race would leave behind. A merge commit's second parent is the head it - // joined; a fast-forward lands the head itself. A squash or a rebase makes new commits that name no head. - if (!merged.merge_commit_sha) { - answer.read_back = "the forge named no merge commit, so what landed cannot be read back"; - } else if (method === "merge") { - const parents = await gitea.commitParents(owner, repo, merged.merge_commit_sha); - if (parents.length === 2) landed = parents[1]; - else answer.read_back = `the merge commit has ${parents.length} parent(s), not two: what landed cannot be read back`; - } else if (method === "fast-forward-only") { - // That the forge names the head itself as a fast-forward's merge commit is assumed, not seen on this forge: - // a different sha is said, not raised as another head having landed. - if (merged.merge_commit_sha === pull.head_sha) landed = merged.merge_commit_sha; - else answer.read_back = `the forge named ${merged.merge_commit_sha.slice(0, 12)} as this fast-forward's commit, ` + - `not the head: that the head landed could not be confirmed`; - } else { - answer.read_back = `a ${method} makes new commits that name no head: what landed cannot be read back`; - } - } catch (err) { - answer.read_back = `the merge happened, but reading back what landed failed: ${(err as Error).message}`; - } - // Loud, not a field beside merged: true that a caller reading only that would miss. The merge cannot be taken - // back by this tool; what must not happen is that it reads as the merge that was asked for. - if (landed !== undefined && landed !== pull.head_sha) { - throw new Error(`${owner}/${repo}#${number} WAS merged (${answer.merge_commit_sha}), but the forge merged ` + - `${landed.slice(0, 12)}, not the ${pull.head_sha.slice(0, 12)} it was handed and you reviewed: a commit nobody ` + - `reviewed has landed. Read it before anything is built from it (novox/hq issue 381)`); - } - if (landed !== undefined) answer.landed = landed; - return answer; - }, - }; -} diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 7d803c79c..7d95544e4 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -64,16 +64,10 @@ ], "emits": [ "repo.created", - "repo.writable", "issue.opened", - "pull.merged", - "pull.updated", - "pull.closed", "forge.push", "forge.create", - "forge.delete", "forge.fork", - "forge.status", "forge.pull-request.opened", "forge.pull-request.reopened", "forge.pull-request.synchronized", @@ -173,7 +167,21 @@ }, { "name": "git", - "scope": "mesh" + "scope": "mesh", + "serves": [ + "pull", + "open-pulls", + "tags", + "trees", + "file", + "contains", + "statuses", + "status", + "comment", + "view", + "note", + "merge-pull" + ] } ], "data": { @@ -231,12 +239,6 @@ "trusted": true, "default": {} }, - "check-gates-merge": { - "type": "boolean", - "description": "Whether gitea_merge_pull_request judges mesh/check before it merges. True (the default): a head merges only on a passing mesh/check judged against the trunk's head (ADR 0447, ADR 0534 \u00a74). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"): a head the operator's Approve names in approved_heads merges whatever its mesh/check says (pending, failed, missing or judged on an older trunk); a head that moved, a branch no rule guards and a merge git refuses are still refused. Set together with mesh-delivery-gate's check-gates-merge.", - "trusted": true, - "default": true - }, "main-mergers": { "type": "list", "description": "The forge logins that may merge into main beside the tools' account, on every repository of the organisation (ADR 0534 \u00a72): the operator's own, so the operator can always merge by hand at the forge. The tools' account is always on main's merge whitelist; no agent's account belongs here.", @@ -249,8 +251,8 @@ "type": "string", "trusted": true, "runtime": true, - "default": "", - "description": "per verb of the two package seats, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); empty is design 51's list: publish and publish-part mesh-build-go and mesh-build-node, remove mesh-delivery-builder, versions anyone" + "default": "{\"publish\":[\"mesh-build-go\",\"mesh-build-node\"],\"publish-part\":[\"mesh-build-go\",\"mesh-build-node\"],\"remove\":[\"mesh-delivery-builder\"],\"versions\":[\"*\"],\"pull\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-gate\"],\"open-pulls\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"tags\":[\"mesh-delivery-forge-bridge\"],\"trees\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"file\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"contains\":[\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"statuses\":[\"mesh-delivery-gate\"],\"status\":[\"mesh-delivery-forge-bridge\"],\"comment\":[\"mesh-delivery-forge-bridge\"],\"view\":[\"mesh-delivery-forge-bridge\"],\"note\":[\"mesh-delivery-forge-bridge\"],\"merge-pull\":[\"mesh-delivery-forge-bridge\"]}", + "description": "per verb of the seats the module holds, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); each holder reads the lines of its own verbs. The package seats' lines are design 51's (empty is that list for them); the seat git's are design 47's: its reads the parts that declare them in invokes, its acts (status, comment, view, note, merge-pull) the forge bridge alone, and a verb of git with no line is called by no one" } }, "facts": { @@ -379,8 +381,7 @@ "takes-config": [ "url", "keep-main-rules", - "main-mergers", - "check-gates-merge" + "main-mergers" ] }, { @@ -629,7 +630,8 @@ "state": [ "bound", "announced", - "findings" + "findings", + { "name": "first-views", "history": 1 } ], "reads": [ "mesh-provisioner.plans", @@ -637,16 +639,14 @@ "ssh-client.keys", "mesh-build-go.keys", "mesh-build-node.keys", - "mesh-build-image.keys", - "mesh-delivery-checker.owed", - "mesh-delivery-gate.source" + "mesh-build-image.keys" ], "invokes": [ - "mesh-delivery-indexer.indexer_read_modules", + "seat:mesh-catalog.revision", "seat:mesh-hand-acts.hand-act", "mesh-delivery-builder.builder_versions" ], "consumes": [ - "mesh-delivery-indexer.views.modules" + "mesh-catalog.catalog-revision" ] } diff --git a/modules/gitea/package.json b/modules/gitea/package.json index 8e33075b9..e5bdab04d 100644 --- a/modules/gitea/package.json +++ b/modules/gitea/package.json @@ -9,7 +9,9 @@ "test": "npm run build && tsc --module NodeNext --moduleResolution NodeNext --target ES2022 --types node --rootDir . --outDir . $(find . -name '*.ts' ! -name '*.d.ts' ! -path './test/*' ! -path './node_modules/*' ! -path './dist/*') && node --test --experimental-strip-types 'test/*.test.ts'" }, "dependencies": { - "@novox/mesh-sdk": "^0.1.1" + "@novox/mesh-delivery-contracts-git": "^0.1.0", + "@novox/mesh-delivery-contracts-mesh-catalog": "^0.1.0", + "@novox/mesh-sdk": "^0.1.19" }, "devDependencies": { "@types/node": "^22.0.0", diff --git a/modules/gitea/protectfull.ts b/modules/gitea/protectfull.ts index e3c836b7a..ddb173163 100644 --- a/modules/gitea/protectfull.ts +++ b/modules/gitea/protectfull.ts @@ -206,40 +206,16 @@ function judgedTrunkRule(rules: BranchProtection[], branch: string): { holds: bo return { holds: true, why: `the rule ${r.rule} takes no push, only ${r.merge_whitelist.join(", ")} merge, and no administrator merges past it` }; } -/** The gate's source as it is read: one key per trunk, each with the heads of the repositories installed from it. */ -export type SourceRead = () => Promise>; +/** Where a repository's trunk is said to come from: the mesh's one trunk per repository (novox/hq design 47, *The trunk + * is what runs*), the branch main. No other module's state is read for it (ADR 0596 §2): the catalog's trunk heads name + * a repository and its commit, not a branch, so a second trunk would first be a branch in the seat mesh-catalog's + * revision. */ +export const TRUNK_FROM = "the mesh's one trunk of every repository (design 47)"; -/** The state the trunks are read from: mesh-delivery-gate's source, whose key `main` is today's one trunk of each - * repository it names (each repository's trunk head the checker judges on); a later trunk is a key of its own. */ -export const TRUNK_SOURCE = "mesh-delivery-gate.source"; - -/** A repository's trunks, the branches the mesh installs from (the operator, 2026-10-28 16:5x): each key of the gate's - * source whose heads name the repository. "main" only when the source names none of its trunks or cannot be read, and - * `from` says which. The one place trunks are read: several trunks later change only this. */ -export async function trunksOf(owner: string, repo: string, read: SourceRead): Promise<{ names: string[]; from: string }> { - const full = `${owner}/${repo}`; - let src: Record; - try { - src = await read(); - } catch (err) { - return { names: [MAIN_BRANCH], from: `${TRUNK_SOURCE} could not be read (${err instanceof Error ? err.message : String(err)}): ${MAIN_BRANCH} assumed` }; - } - const names = Object.keys(src).filter((k) => { - const heads = (src[k] as { heads?: Record } | null)?.heads; - return !!heads && typeof heads === "object" && full in heads; - }).sort(); - if (names.length) return { names, from: TRUNK_SOURCE }; - return { names: [MAIN_BRANCH], from: `${TRUNK_SOURCE} names no trunk of ${full}: ${MAIN_BRANCH} assumed` }; -} - -/** The gate's source read from the bus, every key and its value (the tools bundle's state channel). */ -export function sourceFromBus(get: (name: string) => { keys(): Promise; get(k: string): Promise<{ value: unknown } | null> }): SourceRead { - return async () => { - const st = get(TRUNK_SOURCE); - const out: Record = {}; - for (const k of await st.keys()) out[k] = (await st.get(k))?.value ?? null; - return out; - }; +/** A repository's trunks, the branches the mesh installs from (the operator, 2026-10-28 16:5x): main, and `from` says + * why. The one place trunks are named: a later trunk changes only this. */ +export function trunksOf(): { names: string[]; from: string } { + return { names: [MAIN_BRANCH], from: TRUNK_FROM }; } /** Why a call is the operator's, or null: it lets a force-push onto a rule covering a trunk (the operator, 2026-10-28 diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts new file mode 100644 index 000000000..4078216b8 --- /dev/null +++ b/modules/gitea/seat.ts @@ -0,0 +1,323 @@ +// The forge module holds the seat `git` (novox/hq design 47 *The seat git*, ADR 0596 §1, §2): every read and act a +// delivery part makes of the forge is a verb of the seat, served here as a tool `git.` against the seat's generated +// contract, and no part names this module's own tools. +// +// **Who may call which verb** is the holder's trusted config value `callers` (ADR 0627 §2), the one value the module's +// package seats read too: each holder takes the lines of the verbs it serves, so neither refuses the other's. Unset, no +// one may call a verb of `git`: the list is the manifest's, never a list in this code. The caller is the one the tool +// runner proved in the call's `_meta` (mesh-sdk `callers.callerOf`), never an argument. +// +// **A failed read answers an error, never an empty answer**: the forge's 404 is the kind `not-found`, its refusal (another +// 4xx, or a refusal of this holder) the kind `refused`, a forge that failed or did not answer the kind `failed`; an empty list or an absent file is answered +// only when the forge said so. +// +// **The acts carry the caller's id** (status, comment, view, note, merge-pull): an id the pull request or the note already +// holds is answered `held` and nothing is done again; `merge-pull` merges the head the caller names and judges no check +// (design 47 §314): only into a base whose protection refuses a push, and only that head. + +import type { ToolDefinition } from "@novox/mesh-sdk/tools"; +import { callerOf, refusal, parseVerbCallers, type VerbCallers } from "@novox/mesh-sdk/callers"; +import { config } from "@novox/mesh-sdk/runtime"; +import { + seat as GIT, verbs as GIT_VERBS, + type ContainsAnswer, type CommentAnswer, type ErrorKind, type FileAnswer, type MergePullAnswer, type Merging, type NoteAnswer, + type OpenPull, type OpenPullsAnswer, type Pull, type PullAnswer, type Status, type StatusAnswer, type StatusesAnswer, + type TagsAnswer, type TreesAnswer, type ViewAnswer, +} from "@novox/mesh-delivery-contracts-git"; +import type { CommitStatus, GiteaClient, GiteaComment, GiteaPull } from "./client.js"; +import { MAX_FILE_BYTES } from "./client.js"; +import { containedFrom, newestByContext } from "./checks.js"; +import { Refused, checkName } from "./guard.js"; +import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js"; + +/** The verbs of the seat, in its definition's order. */ +export const VERBS: readonly string[] = Object.values(GIT_VERBS); + +/** The config value whose lines per verb say who may call it. */ +export const CALLERS_KEY = "callers"; + +/** How many tags a page of `tags` holds. */ +export const TAGS_PAGE = 50; + +/** The forge's container, where its repositories and git are: the note is written there (novox/hq ADR 0239). Its name + * is the manifest's container's; no variable names it (a MESH_* word nothing sets is refused, checks/environment). */ +export const FORGE_CONTAINER = "gitea"; + +/** The writer every status of a head takes its turn on (gate.ts): the seat's status lands after every write of the head + * already asked for, so a recheck's pending never lands over a later verdict (novox/hq issue 456). */ +export interface HeadStatuses { + status(owner: string, repo: string, sha: string, status: CommitStatus): Promise; +} + +/** What the holder reaches beside the forge's API: git in the forge's container (the note), the head's one writer of its + * statuses, and the callers' list. */ +export interface SeatDeps { + run: Runner; + container: string; + statuses: HeadStatuses; + /** The config value `callers` as the runtime answers it; the runtime's own config read by default. */ + callers?: () => Promise; +} + +/** The lines of `raw` (the config value, an object or that object as a string) for the seat's verbs; unset is none. */ +export function gitCallers(raw: unknown): VerbCallers { + if (raw === undefined || raw === null || raw === "") return {}; + let all: unknown = raw; + if (typeof raw === "string") { + try { + all = JSON.parse(raw); + } catch (err) { + throw new Error(`callers is not a JSON object of verb to callers: ${err instanceof Error ? err.message : String(err)}`); + } + } + if (!all || typeof all !== "object" || Array.isArray(all)) throw new Error("callers is not a JSON object of verb to callers"); + const mine = Object.fromEntries(Object.entries(all as Record).filter(([verb]) => VERBS.includes(verb))); + return parseVerbCallers(JSON.stringify(mine), VERBS, {}); +} + +/** A failure as the contract says it: the forge's or this holder's refusal, or a forge that failed. */ +export function failure(err: unknown): { error: string; "error-kind": ErrorKind } { + const text = err instanceof Error ? err.message : String(err); + // The forge's 404: what was asked about (a repository, a commit, a pull request, a branch, a tag) does not exist. + if (/^Gitea API [^:]*: 404 /.test(text)) return { error: text, "error-kind": "not-found" }; + const refused = err instanceof Refused || /^Gitea API [^:]*: 4\d\d /.test(text) || /^Gitea API: /.test(text); + return { error: text, "error-kind": refused ? "refused" : "failed" }; +} + +/** owner/name of a `repository` argument, each checked as a name (guard.ts). */ +export function repositoryOf(raw: unknown): { owner: string; repo: string; full: string } { + const parts = String(raw ?? "").split("/"); + if (parts.length !== 2) throw new Refused(`repository ${JSON.stringify(raw)} is not /`); + const owner = checkName("owner", parts[0]), repo = checkName("name", parts[1]); + return { owner, repo, full: `${owner}/${repo}` }; +} + +const SHA = /^[0-9a-f]{40}$/; +const STATES = new Set(["pending", "success", "failure", "error", "warning"]); +function commitOf(name: string, raw: unknown): string { + const sha = String(raw ?? "").trim(); + if (!SHA.test(sha)) throw new Refused(`${name} is a commit's full 40-hex sha`); + return sha; +} +function numberOf(raw: unknown): number { + const n = Number(raw); + if (!Number.isInteger(n) || n < 1) throw new Refused(`${JSON.stringify(raw)} is not a pull request's number`); + return n; +} +function idOf(raw: unknown): string { + const id = String(raw ?? "").trim(); + if (!id || /[\s<>]/.test(id)) throw new Refused("id is the act's id: one word"); + return id; +} + +/** The marker an act's id leaves in a comment, so an id said again is found and skipped. */ +export function actMarker(id: string): string { + return ``; +} + +/** A pull request as the contract says it; never its description (ADR 0659 §1). */ +export function pullOf(full: string, p: GiteaPull): Pull { + const state = p.merged ? "merged" : p.state === "open" ? "open" : "closed"; + return { + repository: full, number: p.number, base: String(p.base ?? ""), branch: String(p.head ?? ""), head: String(p.head_sha ?? ""), + state, ...(p.user ? { author: p.user } : {}), ...(p.merge_commit_sha ? { "merge-commit": p.merge_commit_sha } : {}), + ...(p.merged_at ? { "merged-at": p.merged_at } : {}), ...(p.title ? { title: p.title } : {}), + ...(p.html_url ? { url: p.html_url } : {}), + }; +} + +function statusOf(s: { context: string; state: string; description?: string; target_url?: string; updated_at?: string; creator?: string }): Status { + return { context: s.context, state: s.state, ...(s.description ? { description: s.description } : {}), + ...(s.target_url ? { target: s.target_url } : {}), ...(s.updated_at ? { at: s.updated_at } : {}), + ...(s.creator ? { by: s.creator } : {}) }; +} + +/** What the forge would merge now, for an open pull request: whole, or the verb's error. */ +async function mergingOf(gitea: GiteaClient, owner: string, repo: string, p: GiteaPull): Promise { + if (p.mergeable === undefined) throw new Error(`the forge did not say whether ${owner}/${repo}#${p.number} merges now`); + const rule = await gitea.appliedProtection(owner, repo, String(p.base ?? "")); + const writes = (await gitea.getRepo(owner, repo)).writes; + if (writes === undefined) throw new Error(`the forge did not say whether its account writes ${owner}/${repo}`); + const required = rule?.enable_status_check ? [...(rule.status_check_contexts ?? [])] : []; + return { mergeable: p.mergeable, protected: !!rule, required, writes }; +} + +/** The seat's verbs as tools `git.`, each judging its caller first and answering in the seat's contract. */ +export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] { + const listOf = deps.callers ?? (() => config(CALLERS_KEY)); + const verb = (name: string, description: string, act: (args: Record) => Promise): ToolDefinition => ({ + name: `${GIT}.${name}`, + description, + input: {}, + run: async (args: Record) => { + try { + const why = refusal(gitCallers(await listOf()), name, callerOf(args)); + if (why) return failure(new Refused(why.message)); + return await act(args); + } catch (err) { + return failure(err); + } + }, + }); + + return [ + verb(GIT_VERBS.pull, "The seat git's pull: one pull request by its repository and number or by its merge commit, and for an open one what the forge would merge now.", + async (args): Promise => { + const { owner, repo, full } = repositoryOf(args.repository); + const p = args["merge-commit"] !== undefined + ? await gitea.pullOfCommit(owner, repo, commitOf("merge-commit", args["merge-commit"])) + : await gitea.getPullRequest(owner, repo, numberOf(args.number)); + const pull = pullOf(full, p); + return pull.state === "open" ? { pull, merging: await mergingOf(gitea, owner, repo, p) } : { pull }; + }), + verb(GIT_VERBS.openPulls, "The seat git's open-pulls: every open pull request of every repository, with the statuses its base requires and those its head has.", + async (): Promise => { + const pulls: OpenPull[] = (await gitea.openPulls()).map((o) => ({ + pull: { repository: `${o.owner}/${o.repo}`, number: o.number, base: o.base, branch: String(o.head ?? ""), head: o.head_sha, + state: "open", ...(o.html_url ? { url: o.html_url } : {}) }, + required: [...o.required], + checks: Object.fromEntries(Object.entries(o.checks ?? {}).map(([context, c]) => + [context, statusOf({ context, state: c.state, description: c.description, updated_at: c.updated_at })])), + })); + return { pulls }; + }), + verb(GIT_VERBS.tags, "The seat git's tags: a repository's tags, a page at a time, newest first; more when another page exists.", + async (args): Promise => { + const { owner, repo, full } = repositoryOf(args.repository); + const page = args.page === undefined ? 1 : Number(args.page); + const prefix = String(args.prefix ?? ""); + // One more than the page is asked, so whether another page exists is the forge's answer, not a guess. + const got = await gitea.tags(owner, repo, page, TAGS_PAGE + 1); + const more = got.length > TAGS_PAGE; + const tags = got.slice(0, TAGS_PAGE).filter((t) => t.startsWith(prefix)); + return { repository: full, page, bound: TAGS_PAGE, tags, ...(more ? { more } : {}) }; + }), + verb(GIT_VERBS.trees, "The seat git's trees: the trees of one commit, with files and paths when asked; a listing the forge cut is refused.", + async (args): Promise => { + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit); + const got = await gitea.commitTrees(owner, repo, commit, args.paths as Record | undefined, + args.files as string[] | undefined); + return { commit, ...got }; + }), + verb(GIT_VERBS.file, "The seat git's file: one file at one commit, branch or tag, decoded; over 512 KiB refused; absent only when the forge said so.", + async (args): Promise => { + const { owner, repo } = repositoryOf(args.repository); + const path = String(args.path ?? ""); + const ref = args.ref === undefined ? undefined : String(args.ref); + try { + const f = await gitea.getFile(owner, repo, path, ref); + if (f.size > MAX_FILE_BYTES) throw new Refused(`${path} is ${f.size} bytes, over ${MAX_FILE_BYTES / 1024} KiB`); + return { path: f.path, ...(ref ? { ref } : {}), blob: f.sha, size: f.size, content: f.content }; + } catch (err) { + if (err instanceof Error && /^Gitea API [^:]*\/contents\/[^:]*: 404 /.test(err.message)) return { path, ...(ref ? { ref } : {}), absent: true }; + throw err; + } + }), + verb(GIT_VERBS.contains, "The seat git's contains: whether one commit contains another, an ancestor of it or the same.", + async (args): Promise => { + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit), within = commitOf("in", args.in); + const { contains, ahead } = containedFrom(await gitea.compare(owner, repo, within, commit)); + return { commit, in: within, contains, ahead }; + }), + verb(GIT_VERBS.statuses, "The seat git's statuses: the statuses a commit carries, the newest of each context.", + async (args): Promise => { + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit); + const combined = await gitea.combinedStatus(owner, repo, commit); + return { commit, combined: combined.state, statuses: newestByContext(combined.statuses).map(statusOf) }; + }), + verb(GIT_VERBS.status, "The seat git's status: an act of the forge bridge, a commit status set in its turn after every write of the head already asked for; the same id said again sets it again to the same.", + async (args): Promise => { + const id = idOf(args.id); + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit); + const state = String(args.state ?? ""); + if (!STATES.has(state)) throw new Refused(`state ${JSON.stringify(state)} is none of ${[...STATES].join(", ")}`); + await deps.statuses.status(owner, repo, commit, { state: state as CommitStatus["state"], context: String(args.context ?? ""), + description: String(args.description ?? ""), ...(args.target ? { target_url: String(args.target) } : {}) }); + return { id, performed: true }; + }), + verb(GIT_VERBS.comment, "The seat git's comment: an act of the forge bridge, a comment on a pull request carrying the act's id; an id it already holds is held.", + async (args): Promise => { + const id = idOf(args.id); + const { owner, repo } = repositoryOf(args.repository); + const number = numberOf(args.number); + if (holds(await gitea.listComments(owner, repo, number), id)) return { id, held: true, performed: true }; + await gitea.addComment(owner, repo, number, `${String(args.text ?? "")}\n\n${actMarker(id)}`); + return { id, performed: true }; + }), + verb(GIT_VERBS.view, "The seat git's view: an act of the forge bridge, the delivery's one view comment kept on a pull request; an id it already holds is held.", + async (args): Promise => { + const id = idOf(args.id); + const { owner, repo } = repositoryOf(args.repository); + const number = numberOf(args.number); + const existing = viewComment(await gitea.listComments(owner, repo, number)); + if (existing && existing.body.includes(actMarker(id))) return { id, held: true, performed: true }; + const body = `${viewBody(String(args.text ?? ""))}\n${actMarker(id)}`; + if (existing) await gitea.editComment(owner, repo, existing.id, body); + else await gitea.addComment(owner, repo, number, body); + return { id, performed: true }; + }), + verb(GIT_VERBS.note, "The seat git's note: an act of the forge bridge, one line on a commit's note under refs/notes/mesh-plan carrying the act's id; an id it already holds is held.", + async (args): Promise => { + const id = idOf(args.id); + const { owner, repo } = repositoryOf(args.repository); + const commit = commitOf("commit", args.commit); + const said = await appendNote(deps.run, deps.container, owner, repo, commit, "mesh-plan", `${String(args.line ?? "")} [${id}]`, id); + return said.added ? { id, performed: true } : { id, held: true, performed: true }; + }), + verb(GIT_VERBS.mergePull, "The seat git's merge-pull: an act of the forge bridge, a pull request merged at the head the caller names; it judges no check.", + async (args): Promise => mergePull(gitea, args)), + ]; +} + +function holds(comments: GiteaComment[], id: string): boolean { + return comments.some((c) => String(c.body ?? "").includes(actMarker(id))); +} + +/** merge-pull (design 47 §314): only the head the caller names, only into a base that takes no push, no check judged. */ +export async function mergePull(gitea: GiteaClient, args: Record): Promise { + const id = idOf(args.id); + const { owner, repo, full } = repositoryOf(args.repository); + const number = numberOf(args.number); + const head = commitOf("head", args.head); + const no = (why: string): MergePullAnswer => ({ id, refused: `${full}#${number} is not merged: ${why}` }); + const pull = await gitea.getPullRequest(owner, repo, number); + if (pull.merged) { + // Already merged at that head: performed (the act said again after its answer was lost); at another, refused. + return pull.head_sha === head ? { id, performed: true, held: true, ...(pull.merge_commit_sha ? { "merge-commit": pull.merge_commit_sha } : {}) } + : no(`it was merged at ${String(pull.head_sha).slice(0, 12)}, not the ${head.slice(0, 12)} the caller names`); + } + if (pull.state !== "open") return no("it is closed"); + if (pull.head_sha !== head) { + return no(`its head is ${String(pull.head_sha).slice(0, 12)}, not the ${head.slice(0, 12)} the caller names`); + } + const rule = await gitea.appliedProtection(owner, repo, String(pull.base ?? "")); + if (!rule || rule.enable_push !== false) { + return no(`its base ${pull.base} is guarded by no protection rule that refuses a push`); + } + try { + await gitea.mergePullRequest(owner, repo, number, "merge", true, head); + } catch (err) { + const f = failure(err); + if (f["error-kind"] === "refused") return no(`the forge refused the merge: ${f.error}`); + throw err; + } + const merged = await gitea.getPullRequest(owner, repo, number); + if (!merged.merge_commit_sha) throw new Error(`${full}#${number} WAS merged, but the forge named no merge commit`); + // The merge is done once the forge named its commit: a read-back that fails is said, never answered as a failed act. + let parents: string[]; + try { + parents = await gitea.commitParents(owner, repo, merged.merge_commit_sha); + } catch (err) { + console.error(`[gitea] ${full}#${number} was merged (${merged.merge_commit_sha}); which head it joined could not be read back — ${err instanceof Error ? err.message : err}`); + return { id, performed: true, "merge-commit": merged.merge_commit_sha }; + } + if (parents.length === 2 && parents[1] !== head) { + throw new Error(`${full}#${number} WAS merged (${merged.merge_commit_sha}), but the forge merged ${parents[1].slice(0, 12)}, ` + + `not the ${head.slice(0, 12)} the caller names (novox/hq issue 381)`); + } + return { id, performed: true, "merge-commit": merged.merge_commit_sha }; +} diff --git a/modules/gitea/seatevents.ts b/modules/gitea/seatevents.ts new file mode 100644 index 000000000..8b859f846 --- /dev/null +++ b/modules/gitea/seatevents.ts @@ -0,0 +1,96 @@ +// The seat git's events, said by its holder (novox/hq design 47 *The seat git*): a pull request's new head, its merge and +// its close (pull.updated, pull.merged, pull.closed), the forge's own webhook of a pull request, a status and a deleted +// branch (forge.pull, forge.status, forge.delete), and a repository the forge can write again (repo.writable). Each is +// said on the seat's subject through the runtime (mesh-sdk `runtime.seatPublish`, `mesh/seat.publish`), in the body the +// seat's contract types, never as this module's own event: its hearers consume the seat, not the forge module. +// +// What the module's paths make (the poll, the webhook, the account's writes) is read here once and turned into the +// contract's body; a pull request's description is never carried (ADR 0659 §1). + +import { seatPublish } from "@novox/mesh-sdk/runtime"; +import { + forgeDeleteSubject, forgePullSubject, forgeStatusSubject, pullClosedSubject, pullMergedSubject, pullUpdatedSubject, + repoWritableSubject, + type Change, type ForgeDeleteEvent, type ForgePullEvent, type ForgeStatusEvent, type PullClosedEvent, type PullMergedEvent, + type PullUpdatedEvent, type RepoWritableEvent, +} from "@novox/mesh-delivery-contracts-git"; + +/** The forge's pull request actions the seat's forge.pull says (the contract's word list). */ +export const PULL_ACTIONS = ["opened", "reopened", "synchronized", "edited", "closed", "merged"]; + +type Body = Record; +const s = (v: unknown): string | undefined => (typeof v === "string" && v !== "" ? v : undefined); +const opt = (k: K, v: unknown) => (v === undefined || v === null || v === "" ? {} : { [k]: v } as Record); + +function repositoryOf(b: Body): string { + const full = s(b.full_name) ?? (s(b.owner) && s(b.repo) ? `${b.owner}/${b.repo}` : undefined); + if (!full) throw new Error("the event names no repository"); + return full; +} + +function changeOf(b: Body): Change | undefined { + if (!Array.isArray(b.paths)) return undefined; + return { paths: b.paths.map(String), ...opt("paths-truncated", b.paths_truncated === true ? true : undefined), + ...opt("removed", Array.isArray(b.removed) && b.removed.length ? b.removed.map(String) : undefined), + ...opt("module-dirs", Array.isArray(b.module_dirs) ? b.module_dirs.map(String) : undefined) }; +} + +/** The seat's subject and body for one of the module's announcements, or undefined when the seat says no such event. */ +export function seatEventOf(type: string, b: Body): { subject: string; body: unknown } | undefined { + switch (type) { + case "pull.updated": { + const body: PullUpdatedEvent = { repository: repositoryOf(b), number: Number(b.number), base: String(b.base ?? ""), + branch: String(b.head ?? ""), head: String(b.head_sha ?? ""), ...opt("base-head", s(b.base_sha)), + ...opt("author", s(b.author)), ...opt("title", s(b.title)), ...opt("url", s(b.html_url)), ...opt("change", changeOf(b)) }; + return { subject: pullUpdatedSubject, body }; + } + case "pull.merged": { + const body: PullMergedEvent = { repository: repositoryOf(b), number: Number(b.number), base: String(b.base ?? ""), + branch: String(b.head ?? ""), head: String(b.head_sha ?? ""), "merge-commit": String(b.merge_commit_sha ?? ""), + ...opt("merged-at", s(b.merged_at)), ...opt("head-checks", b.head_checks), ...opt("author", s(b.author)), + ...opt("title", s(b.title)), ...opt("url", s(b.html_url)), ...opt("change", changeOf(b)) }; + return { subject: pullMergedSubject, body }; + } + case "pull.closed": { + const body: PullClosedEvent = { repository: repositoryOf(b), number: Number(b.number), base: String(b.base ?? ""), + branch: String(b.head ?? ""), head: String(b.head_sha ?? ""), ...opt("author", s(b.author)), ...opt("title", s(b.title)), + ...opt("url", s(b.html_url)) }; + return { subject: pullClosedSubject, body }; + } + case "forge.status": { + const body: ForgeStatusEvent = { repository: repositoryOf(b), commit: String(b.sha ?? ""), context: String(b.context ?? ""), + state: String(b.state ?? ""), ...opt("description", s(b.description)), ...opt("sender", s(b.sender)) }; + return { subject: forgeStatusSubject, body }; + } + case "forge.delete": { + const body: ForgeDeleteEvent = { repository: repositoryOf(b), ref: String(b.ref ?? ""), ...opt("ref-type", s(b.ref_type)), + ...opt("sender", s(b.sender)) }; + return { subject: forgeDeleteSubject, body }; + } + case "repo.writable": { + const body: RepoWritableEvent = { repository: repositoryOf(b), account: String(b.account ?? "") }; + return { subject: repoWritableSubject, body }; + } + } + const action = type.startsWith("forge.pull-request.") ? type.slice("forge.pull-request.".length) : ""; + if (PULL_ACTIONS.includes(action)) { + const body: ForgePullEvent = { repository: repositoryOf(b), number: Number(b.number), action, + ...opt("base", s(b.base)), ...opt("branch", s(b.head)), ...opt("head", s(b.head_sha)), ...opt("author", s(b.author)), + ...opt("merge-commit", s(b.merge_commit_sha)), ...(b.merged === true ? { merged: true } : {}), ...opt("sender", s(b.sender)) }; + return { subject: forgePullSubject, body }; + } + return undefined; +} + +/** The module's own events the seat says instead: never said as the module's (its `emits` names none of them). */ +export const SEAT_ONLY = new Set(["pull.updated", "pull.merged", "pull.closed", "forge.status", "forge.delete", "repo.writable"]); + +export type Say = (subject: string, body: unknown, id?: string) => Promise; + +/** Say an announcement on the seat, de-duplicated by `id` (the same announcement by two paths is one message). */ +export async function sayOnSeat(type: string, body: Body, id = "", say: Say = seatPublish): Promise { + const e = seatEventOf(type, body); + if (!e) return false; + await say(e.subject, e.body, id); + return true; +} diff --git a/modules/gitea/test/announce.test.ts b/modules/gitea/test/announce.test.ts index 222fee212..7a3c95eb6 100644 --- a/modules/gitea/test/announce.test.ts +++ b/modules/gitea/test/announce.test.ts @@ -22,25 +22,37 @@ test("the same merge, head or close announced twice has the same id, and another assert.equal(announcementId("pull.closed", closed), "closed:novox/app#7@aaa111"); }); -test("an announcement is published through the broker with its id as the event id, and the provenance emit stamps", async () => { - const { useBroker } = await import("@novox/mesh-sdk/messaging"); - const published: any[] = []; - useBroker(() => ({ publish: async (env: any) => { published.push(env); } }) as any); - process.env.MESH_MODULE = "gitea"; - process.env.MESH_NODE = "anchor"; +test("an announcement is said on the seat git with its id, in the contract's body, never the description", async () => { + const { useRuntime } = await import("@novox/mesh-sdk/runtime"); + const said: any[] = []; + useRuntime(async (method, params) => { said.push({ method, ...(params as object) }); return null; }); const { announce } = await import("../announce.ts"); - await announce("pull.merged", merged); + const { seatEventOf, SEAT_ONLY } = await import("../seatevents.ts"); + await announce("pull.merged", { ...merged, body: "a description", head: "feat/x", paths: ["a/b.go"], removed: [] }); await announce("pull.merged", { ...merged }); - assert.equal(published.length, 2); - assert.equal(published[0].key, "pull.merged"); - assert.equal(published[0].headers["x-event-id"], "merged:novox/app#7@mmm222"); - assert.equal(published[1].headers["x-event-id"], published[0].headers["x-event-id"], "one id for one announcement"); - assert.equal(published[0].headers["x-source"], "gitea"); - assert.equal(published[0].headers["x-node"], "anchor"); - assert.equal(published[0].headers["content-type"], "application/json"); - assert.ok(published[0].headers["x-time"]); - assert.equal(published[0].node, "anchor"); - assert.deepEqual(published[0].body, merged); + useRuntime(undefined); + assert.equal(said.length, 2); + assert.deepEqual([said[0].method, said[0].subject, said[0].id], ["mesh/seat.publish", "mesh.seat.git.event.pull-merged", "merged:novox/app#7@mmm222"]); + assert.equal(said[1].id, said[0].id, "one id for one announcement"); + assert.deepEqual(said[0].body, { repository: "novox/app", number: 7, base: "main", branch: "feat/x", head: "aaa111", + "merge-commit": "mmm222", change: { paths: ["a/b.go"] } }); + // Every event the seat says, from what the module's paths make; the others are none of the seat's. + const rows: [string, Record, string | undefined][] = [ + ["pull.updated", updated, "mesh.seat.git.event.pull-updated"], + ["pull.closed", closed, "mesh.seat.git.event.pull-closed"], + ["forge.status", { full_name: "novox/app", sha: "aaa111", context: "mesh/check", state: "success" }, "mesh.seat.git.event.forge-status"], + ["forge.delete", { owner: "novox", repo: "app", ref: "feat/x", ref_type: "branch" }, "mesh.seat.git.event.forge-delete"], + ["repo.writable", { owner: "novox", repo: "app", account: "mesh-tools" }, "mesh.seat.git.event.repo-writable"], + ["forge.pull-request.synchronized", { full_name: "novox/app", number: 7, head_sha: "aaa111" }, "mesh.seat.git.event.forge-pull"], + ["forge.pull-request.labeled", { full_name: "novox/app", number: 7 }, undefined], + ["forge.push", { full_name: "novox/app" }, undefined], + ]; + for (const [type, body, subject] of rows) assert.equal(seatEventOf(type, body)?.subject, subject, type); + assert.deepEqual(seatEventOf("pull.updated", updated)?.body, { repository: "novox/app", number: 7, base: "main", branch: "", + head: "aaa111", "base-head": "ttt333" }); + assert.equal(seatEventOf("forge.pull-request.edited", { full_name: "novox/app", number: 7, body: "x" })?.body && + "body" in (seatEventOf("forge.pull-request.edited", { full_name: "novox/app", number: 7, body: "x" })!.body as object), false); + assert.deepEqual([...SEAT_ONLY].sort(), ["forge.delete", "forge.status", "pull.closed", "pull.merged", "pull.updated", "repo.writable"]); }); test("an announcement of a kind it has no id for is refused, never published with a made-up one", async () => { diff --git a/modules/gitea/test/bundle-import.test.ts b/modules/gitea/test/bundle-import.test.ts index 14f7786cb..c14f47726 100644 --- a/modules/gitea/test/bundle-import.test.ts +++ b/modules/gitea/test/bundle-import.test.ts @@ -37,85 +37,12 @@ test("importing the bundle resolves while the broker keeps refusing the binds", assert.match(out, /imported/, `the import did not resolve within 3 s while a bind was refused; stderr: ${err.slice(0, 600)}`); }); -// **The bundle writes mesh/check from the checker's owed state, and serves no tool that writes it** (ADR 0521; R4e round 2 -// finding R2-1; finding K1). The bundle is imported whole, as the runtime imports it, while its read of the checker's owed -// is refused (the grant not yet composed): the watch is asked again, and once granted it is handed the bucket as the bus -// keeps it, the latest revision of each key, then an older revision delivered late. The forge stand-in sees the latest -// written, and nothing after it. -test("a bundle whose read of the checker's owed is granted late writes the latest mesh/check, and serves no gitea_check_status", async () => { - const index = pathToFileURL(join(here, "..", "dist", "index.js")).href; - const sha = "0123456789abcdef0123456789abcdef01234567"; - const script = ` - const http = await import("node:http"); - const statuses = []; - const forge = http.createServer((req, res) => { - let b = ""; req.on("data", (d) => (b += d)); req.on("end", () => { - if (req.method === "POST" && /\\/statuses\\//.test(req.url)) statuses.push(req.url + " " + b); - res.setHeader("content-type", "application/json"); res.end(req.method === "GET" ? "[]" : "{}"); - }); - }); - await new Promise((r) => forge.listen(0, "127.0.0.1", r)); - process.env.MESH_GITEA_URL = "http://127.0.0.1:" + forge.address().port; - const bound = []; - const { useBroker } = await import("@novox/mesh-sdk/messaging"); - useBroker(() => ({ - subscribe: async (pattern) => { bound.push(pattern); return () => {}; }, - publish: async () => {}, request: async () => { throw new Error("no responders"); }, - handle: async () => () => {}, close: async () => {}, - })); - const key = "novox/mesh-sdk.12.${sha}.status=mesh/check"; - const owed = (state, text) => ({ v: 1, act: "status", context: "mesh/check", state, text, target: "https://view.example/c" }); - const watched = []; - let granted = 0; - const { useStateChannel } = await import("@novox/mesh-sdk/state"); - useStateChannel({ - ask: async () => null, - watch: async (params, deliver) => { - watched.push(params.state); - if (watched.length === 1) throw new Error('gitea keeps and reads no state called "mesh-delivery-checker.owed"'); - granted++; - // Two revisions were written while the read was refused; the bucket hands the latest, then a late older one. - await deliver({ state: params.state, key, op: "put", value: owed("success", "pass at 4"), revision: 4, current: true }); - await deliver({ state: params.state, key, op: "put", value: owed("pending", "pending at 3"), revision: 3, current: false }); - }, - }); - await import(${JSON.stringify(index)}); - const { collectTools } = await import("@novox/mesh-sdk/tools"); - const tools = collectTools().flatMap((r) => r.tools.map((t) => t.name)); - for (let i = 0; i < 300 && !(granted && statuses.length); i++) await new Promise((r) => setTimeout(r, 20)); - await new Promise((r) => setTimeout(r, 200)); - console.log(JSON.stringify({ tools, bound, watched, statuses })); - process.exit(0);`; - const env = { ...process.env, GITEA_TOKEN: "x" }; - for (const k of ["MESH_GITEA_STATE_DIR", "MESH_GITEA_ADMIN_USER", "MESH_GITEA_ADMIN_PASSWORD", "MESH_SERVED_MODULE", "MESH_GITEA_URL"]) delete env[k]; - const child = spawn(process.execPath, ["--input-type=module", "-e", script], { cwd: join(here, ".."), env, stdio: ["ignore", "pipe", "pipe"] }); - let out = ""; - let err = ""; - child.stdout.on("data", (d) => (out += d)); - child.stderr.on("data", (d) => (err += d)); - const ended = new Promise((r) => child.on("exit", () => r())); - const timer = setTimeout(() => child.kill("SIGKILL"), 15000); - await ended; - clearTimeout(timer); - const line = out.split("\n").find((l) => l.startsWith("{")); - assert.ok(line, `the bundle said nothing; stdout: ${out.slice(0, 400)} stderr: ${err.slice(0, 600)}`); - const got = JSON.parse(line!) as { tools: string[]; bound: string[]; watched: string[]; statuses: string[] }; - assert.ok(got.tools.includes("gitea_commit_status"), `the bundle's tools: ${got.tools}`); - assert.ok(!got.tools.includes("gitea_check_status"), "no tool writes mesh/check"); - assert.ok(!got.bound.some((p) => p.includes("check.status")), `bound: ${got.bound}`); - assert.deepEqual(got.watched, ["mesh-delivery-checker.owed", "mesh-delivery-checker.owed"], "refused, then asked again"); - assert.equal(got.statuses.length, 1, `statuses written: ${got.statuses}`); - assert.match(got.statuses[0], new RegExp(`/repos/novox/mesh-sdk/statuses/${sha} .*"context":"mesh/check"`)); - assert.match(got.statuses[0], /"state":"success".*pass at 4/); - assert.match(err, /mesh-delivery-checker\.owed could not be bound/, "the refusal is said"); -}); - // **The bundle's import does not wait on the forge** (novox/hq issue 1726, 2026-10-24 12:37). The webhook receiver's set-up // asks the forge's admin API for its system webhook; awaited at the top of index.ts, a forge slow to answer (busy while // every bundle of the node starts) held the import past node-tools' 10 s handshake, the bundle was killed and never -// started again, and gitea_commit_status, gitea_merge_pull_request and gitea_events were served nowhere: the forge -// bridge's calls found no responders and no new pull request was announced. Here the forge stand-in takes every -// request and never answers: the import resolves at once all the same, with the bundle's tools registered. +// started again, and gitea_events was served nowhere and no new pull request was announced. Here the forge stand-in takes +// every request and never answers: the import resolves at once all the same, with the bundle's tools registered. The +// bundle serves no merge and no status of its own: those are the seat git's acts (design 47 *The seat git*). test("importing the bundle resolves, its tools registered, while the forge never answers the webhook's set-up", async () => { const index = pathToFileURL(join(here, "..", "dist", "index.js")).href; const script = ` @@ -155,7 +82,8 @@ test("importing the bundle resolves, its tools registered, while the forge never const line = out.split("\n").find((l) => l.startsWith("{")); assert.ok(line, `the import did not resolve within 3 s while the forge did not answer; stdout: ${out.slice(0, 400)} stderr: ${err.slice(0, 600)}`); const got = JSON.parse(line!) as { tools: string[] }; - for (const t of ["gitea_events", "gitea_commit_status", "gitea_merge_pull_request"]) { - assert.ok(got.tools.includes(t), `${t} is registered: ${got.tools}`); + assert.ok(got.tools.includes("gitea_events"), `gitea_events is registered: ${got.tools}`); + for (const t of ["gitea_commit_status", "gitea_merge_pull_request"]) { + assert.ok(!got.tools.includes(t), `${t} is retired: ${got.tools}`); } }); diff --git a/modules/gitea/test/check-owed.test.ts b/modules/gitea/test/check-owed.test.ts deleted file mode 100644 index d3184a145..000000000 --- a/modules/gitea/test/check-owed.test.ts +++ /dev/null @@ -1,207 +0,0 @@ -// **mesh/check is written from the checker's owed state, read by a watch** (ADR 0521; R4e round 2 finding R2-1; finding K1 -// before it). No tool writes mesh/check, since a tool is told no caller and node-tools invokes every tool. Nor does an event: -// nothing in the delivery plan orders this module before the forge bridge, and a durable consumer whose filters change keeps -// its deliver policy, so an event said before this module consumed it was lost. A watch of the checker's state hands over -// the latest value of every key when it starts, whenever that is. - -import { test } from "node:test"; -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; - -const sha = "0123456789abcdef0123456789abcdef01234567"; -const key = (s = sha, repo = "novox/mesh-sdk", n = 12) => `${repo}.${n}.${s}.status=mesh/check`; -const owed = (over: Record = {}) => ({ v: 1, act: "status", context: "mesh/check", state: "success", - text: "pass 0123456789ab on aaaaaaaaaaaa", target: "https://view.example/checks/g.1", ...over }); -const put = (k: string, value: unknown, revision: number, current = false) => - ({ state: "mesh-delivery-checker.owed", key: k, op: "put" as const, value, revision, current }); - -function forge() { - const sent: string[] = []; - let failing = 0; - return { - sent, - fail: (n: number) => { failing = n; }, - setCommitStatus: async (o: string, r: string, s: string, st: any) => { - if (failing > 0) { failing--; throw new Error("Gitea API /repos/x/statuses: 502 bad gateway"); } - sent.push(`${o}/${r}@${s.slice(0, 8)} ${st.context} ${st.state} ${st.description}${st.target_url ? " " + st.target_url : ""}`); - }, - }; -} -const noSleep = async () => {}; - -test("the checker's owed is what this module reads, and it consumes no check.status", async () => { - const { CHECKER_OWED } = await import("../check.ts"); - assert.equal(CHECKER_OWED, "mesh-delivery-checker.owed"); - const manifest = JSON.parse(readFileSync(new URL("../module.json", import.meta.url), "utf8")); - assert.ok(manifest.reads.includes(CHECKER_OWED), "gitea's manifest reads the checker's owed"); - assert.ok(!(manifest.consumes ?? []).some((c: string) => c.includes("check.status")), "no check.status is consumed"); -}); - -test("an owed mesh/check writes the status on its head: its state, its line cut and its link", async () => { - const { CheckStatuses } = await import("../check.ts"); - const f = forge(); - const checks = new CheckStatuses(f, () => {}, noSleep); - assert.equal(await checks.apply(put(key(), owed(), 7, true)), "written"); - const other = sha.replace(/0/g, "f"); - assert.equal(await checks.apply(put(key(other, "novox/my=2erepo"), owed({ state: "pending", text: "y".repeat(300), target: "javascript:x" }), 8)), "written"); - assert.equal(f.sent[0], "novox/mesh-sdk@01234567 mesh/check success pass 0123456789ab on aaaaaaaaaaaa https://view.example/checks/g.1"); - assert.ok(f.sent[1].startsWith("novox/my.repo@f1234567 mesh/check pending yyyy"), f.sent[1]); - assert.ok(f.sent[1].length < 200, "the line is cut as every status's is"); - assert.ok(!f.sent[1].includes("javascript"), "a link that is no web page is not written"); -}); - -test("a key or value that is no owed mesh/check, and a delete, write nothing", async () => { - const { CheckStatuses } = await import("../check.ts"); - const f = forge(); - const checks = new CheckStatuses(f, () => {}, noSleep); - const cases = [ - put(key(sha.slice(0, 12)), owed(), 1), put(`novox/mesh-sdk.12.${sha}.status=mesh/merge-gate`, owed(), 1), - put(`novox/mesh-sdk.12.${sha}.view`, owed(), 1), put(`nokey`, owed(), 1), put(`../x/y.12.${sha}.status=mesh/check`, owed(), 1), - put(key(), owed({ context: "mesh/delivery" }), 1), put(key(), owed({ act: "view" }), 1), put(key(), owed({ state: "green" }), 1), - put(key(), owed({ v: 2 }), 1), put(key(), null, 1), put(key(), "text", 1), put(key(), owed(), -1), - { state: "mesh-delivery-checker.owed", key: key(), op: "delete" as const, revision: 2, current: false }, - ]; - for (const c of cases) assert.equal(await checks.apply(c as any), "dropped", JSON.stringify(c)); - assert.deepEqual(f.sent, []); -}); - -test("a revision older than one written on the key is dropped; the same one is written again", async () => { - const { CheckStatuses } = await import("../check.ts"); - const f = forge(); - const checks = new CheckStatuses(f, () => {}, noSleep); - await checks.apply(put(key(), owed({ state: "success" }), 10)); - assert.equal(await checks.apply(put(key(), owed({ state: "pending" }), 9)), "dropped", "a stale pending never hides the verdict"); - assert.equal(await checks.apply(put(key(), owed({ state: "success" }), 10)), "written"); - assert.equal(await checks.apply(put(key(sha.replace(/1/g, "2")), owed({ state: "pending" }), 3)), "written", "another key is its own"); - assert.equal(await checks.apply(put(key(), owed({ state: "pending" }), 11)), "written"); - assert.deepEqual(f.sent.map((s) => s.split(" ").slice(0, 3).join(" ")), ["novox/mesh-sdk@01234567 mesh/check success", - "novox/mesh-sdk@01234567 mesh/check success", "novox/mesh-sdk@02234567 mesh/check pending", "novox/mesh-sdk@01234567 mesh/check pending"]); -}); - -test("a forge that refuses is asked again until it writes, unless a newer revision of the key came meanwhile", async () => { - const { CheckStatuses } = await import("../check.ts"); - const f = forge(); - const said: string[] = []; - const waits: number[] = []; - const checks = new CheckStatuses(f, (l) => said.push(l), async (ms) => { waits.push(ms); }); - f.fail(2); - await checks.take(put(key(), owed({ state: "success" }), 5)); - await checks.settled(); - assert.deepEqual(f.sent.map((s) => s.split(" ")[2]), ["success"], "written on the third ask"); - assert.deepEqual(waits, [2_000, 5_000]); - assert.match(said[0], /502.*asked again/); - // A refusal whose key moved on meanwhile is not asked again: the newer revision is written, the older never after it. - const g = forge(); - let release: () => void = () => {}; - const gate = new Promise((r) => { release = r; }); - const later = new CheckStatuses(g, () => {}, () => gate); - g.fail(1); - await later.take(put(key(), owed({ state: "pending" }), 6)); - await later.take(put(key(), owed({ state: "success" }), 7)); - release(); - await later.settled(); - assert.deepEqual(g.sent.map((s) => s.split(" ")[2]), ["success"]); -}); - -test("a watch refused at first is asked again, said once, and once granted writes the latest revision of each key", async () => { - const { CheckStatuses, watchOwed } = await import("../check.ts"); - const f = forge(); - const said: string[] = []; - const checks = new CheckStatuses(f, () => {}, noSleep); - // The checker's bucket as the bus keeps it: two revisions of one key written while the read was refused. - const kept = new Map(); - kept.set(key(), { value: owed({ state: "pending" }), revision: 3 }); - kept.set(key(), { value: owed({ state: "success" }), revision: 4 }); - let asked = 0; - const watch = async (handler: (c: any) => Promise) => { - asked++; - if (asked < 3) throw new Error("gitea keeps and reads no state called \"mesh-delivery-checker.owed\""); - for (const [k, e] of kept) await handler(put(k, e.value, e.revision, true)); - }; - await watchOwed(watch, checks, noSleep, (l) => said.push(l)); - await checks.settled(); - assert.equal(asked, 3); - assert.deepEqual(f.sent.map((s) => s.split(" ")[2]), ["success"], "only the latest revision is written"); - assert.equal(said.filter((l) => /could not be bound/.test(l)).length, 1, `said once: ${said.join(" | ")}`); - assert.ok(said.some((l) => /bound after 3 attempts/.test(l)), said.join(" | ")); -}); - -test("no tool writes mesh/check: gitea_check_status is gone and gitea_commit_status refuses it", async () => { - const delivery = await import("../delivery.ts"); - assert.equal((delivery as Record).checkStatusTool, undefined, "delivery.ts serves no gitea_check_status"); - assert.throws(() => delivery.deliveryStatus("mesh/check", "success", "x"), /mesh-delivery-checker\.owed/); - const index = readFileSync(new URL("../index.ts", import.meta.url), "utf8"); - assert.ok(!/gitea_check_status|checkStatusTool/.test(index), "index.ts serves no gitea_check_status"); -}); - -test("the watch's current values are taken without waiting on the forge, so a slow forge never times the watch out (issue 1726)", async () => { - // The tool runtime hands a watch's current values one at a time and waits for each to be taken; it gives the whole set - // 10 s (mesh-tools StateTimeout) and otherwise ends the watch. On 2026-10-23 the owed held ~200 keys, each written to the - // forge before the next was handed over: the set took longer than 10 s, the watch ended at every attempt, and no - // verdict reached the forge's mesh/check again. - const { CheckStatuses, watchOwed } = await import("../check.ts"); - const sent: string[] = []; - let release: () => void = () => {}; - const slow = new Promise((r) => { release = r; }); - const f = { setCommitStatus: async (_o: string, _r: string, s: string, st: any) => { await slow; sent.push(`${s.slice(0, 4)} ${st.state}`); } }; - const checks = new CheckStatuses(f, () => {}, noSleep); - const keys = Array.from({ length: 200 }, (_, i) => key(i.toString(16).padStart(4, "0") + sha.slice(4), "novox/mesh-sdk", i + 1)); - let handedOver = 0; - let attempts = 0; - const watch = async (handler: (c: any) => Promise) => { - if (++attempts > 1) return; // a second attempt is not what this test waits for: the first must hold - // The runtime's bound on the current values: the forge has answered none of them yet. - const timedOut = new Promise<"timeout">((r) => setTimeout(() => r("timeout"), 200)); - const delivered = (async () => { for (const [i, k] of keys.entries()) { await handler(put(k, owed(), i + 1, true)); handedOver++; } return "delivered" as const; })(); - if (await Promise.race([delivered, timedOut]) === "timeout") throw new Error("the current values of mesh-delivery-checker.owed did not arrive in 10s"); - }; - const said: string[] = []; - await watchOwed(watch, checks, noSleep, (l) => said.push(l)); - assert.equal(handedOver, 200, "every current value was taken before the forge answered"); - assert.ok(!said.some((l) => /could not be bound/.test(l)), said.join(" | ")); - assert.equal(sent.length, 0, "nothing is written before the forge answers"); - release(); - await checks.settled(); - assert.equal(sent.length, 200, "every owed status is written once the forge answers"); -}); - -test("the writes a watch's start hands over go to the forge a few at a time, never all at once (issue 1726)", async () => { - const { CheckStatuses, WRITES_AT_ONCE } = await import("../check.ts"); - let running = 0, most = 0, done = 0; - const f = { setCommitStatus: async () => { running++; most = Math.max(most, running); await new Promise((r) => setTimeout(r, 1)); running--; done++; } }; - const checks = new CheckStatuses(f, () => {}, noSleep); - for (let i = 0; i < 50; i++) await checks.take(put(key(i.toString(16).padStart(4, "0") + sha.slice(4), "novox/mesh-sdk", i + 1), owed(), i + 1, true)); - await checks.settled(); - assert.equal(done, 50); - assert.equal(most, WRITES_AT_ONCE); -}); - -// The sweep of 2026-10-30 (issue 774): a status the forge refuses with 403 or 404 was asked again every 30 s forever (hal -// and novadock, 119 times each), and a key the checker deleted kept its loop running. Now a 403 or 404 is said once and -// never asked again, a 5xx is still asked again, and a delete ends a loop that is running. -test("a refusal for good is said once and not asked again; a delete ends a running ask again", async () => { - const { CheckStatuses, refusedForGood } = await import("../check.ts"); - for (const [err, forGood] of [["Gitea API /repos/jschoubben/hal/statuses/x: 403 forbidden", true], - ["Gitea API /repos/jschoubben/novadock/statuses/x: 404 not found", true], ["Gitea API /repos/x/statuses: 502 bad gateway", false], - ["Gitea API /repos/x: no answer in 10 s, the request was cut", false]] as [string, boolean][]) { - assert.equal(refusedForGood(new Error(err)), forGood, err); - } - const said: string[] = []; - const forbidden = { setCommitStatus: async () => { throw new Error("Gitea API /repos/jschoubben/hal/statuses/x: 403 forbidden"); } }; - const checks = new CheckStatuses(forbidden, (l) => said.push(l), async () => { throw new Error("asked again"); }); - await checks.take(put(key(sha, "jschoubben/hal"), owed(), 3)); - await checks.settled(); - assert.equal(said.length, 1); - assert.match(said[0], /refused for good.*403.*not asked again/); - // A 502 asks again; the key's delete meanwhile ends the loop before its next ask. - const g = forge(); - let release: () => void = () => {}; - const gate = new Promise((r) => { release = r; }); - const later = new CheckStatuses(g, () => {}, () => gate); - g.fail(5); - await later.take(put(key(), owed(), 8)); - await later.take({ state: "mesh-delivery-checker.owed", key: key(), op: "delete", revision: 9, current: false } as any); - release(); - await later.settled(); - assert.deepEqual(g.sent, [], "a deleted key is not written"); -}); diff --git a/modules/gitea/test/delivery.test.ts b/modules/gitea/test/delivery.test.ts index 724660283..b134f8696 100644 --- a/modules/gitea/test/delivery.test.ts +++ b/modules/gitea/test/delivery.test.ts @@ -41,41 +41,12 @@ test("a note that cannot be written is said, never read as written", async () => await assert.rejects(appendNote(runner, "gitea", "novox", "x", "abcdef1234567", "mesh-plan", "l"), /could not be appended/); }); -test("one view per pull request, found by its marker; only the mesh's statuses", async () => { - const { viewBody, viewComment, deliveryStatus, VIEW_MARKER } = await import("../delivery.ts"); +test("one view per pull request, found by its marker", async () => { + const { viewBody, viewComment, VIEW_MARKER } = await import("../delivery.ts"); assert.ok(viewBody("**Delivery**").startsWith(VIEW_MARKER)); assert.equal(viewBody(`${VIEW_MARKER}\nx`), `${VIEW_MARKER}\nx`, "a marked body is kept as it is"); const comments = [{ id: 1, body: "a review" }, { id: 2, body: `${VIEW_MARKER}\nold` }, { id: 3, body: `${VIEW_MARKER}\nlater` }]; assert.equal(viewComment(comments)?.id, 2); assert.equal(viewComment([{ id: 1, body: "x" }]), undefined); - const s = deliveryStatus("mesh/delivery", "pending", "y".repeat(300), "https://forge.invalid/novox/x/pulls/1"); - assert.ok(s.description.length <= 140 && s.target_url); - assert.throws(() => deliveryStatus("ci/other", "success", "x"), /mesh's own/); - assert.throws(() => deliveryStatus("mesh/delivery", "green", "x"), /not a status/); - // The merge check's statuses are its verdict's alone: required by a branch, they are never set by hand. - assert.throws(() => deliveryStatus("mesh/merge-gate", "success", "x"), /merge check's status/); - assert.throws(() => deliveryStatus("mesh/repo-check", "warning", "x"), /merge check's status/); - assert.throws(() => deliveryStatus("mesh/merge-gate", "failure", "x"), /merge check's status/); - // Pending is no verdict: a recheck says it so the old verdict does not let the head merge meanwhile. - assert.equal(deliveryStatus("mesh/merge-gate", "pending", "checking again: main moved on").state, "pending"); - assert.equal(deliveryStatus("mesh/repo-check", "pending", "checking again: main moved on").context, "mesh/repo-check"); - assert.equal(deliveryStatus("mesh/delivery", "success", "x", "javascript:alert(1)").target_url, undefined); }); -// Issue 1301-S4 (ADR 0447 §1; R4e findings K1 and R2-1): main requires mesh/check, so no hand may set it. The status tool -// every caller may use refuses it in every state; no tool writes it at all: it is written from the checker's owed state -// (check.ts, test/check-owed.test.ts; ADR 0521). -test("mesh/check is refused by the status tool in every state", async () => { - const { commitStatusTool, deliveryStatus, CHECK_STATUS } = await import("../delivery.ts"); - assert.equal(CHECK_STATUS, "mesh/check"); - const sha = "0123456789abcdef0123456789abcdef01234567"; - const sent: string[] = []; - const setCommitStatus = async (o: string, r: string, s: string, st: any) => void sent.push(`${o}/${r}@${s} ${st.context}`); - const gates = { rechecked: async () => { throw new Error("not the gate writer's"); } }; - for (const state of ["pending", "success", "failure", "error", "warning"]) { - assert.throws(() => deliveryStatus("mesh/check", state, "x"), /mesh\/check is the checker's status/); - await assert.rejects(commitStatusTool({ setCommitStatus, gates } as any).run({ owner: "novox", repo: "mesh-sdk", sha, - context: "mesh/check", state, description: "by hand" }), /mesh-delivery-checker\.owed/); - } - assert.deepEqual(sent, [], "nothing was sent for a hand-set mesh/check"); -}); diff --git a/modules/gitea/test/gate.test.ts b/modules/gitea/test/gate.test.ts index eee5342b6..404452c84 100644 --- a/modules/gitea/test/gate.test.ts +++ b/modules/gitea/test/gate.test.ts @@ -42,8 +42,7 @@ function aForge() { }; } -// **A recheck's pending goes through the one writer** (novox/hq issue 456). mesh-delivery's recheck puts the gate back -// to pending through the gitea_commit_status tool. +// **A recheck's pending goes through the one writer** (novox/hq issue 456). const recheckAt = new Date("2026-10-11T01:30:00Z"); const checkingAgain = { state: "pending" as const, context: "mesh/merge-gate", description: "checking again: main moved on" }; @@ -63,30 +62,8 @@ test("a recheck the forge does not take fails its caller, and the next write sti const gates = new GateWriter(forge, () => recheckAt); await gates.pending("novox", "hq", SHA); forge.failWrites(); - await assert.rejects(gates.rechecked("novox", "hq", SHA, checkingAgain)); - await gates.rechecked("novox", "hq", SHA, checkingAgain); + await assert.rejects(gates.status("novox", "hq", SHA, checkingAgain)); + await gates.status("novox", "hq", SHA, checkingAgain); assert.equal(gateOf(forge), "pending checking again: main moved on"); }); -test("the status tool writes the merge check's pending through the writer, and the mesh's other statuses as they were", async () => { - const { GateWriter } = await import("../gate.ts"); - const { commitStatusTool } = await import("../delivery.ts"); - const forge = aForge(); - const direct: string[] = []; - const gates = new GateWriter(forge, () => recheckAt); - const tool = commitStatusTool({ - setCommitStatus: async (_o, _r, _sha, s) => void direct.push(`${s.context} ${s.state}`), - gates, - }); - await tool.run({ owner: "novox", repo: "hq", sha: SHA, context: "mesh/merge-gate", state: "pending", description: "checking again: x" }); - await tool.run({ owner: "novox", repo: "hq", sha: SHA, context: "mesh/repo-check", state: "pending", description: "checking again: x" }); - await tool.run({ owner: "novox", repo: "hq", sha: SHA, context: "mesh/delivery", state: "success", description: "delivered" }); - assert.equal(gateOf(forge), "pending checking again: x"); - assert.equal(forge.kept["mesh/repo-check"].state, "pending"); - assert.deepEqual(direct, ["mesh/delivery success"]); - // A head named short is another head to the writer: refused, nothing sent. - await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: SHA.slice(0, 12), context: "mesh/merge-gate", state: "pending" }), - /whole sha/); - await assert.rejects(tool.run({ owner: "novox", repo: "hq", sha: SHA, context: "mesh/merge-gate", state: "success" }), - /set by its verdict only/); -}); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 3b93f56e7..df2cfa1fb 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -1,22 +1,17 @@ // What the forge's tools refuse (guard.ts; novox/hq ADR 0266): an agent reaches the forge through these tools // with a token minted for the forge's administrator, so the tools — not the forge — are what keeps it from -// dropping a branch's protection, writing to main, setting the merge check's status, or merging past the -// checks. Each route a review found is reproduced here against a fake forge, and each is refused with nothing -// sent; the operator-approved merge of a pull request whose checks passed still goes through. +// dropping a branch's protection, writing to main, setting the merge check's status, or merging. Each route a review +// found is reproduced here against a fake forge, and each is refused with nothing sent; a merge is the seat git's +// merge-pull alone, at the head the caller names (the last test's rows). import { test, after, beforeEach } from "node:test"; import assert from "node:assert/strict"; import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; -import { mkdtempSync, readFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { readFileSync } from "node:fs"; import { GiteaClient } from "../dist/client.js"; import { ConfiguredToken } from "../dist/token.js"; import { getGiteaTools } from "../dist/tools/index.js"; -import { mergeTool } from "../dist/merge.js"; -import { commitStatusTool } from "../dist/delivery.js"; -import { GateWriter } from "../dist/gate.js"; import { judgeApiCall, judgeProtectionChange, ruleCovers, routedPath } from "../dist/guard.js"; interface Sent { method: string; path: string; body: any } @@ -54,6 +49,8 @@ const state = { groupStatus: "", siblingGroup: "", reviews: [] as { state: string; commit_id: string; dismissed?: boolean }[], + // The forge's own mergeable of pull request 7; unset, the forge does not say. + mergeable: undefined as boolean | undefined, }; const REVIEWED = "abc1234def5678901234567890abcdef12345678"; const SIBLING_HEAD = "bbbbbbbbbbbb5678901234567890abcdef12345678".slice(0, 40); @@ -107,7 +104,7 @@ const server = createServer(async (req, res) => { state.asked.push(`${method} ${path}${url.search}`); if (method !== "GET") state.sent.push({ method, path, body }); const repo = "/repos/novox/mesh-catalog"; - if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: state.defaultBranch }); + if (method === "GET" && path === repo) return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: state.defaultBranch, permissions: { push: true } }); if (method === "PATCH" && path === repo) { if (body?.default_branch) state.defaultBranch = body.default_branch; return json(res, 200, { full_name: "novox/mesh-catalog", default_branch: state.defaultBranch }); @@ -146,7 +143,7 @@ const server = createServer(async (req, res) => { if (method === "GET" && path === `${repo}/pulls/7`) { return json(res, 200, { number: 7, title: "x", state: state.merged ? "closed" : state.prState, merged: state.merged, head: { ref: "feat/x", sha: state.head }, base: { ref: state.base }, html_url: "http://fake/7", - merge_commit_sha: state.merged ? state.mergeCommit : null }); + merge_commit_sha: state.merged ? state.mergeCommit : null, ...(state.mergeable === undefined ? {} : { mergeable: state.mergeable }) }); } if (method === "GET" && path === `${repo}/git/commits/${state.mergeCommit}`) { if (state.commitsFail) return json(res, state.commitsFail, { message: "the forge is not well" }); @@ -165,6 +162,7 @@ const server = createServer(async (req, res) => { return json(res, 200, url.searchParams.get("page") === "1" ? state.reviews : []); } if (method === "GET" && path === "/repos/novox/mesh-host/issues/114/labels") return json(res, 200, []); + if (method === "GET" && path === `${repo}/pulls/404`) return json(res, 404, { message: "pull request does not exist" }); if (method === "POST" && path === `${repo}/pulls/7/merge`) { state.merged = true; return json(res, 200, null); @@ -175,20 +173,7 @@ await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); after(() => new Promise((r) => server.close(() => r()))); const { port } = server.address() as { port: number }; const client = new GiteaClient(`http://127.0.0.1:${port}`, new ConfiguredToken("t")); -// gitea_commit_status is served from the watcher's bundle, beside the merge check's one writer (novox/hq issue 456). -const stateDir = mkdtempSync(join(tmpdir(), "gitea-tools-")); -// The merge check's one writer, as the watcher's bundle holds it: what the merge tool's pending goes through. -const writer = new GateWriter({ setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), - headChecks: async () => ({}) }); -const throughTheWriter: string[] = []; -const gates = { rechecked: (o: string, r: string, sha: string, s: any) => { - throughTheWriter.push(`${sha.slice(0, 12)} ${s.context} ${s.state}`); - return writer.rechecked(o, r, sha, s); -} }; -const tools = [...getGiteaTools(client), commitStatusTool({ - setCommitStatus: (o, r, sha, s) => client.setCommitStatus(o, r, sha, s), gates: writer, -}), mergeTool(client, { gates, stateDir, emit: async (type, body) => { state.emitted.push({ type, body }); }, - now: () => new Date(2026, 9, 12, 15, 1) })]; +const tools = getGiteaTools(client); const tool = (name: string) => tools.find((t) => t.name === name)!; const writes = () => state.sent.map((s) => `${s.method} ${s.path}`); @@ -276,203 +261,6 @@ test("gitea_branch_protection_set writes no required status, and refuses a push assert.equal(sent.enable_push, false); }); -test("gitea_merge_pull_request merges only a head whose required statuses all succeeded, and only that head", async () => { - const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED }); - state.statuses = [judgedAt(TRUNK)[0], judgedAt(TRUNK, "failure")[1]]; - await assert.rejects(merge(), /not merged: .*mesh\/repo-check \(failure\)/); - state.statuses = [judgedAt(TRUNK)[0]]; - await assert.rejects(merge(), /mesh\/repo-check \(missing\)/); - await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED, method: "manually-merged" }), /not a merge method/); - assert.deepEqual(writes(), []); - // The operator said yes and the checks passed: the flow that stays. - state.statuses = judgedAt(TRUNK); - await merge(); - const sent = state.sent.find((s) => s.path.endsWith("/pulls/7/merge"))!; - assert.equal(sent.body.force_merge, false); - assert.equal(sent.body.head_commit_id, REVIEWED); -}); - - -// **The merge tool refuses a verdict whose trunk commit is not the trunk's head, and asks the check itself** (novox/hq -// ADR 0323 §5, issue 341): node-engine #66 merged at 00:54 on a verdict of 00:49 that judged its head alone, after #65 -// had moved the trunk. Both commits are named; one check is asked, the announcement a new head makes, and not a second -// while it is on its way; and once the verdict judges -// the trunk's head and passes, the merge goes through — asked again, never by itself. -test("gitea_merge_pull_request refuses a verdict judged against an earlier trunk, and asks the check once", async () => { - const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }); - - state.trunk = MOVED; // another merge moved the trunk after the verdict - await assert.rejects(merge(), (err: Error) => { - assert.match(err.message, /is not merged: its verdict was judged against trunk 7a8b9c0d1e2f, and main is now 3c4d5e6f7a8b/); - assert.match(err.message, /check of head abc1234def56 merged into trunk 3c4d5e6f7a8b was asked/); - assert.match(err.message, /never merges by itself/); - return true; - }); - assert.equal(state.merged, false); - assert.equal(state.emitted.length, 1, "one check asked"); - assert.equal(state.emitted[0].type, "pull.updated"); - assert.equal(state.emitted[0].body.head_sha, REVIEWED); - assert.equal(state.emitted[0].body.base, "main"); - assert.deepEqual(state.emitted[0].body.paths, ["modules/gitea/x.ts"]); - // The announcement names the trunk's head, for a verdict no build judges (ADR 0323 §3). - assert.equal(state.emitted[0].body.base_sha, MOVED); - // Its pending went through the merge check's one writer (novox/hq issue 456), not around it. - assert.deepEqual(throughTheWriter.splice(0), ["abc1234def56 mesh/merge-gate pending", "abc1234def56 mesh/repo-check pending"]); - const gate = state.statuses.find((s) => s.context === "mesh/merge-gate")!; - assert.equal(gate.state, "pending"); - assert.match(gate.description!, /^waiting in the build queue since 15:01: asked by the merge tool against trunk 3c4d5e6f7a8b/); - assert.equal(state.statuses.find((s) => s.context === "mesh/repo-check")!.state, "pending"); - - // Asked again while that check is on its way: refused, and nothing is asked a second time. - await assert.rejects(merge(), /already asked and not yet answered .*not asked again.*mesh-delivery\.recheck/); - assert.equal(state.emitted.length, 1, "no second check while one is on its way"); - state.statuses = [{ ...gate, description: "building on ace since 15:02" }, ...state.statuses.slice(1)]; - await assert.rejects(merge(), /not asked again/); - assert.equal(state.emitted.length, 1); - - // A verdict put out of date by the trunk's move is asked too, once. - state.statuses = judgedAt(TRUNK).map((s) => ({ ...s, state: "pending", - description: "out of date: judged against trunk 7a8b9c0d1e2f, the trunk is now 3c4d5e6f7a8b; judged again on a recheck or when it is merged" })); - await assert.rejects(merge(), /was judged against trunk 7a8b9c0d1e2f, and main is now 3c4d5e6f7a8b\. The check .* was asked/); - assert.equal(state.emitted.length, 2); - - // A verdict that names no trunk commit — a build seat from before the rule — is not taken as the trunk's head. - state.statuses = [{ context: "mesh/merge-gate", state: "success", description: "pass: x" }, - { context: "mesh/repo-check", state: "success", description: "pass: y" }]; - await assert.rejects(merge(), /names no trunk commit it was judged against, and main is now 3c4d5e6f7a8b/); - assert.equal(state.emitted.length, 3); - assert.equal(state.merged, false); - - // An ask that cannot be announced puts the statuses back to out of date, so the next ask asks again. - state.statuses = judgedAt(TRUNK); - const failing = mergeTool(client, { gates, stateDir, emit: async () => { throw new Error("the bus is away"); } }); - await assert.rejects(failing.run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }), - /its check could not be asked: the bus is away\. Nothing was merged/); - assert.match(state.statuses.find((s) => s.context === "mesh/merge-gate")!.description!, /^out of date: /); - await assert.rejects(merge(), /was asked/); - assert.equal(state.emitted.length, 4, "asked again after the announcement failed"); - - // The check judged the merge into the trunk's head and passed: asked again at the reviewed head, it merges. - state.statuses = judgedAt(MOVED); - const done = (await merge()) as Record; - assert.equal(done.merged, true); - assert.equal(state.emitted.length, 4, "a merge asks no check"); -}); - -// **A group member whose gate waits for its group's composed check is not announced again by the merge tool** (novox/hq issue -// 1106): the member's own check can only say "composed" again, so each refusal's ask brought a verdict that names no trunk and -// the next merge was refused the same way, for ever. The tool says whose verdict is awaited and asks nothing. -test("gitea_merge_pull_request asks no check of a member that waits for its group's composed check", async () => { - const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }); - state.trunk = TRUNK; - const before = state.emitted.length; - state.statuses = [{ context: "mesh/merge-gate", state: "pending", description: "waits for its group's composed check" }, - { context: "mesh/repo-check", state: "success", description: "pass: y" }]; - for (let i = 0; i < 3; i++) { - await assert.rejects(merge(), (err: Error) => { - assert.match(err.message, /waits for its group's composed check/); - assert.match(err.message, /mesh-delivery asks it, this tool asks no check/); - assert.match(err.message, /Nothing was merged/); - return true; - }); - } - assert.equal(state.emitted.length, before, "no check is announced again and again"); - assert.equal(state.merged, false); - // Once the composed verdict names the member's unmoved trunk, it merges. - state.statuses = judgedAt(TRUNK); - assert.equal(((await merge()) as Record).merged, true); -}); - -// A merge lands only the head that was reviewed (novox/hq issue 381): mesh-controller#197 was reviewed at one head, -// force-pushed, checked green at the new one, and merged there, because the tool merged whatever the head was. -test("gitea_merge_pull_request merges only the head it is named, and refuses one that is not the head", async () => { - const merge = (head?: string) => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - ...(head === undefined ? {} : { head }) }); - - // No head named: there is no review to hold the merge to. - await assert.rejects(merge(), /name the head you reviewed/); - await assert.rejects(merge(" "), /name the head you reviewed/); - await assert.rejects(merge("main"), /does not name one commit/); - // Seven characters can be matched by a commit made to match them: twelve are asked for. - await assert.rejects(merge(REVIEWED.slice(0, 7)), /does not name one commit — give its sha, or at least its first 12/); - await assert.rejects(merge(REVIEWED.slice(0, 11)), /does not name one commit/); - - // The head moved after the review, and its checks passed where it is now: the case of mesh-controller#197. - state.head = "3ced96fc6f8ad6cc7b93698912cb08a71635158a"; - await assert.rejects(merge(REVIEWED), /its head is 3ced96fc6f8a, not the abc1234def5678901234567890abcdef12345678 you named/); - await assert.rejects(merge(REVIEWED.slice(0, 12)), /not the abc1234def56 you named\. If you reviewed an earlier head/); - assert.deepEqual(writes(), [], "a refused merge sends nothing"); - - // Named as reviewed, by its first 12 characters, in either case: the forge is handed the whole head, and the merge - // commit's second parent says that head landed. - state.head = REVIEWED; - const done = (await merge(REVIEWED.slice(0, 12).toUpperCase())) as Record; - const sent = state.sent.find((s) => s.path.endsWith("/pulls/7/merge"))!; - assert.equal(sent.body.head_commit_id, REVIEWED); - assert.equal(done.head, REVIEWED); - assert.equal(done.landed, REVIEWED); - assert.equal(done.read_back, undefined); -}); - -// The forge is handed the head, and what landed is read from the merge commit, not from the forge's record of the -// pull request (which here still says the head it was handed). Another head landing is an error that says the merge -// happened, never a field beside merged: true. -test("gitea_merge_pull_request is loud when the forge merged a head other than the one it was handed", async () => { - state.landed = "3ced96fc6f8ad6cc7b93698912cb08a71635158a"; - await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED }), /WAS merged \(feedface0000.*\), but the forge merged 3ced96fc6f8a, not the abc1234def56 it was handed/); - assert.equal(state.merged, true, "the merge happened: the error is what says so"); -}); - -// The merge has happened once the forge says so: a read-back that fails after it must not read as nothing merged. -test("gitea_merge_pull_request says the merge happened when reading back what landed fails", async () => { - state.commitsFail = 500; - const done = (await tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED })) as Record; - assert.equal(state.merged, true); - assert.equal(done.merged, true); - assert.equal(done.landed, undefined); - assert.match(String(done.read_back), /the merge happened, but reading back what landed failed/); - - // A parent the forge names without its sha is a failed read-back, never an empty head said to have landed. - state.merged = false; - state.commitsFail = 0; - state.parents = ["8a53532d89b20f0fe4fd0fbc814f2e86b61cbc6b", ""]; - const blank = (await tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED })) as Record; - assert.equal(blank.merged, true); - assert.match(String(blank.read_back), /without its sha/); -}); - -test("gitea_merge_pull_request reads back a fast-forward, and says it cannot read back a squash or a rebase", async () => { - state.mergeCommit = REVIEWED; // a fast-forward lands the head itself - // Its parents name another commit second, so reading a fast-forward as a merge's second parent would fail here. - state.parents = ["8a53532d89b20f0fe4fd0fbc814f2e86b61cbc6b"]; - const ff = (await tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED, method: "fast-forward-only" })) as Record; - assert.equal(ff.landed, REVIEWED); - assert.equal(ff.read_back, undefined); - - // A fast-forward whose commit the forge names otherwise is said, not raised: that it names the head is assumed. - state.merged = false; - state.mergeCommit = "feedface00000000000000000000000000000000"; - const odd = (await tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED, method: "fast-forward-only" })) as Record; - assert.equal(odd.merged, true); - assert.equal(odd.landed, undefined); - assert.match(String(odd.read_back), /as this fast-forward's commit, not the head/); - - for (const method of ["squash", "rebase", "rebase-merge"]) { - state.merged = false; - state.mergeCommit = "feedface00000000000000000000000000000000"; - const done = (await tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED, method })) as Record; - assert.equal(done.landed, undefined, method); - assert.match(String(done.read_back), new RegExp(`a ${method} makes new commits that name no head`)); - } -}); test("gitea_delete_repo deletes only the tools' own user's repository, unguarded, untouched for 30 days, named in full", async () => { state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) @@ -496,65 +284,6 @@ test("gitea_api sends the path it judged, and refuses moving a pull request's ba assert.equal(writes().length, 2); }); -test("gitea_merge_pull_request merges only into a branch whose protection refuses its push", async () => { - const approved_heads = [`novox/mesh-catalog@${REVIEWED}`]; - state.statuses = [{ context: "mesh/check", state: "success", description: `pass ${REVIEWED.slice(0, 12)} on ${TRUNK.slice(0, 12)}` }]; - state.rules = []; - await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, - approved_heads }), /no protection rule refusing/); - state.rules = [{ rule_name: "main", enable_push: true, enable_status_check: false, status_check_contexts: [] }]; - await assert.rejects(tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, - approved_heads }), /no protection rule refusing/); - assert.deepEqual(writes(), []); -}); - -// **The forge requires no status to merge; the merge tool still judges mesh/check, and asks the operator's Approve** -// (novox/hq ADR 0534 §4): on a main whose rule takes no push and requires no status, a head merges only with -// approved_heads naming it and a mesh/check that passed against the trunk's head at the merge. -test("gitea_merge_pull_request judges mesh/check on a main requiring no status, and merges only an approved head", async () => { - const { keepSiblings, siblingsFile } = await import("../dist/siblings.js"); - keepSiblings(siblingsFile(stateDir), {}); - const approved_heads = [`novox/mesh-catalog@${REVIEWED}`]; - const merge = (args: Record = { approved_heads }) => - tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, ...args }); - state.rules = [{ rule_name: "main", enable_push: false, enable_status_check: false, status_check_contexts: [], - block_admin_merge_override: true, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"] }]; - const line = (trunk: string, s = "success", words = "pass") => - [{ context: "mesh/check", state: s, description: `${words} ${REVIEWED.slice(0, 12)} on ${trunk.slice(0, 12)} | unit 12 run` }]; - const before = state.emitted.length; - - state.statuses = line(TRUNK); - await assert.rejects(merge({}), /Approve/); - state.statuses = []; - await assert.rejects(merge(), /mesh\/check is not set yet/); - state.statuses = line(TRUNK, "pending", "checking"); - await assert.rejects(merge(), /mesh\/check is pending/); - state.statuses = line(TRUNK, "failure", "fail"); - await assert.rejects(merge(), /mesh\/check is failure/); - state.statuses = line(MOVED); - await assert.rejects(merge(), new RegExp(`${MOVED.slice(0, 12)}.*${TRUNK.slice(0, 12)}`)); - state.statuses = judgedAt(TRUNK); // the old two, current: not mesh/check - await assert.rejects(merge(), /mesh\/check/); - assert.deepEqual(writes(), [], "nothing is written: mesh/check is the checker's"); - assert.equal(state.emitted.length, before, "no check is asked"); - assert.equal(state.merged, false); - - state.statuses = line(TRUNK); - assert.equal(((await merge()) as Record).merged, true); - - // check-gates-merge false (the operator, 2026-10-25): a head the Approve names merges whatever its mesh/check says; - // without approved_heads it is refused as before. - const ungated = mergeTool(client, { gates, stateDir, emit: async () => {}, checkGates: () => false }); - const mergeUngated = (args: Record = { approved_heads }) => - ungated.run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, ...args }); - for (const statuses of [[], line(TRUNK, "pending", "checking"), line(TRUNK, "failure", "fail"), line(MOVED)]) { - state.merged = false; - state.statuses = statuses; - await assert.rejects(mergeUngated({}), /Approve/); - assert.equal(((await mergeUngated()) as Record).merged, true, JSON.stringify(statuses)); - } -}); - test("a rule named for a branch is not made weaker than the glob rule that guards it now", async () => { state.admin = true; // reads the rules' list, a repository administrator's (novox/hq ADR 0365) state.rules.push({ rule_name: "release/*", enable_push: false, enable_status_check: true, @@ -635,9 +364,6 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud ["gitea_delete_branch", { ...repo, branch: "%2e%2e" }], ["gitea_get_file", { ...repo, path: "../../../../version" }], ["gitea_get_file", { ...repo, path: "README.md?sudo=jschoubben" }], - ["gitea_commit_status", { ...repo, sha: "abc1234?sudo=jschoubben", context: "mesh/delivery", state: "success" }], - ["gitea_commit_status", { ...repo, sha: "../../../admin/users", context: "mesh/delivery", state: "success" }], - ["gitea_commit_status", { ...repo, sha: "0123456789abcdef0123456789abcdef01234567", context: "mesh/check", state: "success" }], ["gitea_commit_statuses", { ...repo, sha: ".." }], ["gitea_commit_statuses", { ...repo, sha: "main?sudo=jschoubben" }], ["gitea_branch_protection_set", { ...repo, branch: "..", status_checks: "mesh/merge-gate" }], @@ -661,172 +387,6 @@ test("no tool's caller text becomes another path or another user: `..` and `?sud }); -// **A member judged beside its siblings merges only with its group passed and the Approve covering each head** (novox/hq -// ADR 0382 §6 as ADR 0421 §9 changes it, issues 867 and 1022): the verdict names the sibling; mesh/delivery-group must -// be a success at the member's head and the sibling's head that verdict read, and mesh-delivery hands the heads the -// operator's Approve covers: every head read must be among them. No forge review is asked. A verdict that read none -// merges as before. -test("gitea_merge_pull_request merges a verdict that read siblings only with its group passed and each head approved", async () => { - const { keepSiblings, siblingsFile } = await import("../dist/siblings.js"); - const both = ["novox/mesh-catalog@" + REVIEWED, "novox/mesh-host@" + SIBLING_HEAD]; - const merge = (approved_heads?: string[]) => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED, ...(approved_heads ? { approved_heads } : {}) }); - const beside = `head ${REVIEWED.slice(0, 12)} merged into trunk ${TRUNK.slice(0, 12)} beside mesh-host#114@${SIBLING_HEAD.slice(0, 8)}`; - const named = (siblings: unknown[]) => keepSiblings(siblingsFile(stateDir), - { "novox/mesh-catalog#7": { head: REVIEWED, siblings: siblings as any } }); - state.statuses = [{ context: "mesh/merge-gate", state: "success", description: `${beside} — pass: x` }, - { context: "mesh/repo-check", state: "success", description: `${beside} — pass: y` }]; - named([{ repository: "novox/mesh-host", number: 114, head: SIBLING_HEAD, base: "main", base_commit: "c".repeat(40) }]); - - // Nothing passed and no approved heads handed: every missing thing is named, nothing merged. - await assert.rejects(merge(), (err: Error) => { - assert.match(err.message, /is not merged: its verdict was judged beside 1 sibling\(s\), and mesh\/delivery-group is missing at mesh-catalog#7@abc1234def56; mesh-catalog#7@abc1234def56 is covered by no Approve: the merge was handed no approved heads; mesh\/delivery-group is missing at mesh-host#114@bbbbbbbbbbbb; mesh-host#114@bbbbbbbbbbbb is covered by no Approve/); - return true; - }); - assert.equal(state.merged, false); - - // The group passed at the member's head only: the sibling's head still lacks it. - state.groupStatus = "success"; - await assert.rejects(merge(both), /mesh\/delivery-group is missing at mesh-host#114@bbbbbbbbbbbb\. A member judged beside/); - // Passed at both heads, but the Approve covers the sibling at another head, or does not name it: not an approved head. - state.siblingGroup = "success"; - await assert.rejects(merge(["novox/mesh-catalog@" + REVIEWED, "novox/mesh-host@" + "d".repeat(40)]), - /mesh-host#114@bbbbbbbbbbbb is not the head the Approve covers \(dddddddddddd\)\. A member judged beside/); - await assert.rejects(merge(["novox/mesh-catalog@" + REVIEWED]), /mesh-host#114@bbbbbbbbbbbb is not among the heads the Approve covers/); - assert.equal(state.merged, false); - - // Passed and approved at the very heads read, with no forge review of the sibling: it merges. - assert.deepEqual(state.reviews, [], "no review is asked of the forge"); - assert.equal(((await merge(both)) as Record).merged, true); - - // A status that says it read siblings, with no record of them, is no verdict to merge on. - state.merged = false; - keepSiblings(siblingsFile(stateDir), {}); - await assert.rejects(merge(both), /its verdict read siblings, and which ones is not held here/); - assert.equal(state.merged, false); - - // A verdict that read none (an older controller, or a group of one) merges exactly as before. - state.statuses = judgedAt(TRUNK); - state.groupStatus = ""; - assert.equal(((await merge()) as Record).merged, true); -}); - -// **The merge lands only the head the Approve named** (novox/hq ADR 0421 §9, issue 1022): a pull request whose head -// differs from the approved one, or whose repository the Approve does not name, is refused with nothing merged. -test("gitea_merge_pull_request refuses a head the Approve did not name", async () => { - state.merged = false; - state.statuses = judgedAt(TRUNK); - const merge = (approved_heads: unknown) => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, - head: REVIEWED, approved_heads }); - await assert.rejects(merge(["novox/mesh-catalog@" + "e".repeat(40)]), /is not the head the Approve covers \(eeeeeeeeeeee\)/); - await assert.rejects(merge(["novox/mesh-host@" + REVIEWED]), /is not among the heads the Approve covers/); - await assert.rejects(merge(["novox/mesh-catalog@abc123"]), /which is not owner\/repo@/); - await assert.rejects(merge("novox/mesh-catalog@" + REVIEWED), /must be a list/); - assert.equal(state.merged, false); - assert.equal(((await merge(["novox/mesh-catalog@" + REVIEWED.toUpperCase()])) as Record).merged, true); -}); - -// **A group approved together merges in one pass, on the real merge tool** (novox/hq ADR 0418, issue 1049): the member's -// own verdict named its sibling at an old head; the sibling got a new head, the group's composed verdict passed for both, -// the sibling merged that head, and the member then merges on its first ask: the sibling's own merge is no move of its -// base. The records are the module's own (siblings.ts), the weighing is what the sibling's merge announcement does. -test("a group of two merges in one pass: the sibling's merge of the composed head does not stop the member's merge (issue 1049)", async () => { - const { keepSiblings, siblingsFile, weighSiblings } = await import("../dist/siblings.js"); - const both = ["novox/mesh-catalog@" + REVIEWED, "novox/mesh-host@" + SIBLING_HEAD]; - const OLD_SIBLING = "9".repeat(40), HOST_BASE = "c".repeat(40), HOST_MERGE = "f".repeat(40); - const beside = `head ${REVIEWED.slice(0, 12)} merged into trunk ${TRUNK.slice(0, 12)} beside mesh-host#114@${OLD_SIBLING.slice(0, 8)}`; - state.statuses = [{ context: "mesh/merge-gate", state: "success", - description: `head ${REVIEWED.slice(0, 12)} merged into trunk ${TRUNK.slice(0, 12)} \u2014 pass: group g composed: ok` }, - { context: "mesh/repo-check", state: "success", description: `${beside} \u2014 pass: y` }]; - state.groupStatus = "success"; - state.siblingGroup = "success"; - // The member's own verdict named the sibling at its old head. - const record: Record = { "novox/mesh-catalog#7": { head: REVIEWED, - siblings: [{ repository: "novox/mesh-host", number: 114, head: OLD_SIBLING, base: "main", base_commit: HOST_BASE }] } }; - // The composed verdict then judged both new heads. - Object.assign(record, { - "novox/mesh-catalog#7": { head: REVIEWED, siblings: [{ repository: "novox/mesh-host", number: 114, head: SIBLING_HEAD, base: "main", base_commit: HOST_BASE.slice(0, 12) }] }, - "novox/mesh-host#114": { head: SIBLING_HEAD, siblings: [{ repository: "novox/mesh-catalog", number: 7, head: REVIEWED, base: "main", base_commit: TRUNK.slice(0, 12) }] } }); - // The sibling merges its head; its announcement weighs the member's record. The member's gate is written only by what - // that weighing decides, through the writer. - const pulled = { number: 114, title: "", html_url: "", merged: true, state: "closed", head_sha: SIBLING_HEAD, merge_commit_sha: HOST_MERGE }; - const forge = { - async getPullRequest(_o: string, repo: string, n: number) { - return repo === "mesh-host" ? pulled : { number: n, title: "", html_url: "", merged: false, state: "open", head_sha: REVIEWED }; - }, - async branchHead() { return HOST_MERGE; }, - async commitParents() { return [HOST_BASE, SIBLING_HEAD]; }, - }; - const wrote: string[] = []; - const gatesOut = { async outOfDate(_o: string, _r: string, sha: string, decide: (c: any) => any[]) { - const put = decide({ "mesh/merge-gate": { state: "success", description: state.statuses[0].description }, - "mesh/repo-check": { state: "success", description: "x" } }); - for (const s of put) { - wrote.push(`${sha.slice(0, 12)} ${s.context} ${s.state}`); - state.statuses = [{ ...s }, ...state.statuses.filter((x) => x.context !== s.context)]; - } - return put; - } }; - await weighSiblings(forge as any, gatesOut as any, record, "novox/mesh-host", 114); - assert.deepEqual(wrote, [], "the member's statuses were left alone"); - keepSiblings(siblingsFile(stateDir), record); - // The member merges on its first ask, with the one Approve's heads. - const done = (await tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED, - approved_heads: both })) as Record; - assert.equal(done.merged, true); - assert.equal(state.emitted.length, 0, "no check was asked"); -}); - -// **Issue 1301 (ADR 0447 §1, §4; ADR 0503 §3, §5): the merge tool judges on the status the base requires.** A main that -// requires mesh/check is judged by the checker's line ("pass on …"): it merges on a pass that names the -// trunk's head, and refuses one that names an older trunk, naming both, writing nothing and asking nothing — a re-check on -// a trunk's move is the checker's. No mesh/merge-gate is read or written there. -test("gitea_merge_pull_request judges a main requiring mesh/check on mesh/check alone, and asks nothing of it", async () => { - const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }); - state.rules = [{ ...protectedMain(), status_check_contexts: ["mesh/check"] }]; - const line = (trunk: string, s = "success", words = "pass") => - [{ context: "mesh/check", state: s, description: `${words} ${REVIEWED.slice(0, 12)} on ${trunk.slice(0, 12)} | unit 12 run` }]; - const before = state.emitted.length; - - state.statuses = line(MOVED); - await assert.rejects(merge(), (err: Error) => { - assert.match(err.message, /mesh\/check/); - assert.match(err.message, new RegExp(`${MOVED.slice(0, 12)}.*${TRUNK.slice(0, 12)}`)); - assert.match(err.message, /checker checks it again/i); - assert.match(err.message, /Nothing was merged/); - return true; - }); - state.statuses = line(TRUNK, "pending", "checking"); - await assert.rejects(merge(), /mesh\/check is pending/); - state.statuses = judgedAt(TRUNK); // the old two, current: not what this main requires - await assert.rejects(merge(), /mesh\/check/); - assert.deepEqual(writes(), [], "nothing is written: mesh/check is the checker's"); - assert.equal(state.emitted.length, before, "no check is asked"); - assert.equal(state.merged, false); - - state.statuses = line(TRUNK); - assert.equal(((await merge()) as Record).merged, true); -}); - -// hq's main requires mesh/repo-check alone (ADR 0503 §5): judged on its words, and an out-of-date one is asked again on -// mesh/repo-check alone; no mesh/merge-gate is written on it. -test("gitea_merge_pull_request judges a main requiring mesh/repo-check alone on it, and writes no mesh/merge-gate", async () => { - const { keepSiblings, siblingsFile } = await import("../dist/siblings.js"); - keepSiblings(siblingsFile(stateDir), {}); // no sibling record left by the tests before - const merge = () => tool("gitea_merge_pull_request").run({ owner: "novox", repo: "mesh-catalog", number: 7, head: REVIEWED }); - state.rules = [{ ...protectedMain(), status_check_contexts: ["mesh/repo-check"] }]; - state.statuses = [judgedAt(MOVED)[1]]; - const before = state.emitted.length; - await assert.rejects(merge(), /judged against trunk/); - assert.deepEqual(state.sent.filter((s) => s.path.includes("/statuses/")).map((s) => `${s.body.context} ${s.body.state}`), - ["mesh/repo-check pending"]); - assert.equal(state.emitted.length, before + 1, "its check is asked once"); - await assert.rejects(merge(), /already asked/); - state.sent = []; - state.statuses = [judgedAt(TRUNK)[1]]; - assert.equal(((await merge()) as Record).merged, true); -}); - // Issue 1422 note 7: the two settings gitea_api refuses (ADR 0266) have named tools, held to the same guard as // gitea_branch_protection_set: the tools' account, a stated reason, a branch that exists and that a rule guards. test("gitea_default_branch_set answers the default before and after, and refuses a branch that is missing, unguarded or unexplained", async () => { @@ -870,3 +430,167 @@ test("the two new tools are named for what mesh_search is asked, and gitea_api s assert.ok("refused" in judgeApiCall("PATCH", "/repos/a/b", { default_branch: "x" })); assert.ok("refused" in judgeApiCall("PATCH", "/repos/a/b/pulls/7", { base: "x" })); }); + +// **The seat git's verbs judge their caller, and merge-pull judges the Approve's head and no check** (design 47 *The seat +// git*, ADR 0661 §3): a caller the trusted list does not name is refused before the forge is asked; merge-pull merges the +// head it is named into a base that takes no push, whatever its statuses say, and an id said again after the merge is +// answered performed. +test("the seat git's verbs judge their caller, and merge-pull merges the Approve's head and judges no check", async () => { + const { seatTools, gitCallers } = await import("../dist/seat.js"); + const { runAs } = await import("@novox/mesh-sdk/callers"); + const bridge = "ace.mesh-delivery-forge-bridge"; + const list = JSON.stringify({ publish: ["mesh-build-go"], "merge-pull": ["mesh-delivery-forge-bridge"], + pull: ["mesh-delivery-forge-bridge", "mesh-delivery-gate"], status: ["mesh-delivery-forge-bridge"] }); + assert.deepEqual(Object.keys(gitCallers(list)).sort(), ["merge-pull", "pull", "status"], "only the seat git's lines are its"); + assert.deepEqual(gitCallers(undefined), {}, "unset, no one may call"); + // The head's one writer of its statuses (gate.ts), over a forge whose first write is still in flight when the next is asked. + const { GateWriter } = await import("../dist/gate.js"); + const landed: string[] = []; + let inFlight: Promise | null = null, land = () => {}; + const slow = { + headChecks: async () => ({}), + setCommitStatus: async (_o: string, _r: string, _sha: string, s: { state: string; description: string }) => { + if (inFlight) { const w = inFlight; inFlight = null; await w; } + landed.push(`${s.state} ${s.description}`); + }, + }; + const gates = new GateWriter(slow); + const seat = seatTools(Object.assign(Object.create(client), { setCommitStatus: slow.setCommitStatus }), + { run: async () => ({ stdout: "", code: 0 }), container: "gitea", callers: async () => list, statuses: gates }); + const call = (verb: string, args: Record, caller = bridge) => + runAs({ "mesh/caller": caller }, args, () => seat.find((t: any) => t.name === `git.${verb}`)!.run(args)) as Promise; + const merge = (extra: Record = {}, caller?: string) => + call("merge-pull", { id: "act-1", repository: "novox/mesh-catalog", number: 7, head: REVIEWED, ...extra }, caller); + const rows: { name: string; set: () => void; caller?: string; extra?: Record; want: RegExp | "merged" | "held"; loud?: boolean }[] = [ + { name: "a caller not on the list", set: () => {}, caller: "ace.mesh-delivery-checker", want: /refused for ace.mesh-delivery-checker/ }, + { name: "an unproven caller", set: () => {}, caller: "", want: /no caller the tool runner proved/ }, + { name: "a head that moved", set: () => { state.head = MOVED; }, want: /its head is 3c4d5e6f7a8b, not the abc1234def56/ }, + { name: "a base no rule guards", set: () => { state.rules = []; }, want: /guarded by no protection rule/ }, + { name: "a base that takes a push", set: () => { state.rules = [{ rule_name: "main", enable_push: true }]; }, want: /guarded by no protection rule/ }, + { name: "a failed check", set: () => { state.statuses = [{ context: "mesh/check", state: "failure", description: "fail" }]; }, want: "merged" }, + { name: "no status at all", set: () => { state.statuses = []; }, want: "merged" }, + { name: "a check of another trunk", set: () => { state.statuses = judgedAt(MOVED); }, want: "merged" }, + { name: "the id said again after the merge", set: () => { state.merged = true; }, want: "held" }, + { name: "a pull request merged at another head", set: () => { state.merged = true; state.head = MOVED; }, want: /merged at 3c4d5e6f7a8b, not the abc1234def56/ }, + { name: "a closed pull request", set: () => { state.prState = "closed"; }, want: /it is closed/ }, + { name: "a fast-forward", set: () => { state.parents = [REVIEWED]; }, want: "merged" }, + { name: "the merge read back fails", set: () => { state.commitsFail = 502; }, want: "merged" }, + { name: "the forge merged another head", set: () => { state.landed = MOVED; }, want: /WAS merged .*the forge merged 3c4d5e6f7a8b, not the abc1234def56/, loud: true }, + ]; + for (const r of rows) { + state.merged = false; state.head = REVIEWED; state.statuses = judgedAt(TRUNK); state.sent = []; + state.rules = [protectedMain()]; state.prState = "open"; state.parents = null; state.commitsFail = 0; state.landed = ""; + r.set(); + const got = await merge(r.extra, r.caller); + if (r.want === "merged") { + assert.equal(got.performed, true, `${r.name}: ${JSON.stringify(got)}`); + assert.equal(got["merge-commit"], state.mergeCommit, r.name); + assert.equal(state.sent.find((s) => s.path.endsWith("/pulls/7/merge"))?.body.head_commit_id, REVIEWED, r.name); + } else if (r.want === "held") { + assert.deepEqual([got.performed, got.held], [true, true], r.name); + assert.deepEqual(writes(), [], `${r.name}: nothing merged again`); + } else { + assert.match(String(got.refused ?? got.error), r.want, r.name); + assert.deepEqual(writes(), r.loud ? ["POST /repos/novox/mesh-catalog/pulls/7/merge"] : [], `${r.name}: what was merged`); + } + } + // A read: an open pull request answers what the forge would merge now; a forge that fails is the kind failed. + state.merged = false; state.head = REVIEWED; state.rules = [protectedMain()]; + const pull = await call("pull", { repository: "novox/mesh-catalog", number: 7 }); + assert.equal(pull.pull, undefined, "answered whole or not at all"); + assert.equal(pull.error?.startsWith("the forge did not say whether novox/mesh-catalog#7 merges now"), true, JSON.stringify(pull)); + assert.equal(pull["error-kind"], "failed", "a read the forge could not make answers the verb's error, never a guess"); + state.mergeable = true; + const open = await call("pull", { repository: "novox/mesh-catalog", number: 7 }); + state.mergeable = undefined; + assert.deepEqual([open.pull?.head, open.pull?.state, open.merging?.protected, open.merging?.required], + [REVIEWED, "open", true, ["mesh/merge-gate", "mesh/repo-check"]], JSON.stringify(open)); + assert.equal("body" in (open.pull ?? {}), false, "never its description"); + // Two writes of one head out of order (issue 456): a recheck's pending still in flight when the verdict is set through the + // seat; the verdict takes its turn after it and is what the head carries. + inFlight = new Promise((r) => (land = r)); + const recheck = gates.pending("novox", "mesh-catalog", REVIEWED); + const verdict = call("status", { id: "act-s", repository: "novox/mesh-catalog", commit: REVIEWED, state: "success", + context: "mesh/merge-gate", description: "passed" }); + await new Promise((r) => setImmediate(r)); + land(); + await recheck; + assert.equal((await verdict).performed, true); + assert.equal(landed.at(-1), "success passed", `the verdict lands last: ${JSON.stringify(landed)}`); + // A pull request the forge does not have is the kind not-found, so a caller never reads it out of the words. + const gone = await call("pull", { repository: "novox/mesh-catalog", number: 404 }); + assert.deepEqual([gone.pull, gone["error-kind"]], [undefined, "not-found"], JSON.stringify(gone)); +}); + +// Each verb of the seat git answers the contract's shape from what the forge gives, and an error naming why when the forge +// does not (design 47 §310, §312, §317): an empty list or an absent file only when the forge said so, a forge that failed +// the kind failed, a 404 the kind not-found, and an act whose id the pull request or the note already holds is held. +test("every read and act of the seat git answers its contract, and an error is an error, never an empty answer", async () => { + const { seatTools } = await import("../dist/seat.js"); + const { runAs } = await import("@novox/mesh-sdk/callers"); + const SHA = "a".repeat(40), OTHER = "b".repeat(40); + const gone = () => { throw new Error("Gitea API GET /repos/novox/app/pulls/9: 404 Not Found"); }; + const down = () => { throw new Error("Gitea API GET /repos/novox/app/x: 502 Bad Gateway"); }; + const pr = { number: 9, title: "t", state: "open", merged: false, head: "feat/x", head_sha: SHA, base: "main", user: "jochen", html_url: "http://f/9", mergeable: true }; + let comments: { id: number; body: string }[] = [], noteShown = "", added = 0, noteWritten = 0; + const forge: Record = { + getPullRequest: async (_o: string, _r: string, n: number) => (n === 9 ? pr : gone()), + pullOfCommit: async () => ({ ...pr, state: "closed", merged: true, merge_commit_sha: OTHER, merged_at: "2026-11-11T10:00:00Z" }), + appliedProtection: async () => ({ rule_name: "main", enable_status_check: true, status_check_contexts: ["mesh/check"] }), + getRepo: async () => ({ writes: true }), + openPulls: async () => [{ owner: "novox", repo: "app", number: 9, base: "main", head: "feat/x", head_sha: SHA, required: ["mesh/check"], checks: {} }], + tags: async () => ["go/v0.1.1", "go/v0.1.0", "v1"], + commitTrees: async () => ({ root: SHA, trees: { "": SHA } }), + getFile: async (_o: string, _r: string, path: string) => (path === "gone" ? (() => { throw new Error("Gitea API GET /repos/novox/app/contents/gone: 404 Not Found"); })() : path === "down" ? down() : { path, sha: OTHER, size: 3, content: "abc" }), + compare: async () => ({ total_commits: 2, commits: [{ sha: OTHER }, { sha: SHA }] }), + combinedStatus: async () => ({ sha: SHA, state: "success", statuses: [{ context: "mesh/check", status: "success", description: "ok" }] }), + listComments: async () => comments, + addComment: async (_o: string, _r: string, _n: number, body: string) => { added++; comments.push({ id: added, body }); }, + editComment: async () => {}, + }; + const run = async (_f: string, args: string[]) => args.includes("show") || args.includes("--ref") && !args.includes("append") + ? { stdout: noteShown, code: noteShown ? 0 : 1 } : (noteWritten++, { stdout: "", code: 0 }); + const seat = seatTools(forge as any, { run: run as any, container: "gitea", callers: async () => JSON.stringify({}), statuses: { status: async () => {} } }); + const everyone = JSON.stringify(Object.fromEntries(seat.map((t: any) => [t.name.slice(4), ["*"]]))); + const call = (verb: string, args: Record, over: Record = {}) => { + const t = seat.find((x: any) => x.name === `git.${verb}`)!; + const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", callers: async () => everyone, statuses: { status: async () => {} } }); + return runAs({ "mesh/caller": "ace.x" }, args, () => wrapped.find((x: any) => x.name === t.name)!.run(args)) as Promise; + }; + const rows: [string, string, Record, Record, (a: any) => unknown, unknown][] = [ + ["pull", "by number", { repository: "novox/app", number: 9 }, {}, (a) => [a.pull.head, a.pull.state, a.merging.mergeable, a.merging.required, a.merging.writes], [SHA, "open", true, ["mesh/check"], true]], + ["pull", "by merge commit", { repository: "novox/app", "merge-commit": OTHER }, {}, (a) => [a.pull.state, a.pull["merge-commit"], a.merging], ["merged", OTHER, undefined]], + ["pull", "unknown", { repository: "novox/app", number: 404 }, {}, (a) => [a.pull, a["error-kind"]], [undefined, "not-found"]], + ["pull", "forge down", { repository: "novox/app", number: 9 }, { getPullRequest: down }, (a) => [a.pull, a["error-kind"]], [undefined, "failed"]], + ["open-pulls", "one", {}, {}, (a) => [a.pulls.length, a.pulls[0].required], [1, ["mesh/check"]]], + ["open-pulls", "forge down", {}, { openPulls: down }, (a) => [a.pulls, a["error-kind"]], [undefined, "failed"]], + ["tags", "a page", { repository: "novox/app", prefix: "go/" }, {}, (a) => [a.tags, a.page, a.bound, a.more], [["go/v0.1.1", "go/v0.1.0"], 1, 50, undefined]], + ["tags", "forge down", { repository: "novox/app" }, { tags: down }, (a) => [a.tags, a["error-kind"]], [undefined, "failed"]], + ["trees", "root", { repository: "novox/app", commit: SHA }, {}, (a) => [a.commit, a.root], [SHA, SHA]], + ["trees", "forge down", { repository: "novox/app", commit: SHA }, { commitTrees: down }, (a) => [a.root, a["error-kind"]], [undefined, "failed"]], + ["file", "present", { repository: "novox/app", path: "a.txt" }, {}, (a) => [a.content, a.absent], ["abc", undefined]], + ["file", "the forge said absent", { repository: "novox/app", path: "gone" }, {}, (a) => [a.absent, a.error], [true, undefined]], + ["file", "forge down is no absent file", { repository: "novox/app", path: "down" }, {}, (a) => [a.absent, a["error-kind"]], [undefined, "failed"]], + ["contains", "ahead", { repository: "novox/app", commit: SHA, in: OTHER }, {}, (a) => [a.contains, a.ahead], [false, 2]], + ["contains", "forge down", { repository: "novox/app", commit: SHA, in: OTHER }, { compare: down }, (a) => [a.contains, a["error-kind"]], [undefined, "failed"]], + ["statuses", "newest of each", { repository: "novox/app", commit: SHA }, {}, (a) => [a.combined, a.statuses.map((s: any) => s.context)], ["success", ["mesh/check"]]], + ["statuses", "forge down", { repository: "novox/app", commit: SHA }, { combinedStatus: down }, (a) => [a.statuses, a["error-kind"]], [undefined, "failed"]], + ["status", "refused state", { id: "s1", repository: "novox/app", commit: SHA, context: "c", state: "bogus" }, {}, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], + ["comment", "first time", { id: "c1", repository: "novox/app", number: 9, text: "hi" }, {}, (a) => [a.performed, a.held, added], [true, undefined, 1]], + ["comment", "forge down", { id: "c2", repository: "novox/app", number: 9, text: "hi" }, { listComments: down }, (a) => [a.performed, a["error-kind"]], [undefined, "failed"]], + ["view", "first time", { id: "v1", repository: "novox/app", number: 9, text: "state" }, {}, (a) => [a.performed, a.held], [true, undefined]], + ["note", "a line", { id: "n1", repository: "novox/app", commit: SHA, line: "planned" }, {}, (a) => [a.performed, a.held, noteWritten], [true, undefined, 1]], + ]; + for (const [verb, name, args, over, pick, want] of rows) { + assert.deepEqual(pick(await call(verb, args, over)), want, `${verb}: ${name}`); + } + // The same comment id said again is held, and nothing is written again; so is a note line whose id the note holds. + const before = added; + const again = await call("comment", { id: "c1", repository: "novox/app", number: 9, text: "hi" }); + assert.deepEqual([again.performed, again.held, added], [true, true, before]); + const view = await call("view", { id: "v1", repository: "novox/app", number: 9, text: "state" }); + assert.deepEqual([view.performed, view.held], [true, true]); + noteShown = "planned [n1]"; noteWritten = 0; + const note = await call("note", { id: "n1", repository: "novox/app", commit: SHA, line: "worded differently" }); + assert.deepEqual([note.performed, note.held, noteWritten], [true, true, 0]); +}); diff --git a/modules/gitea/test/kept.test.ts b/modules/gitea/test/kept.test.ts index a526eb104..72144e862 100644 --- a/modules/gitea/test/kept.test.ts +++ b/modules/gitea/test/kept.test.ts @@ -16,8 +16,8 @@ test("the states the module announces are the states its manifest declares", asy test("the announcement has the layout mesh-sdk's go/announce reads: version 1, the module, the node, the states sorted by name", async () => { const { statesAnnouncement } = await import("../kept.ts"); - assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "findings", readable: true }] }); - assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "findings", readable: true }] }); + assert.deepEqual(statesAnnouncement("gitea", "ace"), { v: 1, module: "gitea", node: "ace", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "findings", readable: true }, { name: "first-views" }] }); + assert.deepEqual(statesAnnouncement("gitea", undefined), { v: 1, module: "gitea", states: [{ name: "announced" }, { name: "bound", readable: true }, { name: "findings", readable: true }, { name: "first-views" }] }); }); test("index.ts says the announcement as the event `states` before the tools are served", () => { diff --git a/modules/gitea/test/protection-full.test.ts b/modules/gitea/test/protection-full.test.ts index f07e22e76..5da3d6016 100644 --- a/modules/gitea/test/protection-full.test.ts +++ b/modules/gitea/test/protection-full.test.ts @@ -23,7 +23,6 @@ beforeEach(() => { enable_status_check: false, status_check_contexts: [], block_admin_merge_override: false, required_approvals: 0, enable_push_whitelist: false, push_whitelist_deploy_keys: false, enable_force_push: false, enable_force_push_allowlist: false, force_push_allowlist_deploy_keys: false, merge_whitelist_teams: [] }]; - source = {}; repoState = { full_name: "novox/de-spiegel", default_branch: "master", description: "", has_wiki: true }; sent = []; }); @@ -88,11 +87,7 @@ await new Promise((r) => server.listen(0, "127.0.0.1", () => r())); after(() => new Promise((r) => server.close(() => r()))); const { port } = server.address() as { port: number }; const url = `http://127.0.0.1:${port}`; -// The gate's source as the tools read it (mesh-delivery-gate.source): one key per trunk, each naming the repositories -// installed from it. A test sets it; empty is a source that names no trunk of de-spiegel. -let source: Record }> = {}; -const readSource = async () => source; -const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret"), readSource); +const tools = getGiteaTools(new GiteaClient(url, new ConfiguredToken("tools-token")), () => new GiteaAdmin(url, "mesh-admin", "secret")); const setRepo = (args: Record) => tools.find((t) => t.name === "gitea_repo_settings_set_full")!.run({ owner: "novox", repo: "de-spiegel", why: "L2", ...args }); const setFull = (args: Record) => @@ -227,24 +222,12 @@ test("a force-push onto a rule covering main is refused, nothing sent; on anothe assert.equal(tight.trunk_rule.holds, false, "trunk_rule is in the answer"); }); -// The operator, 2026-10-28 16:5x: the gates and trunk_rule apply to every trunk, the branches the mesh installs from, as -// the gate's source names them; "main" only when it names none, and said. -test("a trunk the gate's source names, beta, is gated and judged as main is; main is then no trunk of the repository", async () => { - source = { main: { heads: { "novox/okioki": "a1" } }, beta: { heads: { "novox/de-spiegel": "b2" } } }; - await assert.rejects(setFull({ branch: "beta", enable_force_push: true }), /force-push to a trunk is the operator's/); - await assert.rejects(setFull({ branch: "b*", force_push_allowlist_usernames: ["x"] }), /covers beta/); - assert.deepEqual(sent, []); - const main: any = await setFull({ enable_force_push: true }); - assert.deepEqual(main.trunks, { names: ["beta"], from: "mesh-delivery-gate.source" }); - assert.equal(main.trunk_rule.is_trunk, false); - const beta: any = await setFull({ branch: "beta", enable_push: false, enable_merge_whitelist: true, merge_whitelist_usernames: ["mesh-tools"], block_admin_merge_override: true }); - assert.equal(beta.trunk_rule.is_trunk, true); - assert.equal(beta.trunk_rule.holds, true, beta.trunk_rule.why); -}); - -test("a source that names no trunk of the repository falls back to main, and says so", async () => { +// The operator, 2026-10-28 16:5x: the gates and trunk_rule apply to every trunk, the branch the mesh installs from: main, +// the mesh's one trunk of every repository (design 47); no other module's state is read for it (ADR 0596 §2). +test("main is the repository's trunk, read from no other module, and another branch is none", async () => { const out: any = await setFull({ block_admin_merge_override: true }); - assert.equal(out.trunks.names[0], "main"); - assert.match(out.trunks.from, /names no trunk of novox\/de-spiegel: main assumed/); + assert.deepEqual(out.trunks, { names: ["main"], from: "the mesh's one trunk of every repository (design 47)" }); assert.equal(out.trunk_rule.is_trunk, true); + const beta: any = await setFull({ branch: "beta", enable_force_push: true }); + assert.equal(beta.trunk_rule.is_trunk, false); }); diff --git a/modules/gitea/test/token.test.ts b/modules/gitea/test/token.test.ts index 54fc995fc..4e9fd2af0 100644 --- a/modules/gitea/test/token.test.ts +++ b/modules/gitea/test/token.test.ts @@ -527,8 +527,7 @@ test("the tools register once there is a way to a token, and the first call mint withAdmin.tools.map((t) => t.name), [ "gitea_list_repos", "gitea_create_repo", "gitea_delete_repo", - "gitea_list_issues", "gitea_get_issue", "gitea_create_issue", "gitea_close_issue", "gitea_add_comment", - // gitea_merge_pull_request is served from the watcher's bundle, beside the merge check's one writer (ADR 0323). + "gitea_list_issues", "gitea_get_issue", "gitea_create_issue", "gitea_close_issue", "gitea_list_pull_requests", "gitea_get_pull_request", "gitea_create_pull_request", "gitea_close_pull_request", "gitea_reopen_pull_request", @@ -542,7 +541,7 @@ test("the tools register once there is a way to a token, and the first call mint "gitea_delete_branch", "gitea_branch_protection_get", "gitea_team_rights", "gitea_branch_protection_set", "gitea_branch_protection_set_full", "gitea_repo_settings_set_full", "gitea_default_branch_set", "gitea_pull_base_set", - "gitea_note_append", "gitea_delivery_view", "gitea_commit_statuses", "gitea_open_pulls", "gitea_commit_trees", "gitea_contains", + "gitea_commit_statuses", "gitea_open_pulls", "gitea_commit_trees", "gitea_contains", "gitea_list_labels", "gitea_create_label", "gitea_api", ], diff --git a/modules/gitea/test/tools-account.test.ts b/modules/gitea/test/tools-account.test.ts index d05ec24b6..f77b199ae 100644 --- a/modules/gitea/test/tools-account.test.ts +++ b/modules/gitea/test/tools-account.test.ts @@ -638,7 +638,7 @@ test("on the forge's events: a repository made is granted read at once, a known // The merging account writes every repository that builds a registered module, and no other (writes.ts; the operator, // 2026-10-26; issue 1945): car-hunter #2 was approved and its commit status refused 403, mesh-tools reading only there. -test("the merging account is given write on each repository the indexer's modules name, read back and said; the operator's other repositories stay read-only; the tools say both", async () => { +test("the merging account is given write on each repository the catalog half's modules name, read back and said; the operator's other repositories stay read-only; the tools say both", async () => { const env = await environment(); const client = fresh(env); await client.listRepos(); // the account and its team exist, as after any mint @@ -652,13 +652,17 @@ test("the merging account is given write on each repository the indexer's module const writes = new DeliveredWrites(GiteaAdmin.fromEnv(env), TOOLS_ACCOUNT, "novox", (l: string) => lines.push(l), async (type: string, body: unknown) => { said.push({ type, body }); }); const [unwritableTool, grantTool] = writeTools(() => writes); - await assert.rejects(grantTool.run({}), /not known yet/, "nothing is granted before the indexer's modules were read"); + await assert.rejects(grantTool.run({}), /not known yet/, "nothing is granted before the catalog half's first view is held"); assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, false, "the statuses' answer says the account cannot write"); - // 1945/AC1: the watch hands the indexer's modules; a user's repository a module builds from is made writable, once. - await writes.take({ key: "car-hunter", op: "put", value: { v: 1, repository: "jschoubben/car-hunter", commit: "c" }, current: true }); - await writes.take({ key: "gitea", op: "put", value: { v: 1, repository: "novox/hq", commit: "c" }, current: true }); - await writes.take({ key: "broken", op: "put", value: { v: 1, repository: "no-slash" }, current: true }); + // 1945/AC1: the first view hands the catalog half's modules; a user's repository a module builds from is made writable, + // once, and only once the first view is held (design 47: a part acts on nothing before its first view). + const viewing = forge.calls.length; + await writes.take({ key: "car-hunter", op: "put", value: { v: 1, repository: "jschoubben/car-hunter", commit: "c" } }); + await writes.take({ key: "gitea", op: "put", value: { v: 1, repository: "novox/hq", commit: "c" } }); + await writes.take({ key: "broken", op: "put", value: { v: 1, repository: "no-slash" } }); + assert.equal(forge.calls.length, viewing, "the first view's entries ask the forge nothing until it is held"); + await writes.viewed(); const from = forge.calls.length; await writes.take({ key: "car-hunter-2", op: "put", value: { v: 1, repository: "jschoubben/car-hunter" } }); assert.equal(forge.calls.length, from, "a repository already written asks nothing more"); @@ -668,7 +672,7 @@ test("the merging account is given write on each repository the indexer's module assert.equal(repo("novox/hq").collaborators.size, 0, "the organisation's repository is written through its team, no collaborator"); assert.equal((await client.getRepo("jschoubben", "car-hunter")).writes, true); assert.deepEqual(said, [{ type: WRITABLE_EVENT, body: { owner: "jschoubben", repo: "car-hunter", account: TOOLS_ACCOUNT } }]); - assert.match(lines.join("\n"), /mesh-tools was given write on jschoubben\/car-hunter, which builds the registered module car-hunter/); + assert.match(lines.join("\n"), /mesh-tools was given write on jschoubben\/car-hunter, which builds a registered module/); assert.match(lines.join("\n"), /broken names no repository/); // 1945/AC2: a delivered repository the forge refuses is said, in its words, by the listing and the grant; the operator's @@ -723,7 +727,7 @@ test("the merging account is given write on each repository the indexer's module await assert.rejects(GiteaAdmin.fromEnv(env).ensureMergerListed("jschoubben", "dotfiles", TOOLS_ACCOUNT), /mesh-tools was added to the merge whitelist of main's rule main of jschoubben\/dotfiles, but the forge holds \[jschoubben\]: it drops a login that cannot write/); userRules.delete("jschoubben/dotfiles"); - // A module gone from the indexer leaves its repository out of the delivered list. + // A module gone from the catalog half leaves its repository out of the delivered list. await writes.take({ key: "snake", op: "delete" }); assert.deepEqual(writes.delivered(), ["jschoubben/car-hunter", "novox/hq"]); }); diff --git a/modules/gitea/test/views.test.ts b/modules/gitea/test/views.test.ts index 3f53160c9..48ee8bcaf 100644 --- a/modules/gitea/test/views.test.ts +++ b/modules/gitea/test/views.test.ts @@ -1,62 +1,95 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { ViewFollower, findingKey, unansweredFinding, viewTools, type ViewAnswer, type ViewChange, type ViewEvent } from "../views.ts"; +import { viewWaitsFinding } from "@novox/mesh-sdk/views"; +import { CatalogView, REVISION_VERB, findingValue, revisionAnswer, unansweredFinding } from "../catalog.ts"; -// ADR 0615: the module asks the indexer's read tool once, then keeps its picture of `modules` from the indexer's views -// events; a gap asks once more; an owner that is down is asked again and never read instead. +// Design 47 *Every part takes a first view, then follows the events*: the forge module asks the seat mesh-catalog's verb +// `revision` for the catalog half (every page of one trunk revision), then hears its event `catalog-revision`. A candidate's +// revision moves nothing; a newer trunk revision is waited for, said, and asked; waited for past the bound, view-waits is +// written, and cleared once held; an owner that does not answer is asked again and raised past its bound. -const made = "2026-11-04T12:00:00Z"; -const answer = (revision: number, values: Record): ViewAnswer => - ({ v: 1, owner: "o", state: "modules", revision, created: made, values }); -const ev = (revision: number, key: string, value?: unknown, extra: Partial = {}): ViewEvent => - ({ v: 1, owner: "o", state: "modules", key, revision, created: made, value, deleted: value === undefined, ...extra }); +const entry = (module: string, repository: string) => ({ module, repository, dir: ".", manifest: "m", fingerprint: "f", verdict: "accepted" }); +const revisionOf = (revision: number, kind = "trunk") => ({ revision, kind, source: revision, entries: [] }); -test("a follower takes the first view, then the events, and asks again only on a gap", async () => { - const changes: ViewChange[] = []; +/** A holder of the seat whose trunk revisions are pages of entries, and who answers what is asked of it. */ +function aHolder() { + const asked: Record[] = []; + const trunk = new Map([ + [7, [[entry("a", "u/a"), entry("b", "u/b")], [entry("c", "u/c")]]], + [8, [[entry("a", "u/a"), entry("c", "u/c2")]]], + ]); + let newest = 7; + let down = 0; + return { + asked, + set newest(n: number) { newest = n; }, + set down(n: number) { down = n; }, + ask: async (verb: string, input: { after?: string; revision?: number }) => { + assert.equal(verb, REVISION_VERB); + asked.push({ ...input }); + if (down > 0) { down--; return { error: "no responders" }; } + const rev = input.revision ?? newest; + const pages = trunk.get(rev)!; + const at = input.after ? pages.findIndex((p) => p[0].module > input.after!) : 0; + return { revision: revisionOf(rev), entries: pages[at], ...(at < pages.length - 1 ? { next: pages[at].at(-1)!.module } : {}) }; + }, + }; +} + +test("the catalog view takes the trunk's first view page by page, then a newer trunk revision, and a candidate's moves nothing", async () => { + const holder = aHolder(); + const changes: string[] = []; + const view = new CatalogView(holder.ask, { pause: async () => {}, changed: (k, v, deleted) => { changes.push(deleted ? `-${k}` : `${k}=${(v as any).repository}`); } }); + await view.follower.start(); + assert.deepEqual(holder.asked, [{}, { after: "b", revision: 7 }], "the second page names the first page's revision"); + assert.deepEqual([view.held, view.follower.status().view, changes.sort()], [7, "7", ["a=u/a", "b=u/b", "c=u/c"]]); + const rows: { name: string; ev: Record; asks: number; held: number }[] = [ + { name: "a candidate's revision", ev: { v: 1, kind: "candidate", revision: 9, source: 8, group: "g", heads: "h" }, asks: 0, held: 7 }, + { name: "the revision it holds", ev: { v: 1, kind: "trunk", revision: 7, source: 7 }, asks: 0, held: 7 }, + { name: "a newer trunk revision", ev: { v: 1, kind: "trunk", revision: 8, source: 8 }, asks: 1, held: 8 }, + ]; + for (const r of rows) { + holder.newest = 8; + holder.asked.splice(0); + changes.splice(0); + await view.hear(r.ev as any); + await view.asked; + assert.deepEqual([holder.asked.length, view.held], [r.asks, r.held], r.name); + } + assert.deepEqual(changes.sort(), ["-b", "c=u/c2"], "the newer revision hands what changed"); + const s = view.follower.status(); + assert.deepEqual([s.view, s["waits-for"]], ["8", undefined], "it holds what it waited for"); + await assert.rejects(view.hear({ v: 1, kind: "trunk" } as any), /not a mesh-catalog.catalog-revision event/); + assert.throws(() => revisionAnswer({ error: "refused for gitea" }), /answered no revision: refused for gitea/); +}); + +test("a revision waited for past the bound writes view-waits, cleared once held; an unanswered first view is raised, then cleared", async () => { + const holder = aHolder(); const said: string[] = []; - let asks = 0; - let down = 2; - const raised: string[][] = []; - const f = new ViewFollower({ owner: "o", state: "modules", tool: "o.read", - ask: async () => { if (down-- > 0) throw new Error("no responders"); asks++; return answer(9, { a: { repository: "novox/a" }, b: { repository: "novox/b" } }); }, - take: (c) => { changes.push(c); }, raise: (_a, e) => { raised.push(["raised", e]); }, clear: () => { raised.push(["cleared", ""]); }, say: (l) => said.push(l), sleep: async () => {}, now: (() => { let t = 0; return () => (t += 200_000); })() }); - await f.start(); - assert.equal(f.ready, true); - assert.equal(f.values.size, 2); - assert.deepEqual(raised.map((r) => r.slice(0, 2)), [["raised", "no responders"], ["cleared", ""]]); - assert.ok(said.some((l) => l.includes("no first view of o's modules") && l.includes("o.read")), said.join("|")); - await f.hear(ev(8, "a", { repository: "old" })); // already in the picture - await f.hear(ev(10, "c", { repository: "novox/c" })); - await f.hear(ev(11, "a")); // deleted - assert.deepEqual([f.revision, f.values.has("a"), f.values.has("c"), asks], [11, false, true, 1]); - assert.deepEqual(changes.slice(-2).map((c) => [c.key, c.op]), [["c", "put"], ["a", "delete"]]); - await f.hear(ev(13, "d", { repository: "x" })); // revision 12 missed: a gap - assert.equal(asks, 2); - await f.hear(ev(1, "e", { repository: "x" }, { created: "2026-11-04T13:00:00Z" })); // the state made again: a gap - assert.equal(asks, 3); - await f.hear(ev(10, "f", undefined, { gap: true })); // the next revision, too large to carry: a gap - assert.equal(asks, 4); - await f.hear({ ...ev(10, "g", { repository: "x" }), state: "other" }); // another state - assert.equal(f.values.has("g"), false); -}); - -test("the reader's two tools say where the first view stands and ask it again by hand", async () => { - let asks = 0; - const acts: Record[] = []; - const follower = new ViewFollower({ owner: "o", state: "modules", tool: "o.read", ask: async () => { asks++; return answer(5, { a: 1 }); }, - take: () => {}, sleep: async () => {} }); - const [first, again] = viewTools(() => ({ follower, owner: "o", tool: "o.read", state: "modules" }), async (act) => { acts.push(act); }); - assert.deepEqual(await first.run({}), { views: [{ owner: "o", tool: "o.read", state: "modules", revision: 0, created: "", keys: 0, answered: false, asks: 0, "by-hand": 0 }] }); - const after = (await again.run({})) as { views: { revision: number; "by-hand": number; asks: number }[] }; - assert.deepEqual([after.views[0].revision, after.views[0]["by-hand"], asks, acts.length], [5, 1, 1, 1]); - await assert.rejects(viewTools(() => ({ follower, owner: "o", tool: "o.read", state: "modules" }), async () => { throw new Error("no holder"); })[1].run({}), /not recorded/); - assert.deepEqual(await viewTools(() => undefined, async () => {})[0].run({}), { views: [] }); -}); - -test("the finding of a first view unanswered is the one mesh-sdk writes: key, kind, subject and the words", () => { - const f = unansweredFinding("gitea", "mesh-delivery-indexer", "modules", "mesh-delivery-indexer.indexer_read_modules", new Date("2026-11-04T12:00:00Z"), "no responders", new Date("2026-11-04T12:03:00Z")); - assert.equal(f.key, "module.gitea.mesh-delivery-indexer.modules.first-view-unanswered"); - assert.equal(findingKey("module", "a.b", "k"), "module.a.b.k"); - assert.equal(f.value.severity, "warning"); - assert.match(String(f.value.summary), /gitea has no first view of mesh-delivery-indexer's modules: .* \(no responders\)/); + let bound = () => {}; + let clock = 0; + const view = new CatalogView(holder.ask, { + pause: () => new Promise((r) => setImmediate(r)), bound: 1000, now: () => new Date(Date.parse("2026-11-09T22:00:00Z") + (clock += 600)), + waitsBound: () => new Promise((r) => (bound = r)), + raise: (s) => { said.push(`raised ${findingValue(unansweredFinding("gitea", s, new Date())).key}`); }, + clear: () => { said.push("cleared"); }, + raiseWaits: (s) => { said.push(`waits ${findingValue(viewWaitsFinding("gitea", s, new Date())).key}`); }, + clearWaits: () => { said.push("waits cleared"); }, + }); + holder.down = 3; + await view.follower.start(); + assert.deepEqual(said, ["raised module.gitea.mesh-catalog.catalog-revisions.first-view-unanswered", "cleared"]); + said.splice(0); + holder.newest = 8; + holder.down = 100; // the holder is away when the newer revision is said + const heard = view.hear({ v: 1, kind: "trunk", revision: 8, source: 8 } as any); + await heard; + assert.equal(view.follower.status()["waits-for"], "8", "it says what it waits for"); + bound(); + await new Promise((r) => setImmediate(r)); + assert.deepEqual(said.filter((l) => l.startsWith("waits")), ["waits module.gitea.mesh-catalog.catalog-revisions.view-waits"]); + holder.down = 0; + await view.follower.gap("the holder is back"); + await view.asked; + assert.deepEqual([view.held, view.follower.status()["waits-for"], said.includes("waits cleared")], [8, undefined, true]); }); diff --git a/modules/gitea/tools/index.ts b/modules/gitea/tools/index.ts index 94736c81c..8c7e73e2b 100644 --- a/modules/gitea/tools/index.ts +++ b/modules/gitea/tools/index.ts @@ -12,9 +12,7 @@ import { registerModuleTools, type ToolDefinition } from "@novox/mesh-sdk/tools" import { emit } from "@novox/mesh-sdk/events"; import { GiteaAdmin, GiteaClient, type BranchProtection } from "../client.js"; import { readTeamRights } from "../teamrights.js"; -import { state } from "@novox/mesh-sdk/state"; -import { PROTECTION_FIELDS, REPO_FIELDS, alsoChanged, forcePushToTrunk, fullProtectionBody, sourceFromBus, trunksOf, type SourceRead, givenFields, trunkRule, unapplied } from "../protectfull.js"; -import { appendNote, run, viewBody, viewComment } from "../delivery.js"; +import { PROTECTION_FIELDS, REPO_FIELDS, alsoChanged, forcePushToTrunk, fullProtectionBody, trunksOf, givenFields, trunkRule, unapplied } from "../protectfull.js"; import { containedFrom, isCommit, mergeableUnder, newestByContext } from "../checks.js"; import { MERGE_METHODS, Refused, checkName, judgeApiCall, judgeContentsWrite, judgeDelete, judgeProtectionChange, pathSegments, routedPath, sentQuery, @@ -24,10 +22,6 @@ import { /** A head named for a merge: at least 12 hex characters of its sha (novox/hq issue 381), up to a sha-256 commit's 64. */ const REVIEWED_HEAD = /^[0-9a-f]{12,64}$/; -/** The forge's container, where its repositories and git are: the note is written there (novox/hq ADR 0239). Its name - * is the manifest's container's; no variable names it (a MESH_* word nothing sets is refused, checks/environment). */ -const forgeContainer = "gitea"; - /** A protection rule as a person reads it: what it guards, not every field the forge keeps. */ export function summarised(p: BranchProtection) { return { @@ -91,13 +85,14 @@ function filesDirs(raw: unknown): string[] | undefined { return raw as string[]; } -export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin, source: SourceRead = sourceFromBus(state)): ToolDefinition[] { - return namesChecked(allGiteaTools(gitea, admin, source)); +export function getGiteaTools(gitea: GiteaClient, admin?: () => GiteaAdmin): ToolDefinition[] { + // The seat git's verbs are served by the events entrypoint (index.ts), beside the head's one writer of its statuses. + return namesChecked(allGiteaTools(gitea, admin)); } -// gitea_merge_pull_request is served from the watcher's bundle (merge.ts, index.ts), beside the merge check's one -// writer (novox/hq issue 456, ADR 0323). -function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined, source: SourceRead): ToolDefinition[] { +// A merge, a status, a comment, a delivery's view and its note are the seat git's acts (seat.ts), asked by the forge +// bridge alone: no tool of the module's own does them (novox/hq design 47 *The seat git*). +function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined): ToolDefinition[] { return [ // ---- Repositories ---- { @@ -233,19 +228,6 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined issue: await gitea.setIssueState(String(args.owner), String(args.repo), Number(args.number), "closed"), }), }, - { - name: "gitea_add_comment", - description: "Add a comment to an issue or pull request.", - input: { - owner: { type: "string", description: "the repository owner" }, - repo: { type: "string", description: "the repository name" }, - number: { type: "number", description: "the issue or PR number" }, - body: { type: "string", description: "the comment body (markdown)" }, - }, - run: async (args) => ({ - comment: await gitea.addComment(String(args.owner), String(args.repo), Number(args.number), String(args.body)), - }), - }, // ---- Pull requests ---- { @@ -502,7 +484,7 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined "enable_push_whitelist and push_whitelist_deploy_keys need enable_push: true in the same call, the force-push allowlist " + "flags enable_force_push: true (the forge ignores them otherwise). The rule is read back after the write: a field the " + "forge did not apply fails the call, naming it; also_changed names what the forge changed beside the call. Generic: it " + - "tightens or loosens as told (the operator, 2026-10-28), so a reason (why) is required and logged; but a call that lets a force-push onto a rule covering a trunk is refused, nothing sent, until the module can ask the operator's Approve for it. A repository's trunks, the branches the mesh installs from, are read from mesh-delivery-gate's source at the call (main when it names none, said in trunks.from). As the forge's " + + "tightens or loosens as told (the operator, 2026-10-28), so a reason (why) is required and logged; but a call that lets a force-push onto a rule covering a trunk is refused, nothing sent, until the module can ask the operator's Approve for it. A repository's trunk, the branch the mesh installs from, is main, the mesh's one trunk of every repository (said in trunks.from). As the forge's " + "administrator. Answers the rule before and after, the repository's trunks, and trunk_rule: whether the branch is a trunk (is_trunk) and whether the controller now registers a module from " + "this branch (judgedRule, mesh-control build_source.go: no push, a merge whitelist naming mesh-tools and no " + "agent-, no administrator override) and why not.", @@ -524,7 +506,7 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined const given = givenFields(args); if (!given.length) throw new Error(`name at least one field to set: ${Object.keys(PROTECTION_FIELDS).join(", ")}`); const body = fullProtectionBody(args, given); - const trunks = await trunksOf(owner, repo, source); + const trunks = trunksOf(); const operators = forcePushToTrunk(branch, body, trunks.names); if (operators) throw new Refused(operators); const forge = admin ? admin() : gitea; @@ -640,40 +622,7 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined }, }, - // ---- A delivery's note, view and statuses (novox/hq ADR 0239), asked by mesh-delivery ---- - { - name: "gitea_note_append", - description: "Append one line to a commit's git note under refs/notes/ (mesh-plan for a delivery), written in the forge's own repository as the forge's own account; a line the note already holds is not added again. `git log --notes=mesh-plan` shows it.", - input: { - owner: { type: "string", description: "the repository owner" }, - repo: { type: "string", description: "the repository name" }, - sha: { type: "string", description: "the commit" }, - ref: { type: "string", description: "the notes ref's name (default mesh-plan)" }, - line: { type: "string", description: "the line, one line" }, - }, - run: async (args) => { - const said = await appendNote(run, forgeContainer, String(args.owner), String(args.repo), String(args.sha), - String(args.ref ?? "mesh-plan") || "mesh-plan", String(args.line ?? "")); - return { commit: String(args.sha), ...said }; - }, - }, - { - name: "gitea_delivery_view", - description: "Keep a delivery's view on its pull request: one comment, marked as the delivery's, created the first time and edited in place after — the page every status of the delivery links to.", - input: { - owner: { type: "string", description: "the repository owner" }, - repo: { type: "string", description: "the repository name" }, - number: { type: "number", description: "the pull request's number" }, - body: { type: "string", description: "the view, markdown" }, - }, - run: async (args) => { - const owner = String(args.owner), repo = String(args.repo), number = Number(args.number); - const body = viewBody(String(args.body ?? "")); - const existing = viewComment(await gitea.listComments(owner, repo, number)); - if (existing) return { edited: await gitea.editComment(owner, repo, existing.id, body) }; - return { created: await gitea.addComment(owner, repo, number, body) }; - }, - }, + // ---- A commit's statuses (a delivery's note, view and statuses are the seat git's acts, seat.ts) ---- { name: "gitea_commit_statuses", description: "Read a commit's statuses — or a pull request's head's — as the forge keeps them: the newest of each context (mesh/check, mesh/delivery, …) with its state, description, link, when it was set and by whom; and whether the branch it merges into would let it merge, every status that branch's protection requires being a success; for a pull request, also the forge's own mergeable (false while it is in conflict with its base or the forge is still working that out); and merge.writes, whether the mesh's merging account may write the repository at all (false: its statuses and its merge are refused until gitea_grant_write). Reads only.", @@ -768,8 +717,6 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined return { commit, in: within, contains, ahead }; }, }, - // gitea_commit_status is served from the watcher's bundle (index.ts), beside the merge check's one writer - // (novox/hq issue 456). // ---- Labels ---- { @@ -805,7 +752,7 @@ function allGiteaTools(gitea: GiteaClient, admin: (() => GiteaAdmin) | undefined name: "gitea_api", description: "Make an authenticated Gitea API call for an endpoint without a dedicated tool. Path is relative to /api/v1. " + "Reads anything but the forge's administration; writes only what an author writes — issues, comments, labels, " + - "pull requests (not their merge: gitea_merge_pull_request), branches, and files on a branch that is not the " + + "pull requests (not their merge: the seat git's merge-pull), branches, and files on a branch that is not the " + "default branch and that no protection rule names. Administration, protection, settings, collaborators, teams, " + "hooks, keys, tokens and statuses are refused (novox/hq ADR 0266).", input: { diff --git a/modules/gitea/views.ts b/modules/gitea/views.ts deleted file mode 100644 index dbcc7d6a4..000000000 --- a/modules/gitea/views.ts +++ /dev/null @@ -1,241 +0,0 @@ -// A reader's picture of one state of the indexer or the assigner (novox/hq ADR 0615 §1, §4 to §6; design 25 § 12.2): asked once -// of the owner's read tool, then kept by the owner's views events alone. It is the TypeScript form of the Go reader -// (mesh-delivery sdk/views Follower) and follows the same rules: -// -// - at its start it asks the tool until it is answered, waiting longer between asks (up to a minute), and past two minutes -// says so naming the owner and the tool; it never reads the owner's state instead; -// - an event of another `created` is a gap (the owner's state was made again); one at or below the revision held is already -// in the picture; one past the next revision is a gap; one with `gap: true` (a value too large to carry) is a gap; -// - a gap asks the tool once more, said in the log; -// - events heard while an ask runs are taken again after its answer, so none is lost. - -/** The answer of the owner's read tool with `values`: one page of the state, at the owner's revision. */ -export interface ViewAnswer { - v: number; - owner: string; - state: string; - revision: number; - created: string; - values: Record; - next?: string; -} - -/** One change of one key, said on the owner's views family. */ -export interface ViewEvent { - v: number; - owner: string; - state: string; - key: string; - revision: number; - created: string; - deleted?: boolean; - value?: unknown; - gap?: boolean; -} - -/** One change of the picture, in the form the module's watch handed before. */ -export interface ViewChange { - key: string; - op: "put" | "delete"; - value?: unknown; - revision?: number; - current?: boolean; -} - -export interface ViewFollowerOptions { - owner: string; - state: string; - /** The owner's read tool as the manifest invokes it, for the log. */ - tool: string; - /** Asks the tool for the page after `after` (none: the first). */ - ask: (after?: string) => Promise; - take: (change: ViewChange) => void | Promise; - say?: (line: string) => void; - sleep?: (ms: number) => Promise; - now?: () => number; - /** How long it asks before it says so; two minutes. */ - boundMs?: number; - /** Told once when the first view stays unanswered past the bound (the module writes its finding), and once when it is answered after. */ - raise?: (asked: Date, error: string) => void | Promise; - clear?: () => void | Promise; -} - -export class ViewFollower { - readonly values = new Map(); - revision = 0; - created = ""; - ready = false; - asks = 0; - byHand = 0; - private asking = false; - private during: ViewEvent[] = []; - - private readonly o: ViewFollowerOptions; - - constructor(o: ViewFollowerOptions) { - this.o = o; - } - - private say(line: string): void { - (this.o.say ?? ((l) => console.error(`[gitea] ${l}`)))(line); - } - - /** Asks until answered: each failed ask waits, doubling to a minute, and past the bound says what waits. */ - async start(): Promise { - await this.askUntil("start"); - } - - /** Asks once more by hand, until answered (a hand act, counted). */ - async again(why: string): Promise { - this.byHand++; - await this.askUntil(`by hand: ${why}`); - } - - private async askUntil(why: string): Promise { - if (this.asking) return; // one ask at a time; the one running brings the picture to its newest - this.asking = true; - this.during = []; - const sleep = this.o.sleep ?? ((ms: number) => new Promise((r) => setTimeout(r, ms))); - const now = this.o.now ?? Date.now; - const first = now(); - let said = false; - try { - for (let wait = 5000; ; wait = Math.min(wait * 2, 60_000)) { - try { - await this.askOnce(); - if (said) { - this.say(`the first view ${this.o.tool} of ${this.o.owner}'s ${this.o.state} is answered`); - await this.o.clear?.(); - } - return; - } catch (err) { - if (!said && now() - first >= (this.o.boundMs ?? 120_000)) { - said = true; - await this.o.raise?.(new Date(first), err instanceof Error ? err.message : String(err)); - this.say(`no first view of ${this.o.owner}'s ${this.o.state}: ${this.o.tool} has not answered since ${new Date(first).toISOString()} ` + - `(${err instanceof Error ? err.message : String(err)}); it keeps asking and reads the state of nobody (${why})`); - } - await sleep(wait); - } - } - } finally { - this.asking = false; - } - } - - private async askOnce(): Promise { - this.asks++; - const values = new Map(); - let firstPage: ViewAnswer | undefined; - for (let after: string | undefined; ;) { - const a = await this.o.ask(after); - firstPage ??= a; - for (const [k, v] of Object.entries(a.values ?? {})) values.set(k, v); - if (!a.next || a.next === after) break; - after = a.next; - } - const old = new Map(this.values); - this.values.clear(); - for (const [k, v] of values) this.values.set(k, v); - this.revision = firstPage.revision; - this.created = firstPage.created; - this.ready = true; - const during = this.during.sort((a, b) => a.revision - b.revision); - this.during = []; - for (const ev of during) { - if (ev.created === this.created && ev.revision === this.revision + 1 && !ev.gap) { - if (ev.deleted) this.values.delete(ev.key); - else this.values.set(ev.key, ev.value); - this.revision = ev.revision; - } - } - for (const [k, v] of this.values) { - if (!old.has(k) || JSON.stringify(old.get(k)) !== JSON.stringify(v)) await this.o.take({ key: k, op: "put", value: v, current: true }); - } - for (const k of old.keys()) if (!this.values.has(k)) await this.o.take({ key: k, op: "delete", current: true }); - } - - /** One event of the owner's views family: taken into the picture, or a gap that asks the tool once more. */ - async hear(ev: ViewEvent): Promise { - if (ev.state !== this.o.state) return; - if (this.asking) this.during.push(ev); - if (!this.ready) return; // the first view holds it - let why = ""; - if (ev.created !== this.created) why = `${this.o.owner}'s ${this.o.state} was made again`; - else if (ev.revision <= this.revision) return; - else if (ev.revision > this.revision + 1) why = `${this.o.owner}'s ${this.o.state} at revision ${ev.revision}, the picture holds ${this.revision}`; - else if (ev.gap) why = `${this.o.owner}'s ${this.o.state}.${ev.key} was too large to carry`; - if (why) { - this.say(`a gap in ${this.o.state} (${why}): its first view ${this.o.tool} is asked once more`); - await this.askUntil(`a gap: ${why}`); - return; - } - if (ev.deleted) this.values.delete(ev.key); - else this.values.set(ev.key, ev.value); - this.revision = ev.revision; - await this.o.take({ key: ev.key, op: ev.deleted ? "delete" : "put", value: ev.value, revision: ev.revision, current: false }); - } -} - -type Tool = { name: string; description: string; input: Record; run: (args: Readonly>) => Promise }; - -/** Where a first view stands (design 25 § 12.2): the owner and its tool, the revision the picture holds, how many asks. (r) */ -export function status(f: ViewFollower, owner: string, tool: string, state: string): Record { - return { owner, tool, state, revision: f.revision, created: f.created, keys: f.values.size, answered: f.ready, asks: f.asks, "by-hand": f.byHand }; -} - -/** The reader's two tools: `gitea_first_views` says where its first views stand, `gitea_first_view_again` asks one once more by hand. */ -export function viewTools(view: () => { follower: ViewFollower; owner: string; tool: string; state: string } | undefined, - record: (act: Record) => Promise): Tool[] { - return [ - { - name: "gitea_first_views", - description: "Where each first view this module keeps stands (novox/hq ADR 0615): the owner and its tool, the revision the picture holds, how many asks (by hand among them). (r)", - input: {}, - run: async () => { - const v = view(); - return { views: v ? [status(v.follower, v.owner, v.tool, v.state)] : [] }; - }, - }, - { - name: "gitea_first_view_again", - description: "Ask the indexer's modules again by hand: a hand act, for a picture the operator doubts. Answers where it stands after. (a)", - input: {}, - run: async () => { - const v = view(); - if (!v) throw new Error("the module follows no first view yet"); - // A hand act, recorded through the seat mesh-hand-acts before it is done; one that cannot be recorded asks nothing. - try { - await record({ what: "asked a first view again by hand (gitea)", why: "the operator asked", cause: "first-view-again", verb: "first-view-again" }); - } catch (err) { - throw new Error(`the hand act was not recorded through mesh-hand-acts, so nothing was asked again: ${err instanceof Error ? err.message : String(err)}`); - } - await v.follower.again("the operator asked"); - return { views: [status(v.follower, v.owner, v.tool, v.state)] }; - }, - }, - ]; -} - -/** The key of a finding about a module: scope, the id's dotted parts, and the kind, each part made safe (mesh-sdk health.Key). */ -export function findingKey(scope: string, id: string, kind: string): string { - return [scope, ...id.split("."), kind].map((p) => p.trim().replace(/[^A-Za-z0-9_=/-]/g, "_") || "_").join("."); -} - -export const UNANSWERED_KIND = "first-view-unanswered"; - -/** A reader's finding of a first view unanswered past its bound, as mesh-sdk views.UnansweredFinding writes it (ADR 0615 §6): the - * condition keeper raises the condition from the module's state `findings`. */ -export function unansweredFinding(reader: string, owner: string, state: string, tool: string, asked: Date, error: string, at: Date): - { key: string; value: Record } { - const id = `${reader}.${owner}.${state}`; - const key = findingKey("module", id, UNANSWERED_KIND); - const summary = `${reader} has no first view of ${owner}'s ${state}: ${tool} has not answered since ${asked.toISOString()} (${error.trim()})`; - return { key, value: { - v: 1, key, kind: UNANSWERED_KIND, subject: { scope: "module", id }, severity: "warning", summary, - headline: `${reader} waits for ${owner}`, - explanation: `${reader} reads ${owner}'s ${state} only through ${tool}, and keeps asking it; it never reads the state itself. The answer clears this. ` + - `Look at ${owner}'s runtime of its own: is it running and connected?`, - resolved: "", evidence: [{ at: at.toISOString(), said: summary }], source: `${reader}.${UNANSWERED_KIND}`, observer: reader, at: at.toISOString(), - } }; -} diff --git a/modules/gitea/writes.ts b/modules/gitea/writes.ts index 029e4d8ba..2bd8aa980 100644 --- a/modules/gitea/writes.ts +++ b/modules/gitea/writes.ts @@ -3,15 +3,16 @@ // 1945). car-hunter #2 (a repository of a user, not of the organisation novox) was approved, and mesh-delivery's first // act, its commit status, was refused 403: the tools' account was only a collaborator that reads there (readall.ts). // -// **Which account.** The tools' account, `mesh-tools`: the gitea module's own tools set every commit status and merge -// (gitea_commit_status, gitea_merge_pull_request) with its token. The build modules' `nox-builder` and the machines' +// **Which account.** The tools' account, `mesh-tools`: the seat git's acts (seat.ts) set every commit status and merge +// with its token. The build modules' `nox-builder` and the machines' // `nox-agent` only fetch, and keep read (readall.ts). // -// **Which repositories: the indexer's word, read over the bus.** mesh-delivery-indexer keeps every registered module in its -// state `modules` (mesh-sdk go/delivery ModulesBucket), each value naming the repository it builds from. This module reads -// that state through the indexer's read tool (its manifest's `invokes`) and then its views events (`consumes`; ADR 0615): the -// first view hands the latest value of every key when the module starts, and then every change, so a repository is granted -// at the module's start and when a module of it is first registered. Nothing polls and the state is not read. A repository of the organisation needs nothing: the organisation's team writes it (tools-account.ts). Every other +// **Which repositories: the catalog half's word, read over the bus.** The seat mesh-catalog's trunk revision holds every +// registered module's entry, each naming the repository it builds from. This module asks it of the seat's verb `revision` +// (its manifest's `invokes`) and then hears the seat's event `catalog-revision` (`consumes`; catalog.ts, design 47): the +// first view hands every entry when the module starts, and each newer trunk revision what changed, so a repository is +// granted at the module's start and when a module of it is first registered. A candidate's revision moves nothing, and +// before the first view nothing is granted. Nothing polls and no holder's state is read. A repository of the organisation needs nothing: the organisation's team writes it (tools-account.ts). Every other // repository of a user (the operator's dotfiles, say) is left as it is: read only. // // **Every grant is read back** from the forge (its permission for the account), and said on the bus as @@ -24,19 +25,15 @@ // set, approvals the gate does not give) is changed by nobody but said: both tools list such a repository as writable, but // blocked, with why (merge_blocked). // -// Not taken away: a repository whose last module goes from the indexer keeps its collaborator until a hand removes it; +// Not taken away: a repository whose last module leaves the catalog keeps its collaborator until a hand removes it; // gitea_unwritable_repos and gitea_grant_write say the delivered list as it is now. import type { ForgeRepository } from "./client.js"; import type { BranchProtection } from "./protection.js"; import { CHECK_CONTEXT, CHECK_STATUS, REPO_CHECK_CONTEXT } from "./pulls.js"; -/** The indexer, whose state `modules` this module follows: one key per registered module, its value naming the repository. - * It asks the indexer's read tool once and then hears its views events (ADR 0615), never reading the state. */ -export const INDEXER_OWNER = "mesh-delivery-indexer"; -export const INDEXER_MODULES_TOOL = `${INDEXER_OWNER}.indexer_read_modules`; -export const INDEXER_MODULES_EVENTS = `${INDEXER_OWNER}.views.modules`; -const INDEXER_MODULES = `${INDEXER_OWNER}.modules`; +/** What the view is called in what this module says: the catalog half, through the seat mesh-catalog. */ +const CATALOG_HALF = "the catalog half (the seat mesh-catalog's revision)"; /** The event said once an account writes a repository it could not write before. */ export const WRITABLE_EVENT = "repo.writable"; @@ -75,18 +72,16 @@ export function mergeBlocks(rule: BranchProtection | null, account: string): str const blockedWhy = (blocks: string[]): string => `writable, but main's rule blocks the mesh's merge: ${blocks.join("; ")}`; -/** One change of the indexer's modules state, as the SDK's state watch hands it. */ +/** One change of the catalog half's entries: a module, put with its entry or deleted. */ export interface ModulesChange { key: string; op: "put" | "delete"; value?: unknown; - revision?: number; - current?: boolean; } export interface Written { account: string; - /** The repositories that build a registered module, as the indexer says now. */ + /** The repositories that build a registered module, as the catalog half says now. */ delivered: string[]; /** Made writable now, each read back from the forge. */ granted: string[]; @@ -113,9 +108,9 @@ function split(full: string): [string, string] { } export class DeliveredWrites { - /** Module to the repository it builds from, as the indexer's state says. */ + /** Module to the repository it builds from, as the catalog half says. */ readonly modules = new Map(); - /** Whether the watch has handed the indexer's state at all: until then, which repositories are delivered is not known. */ + /** Whether the first view of the catalog half is held: until then, which repositories are delivered is not known. */ heard = false; /** Repositories seen written by the account in this process: a module registered again asks nothing more. */ readonly writable = new Set(); @@ -129,9 +124,23 @@ export class DeliveredWrites { return [...new Set(this.modules.values())].sort(); } - /** One change of the indexer's modules: a repository newly delivered is made writable now. Never throws. */ - async take(change: ModulesChange): Promise { + /** The first view is held, its entries taken: from now on which repositories are delivered is known, even when the + * catalog lists none, and every one the account cannot write yet is granted now, in one pass. Never throws. */ + async viewed(): Promise { + if (this.heard) return; this.heard = true; + try { + const r = await this.grant(); + for (const g of r.granted) this.log(`${this.account} was given write on ${g}, which builds a registered module`); + for (const x of r.refused) this.log(`${this.account} could not be given write on ${x.repository} — ${x.why}`); + } catch (err) { + this.log(`which repositories ${this.account} writes could not be settled at the first view — ${said(err)}`); + } + } + + /** One change of the catalog half: a repository newly delivered is made writable now. Never throws. Before the first + * view is held (viewed) nothing is granted: the change is kept, and the first view's changes follow it. */ + async take(change: ModulesChange): Promise { if (change.op === "delete") { this.modules.delete(change.key); return; @@ -140,11 +149,11 @@ export class DeliveredWrites { try { split(repository); } catch { - this.log(`${INDEXER_MODULES} ${change.key} names no repository (${JSON.stringify(repository)}): no write is granted for it`); + this.log(`${CATALOG_HALF}: ${change.key} names no repository (${JSON.stringify(repository)}): no write is granted for it`); return; } this.modules.set(change.key, repository); - if (this.writable.has(repository)) return; + if (!this.heard || this.writable.has(repository)) return; try { const r = await this.grant([repository]); for (const g of r.granted) this.log(`${this.account} was given write on ${g}, which builds the registered module ${change.key}`); @@ -158,8 +167,8 @@ export class DeliveredWrites { * delivered); with none named, also say which repositories it cannot write are left so, and why. */ async grant(only?: string[]): Promise { if (!this.heard) { - throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read ` + - "(the watch is not granted or not answered); nothing is granted"); + throw new Error(`which repositories build a registered module is not known yet: ${CATALOG_HALF} has no first view ` + + "held (not answered, or waited for); nothing is granted"); } const delivered = this.delivered(); const result: Written = { account: this.account, delivered, granted: [], already: [], merge_whitelist: [], merge_blocked: [], refused: [], left_read_only: [] }; @@ -169,7 +178,7 @@ export class DeliveredWrites { split(full); if (delivered.includes(full)) targets.push(full); else result.left_read_only.push({ repository: full, permission: "unchanged", - why: "builds no registered module (the indexer names none from it), so the account is given no write there" }); + why: "builds no registered module (the catalog half names none from it), so the account is given no write there" }); } for (const full of targets) { const [owner, name] = split(full); @@ -248,7 +257,7 @@ export class DeliveredWrites { } if (WRITES.has(permission)) continue; result.left_read_only.push({ repository: repo.full_name, permission, - why: "builds no registered module (the indexer names none from it), so the account is given no write there" }); + why: "builds no registered module (the catalog half names none from it), so the account is given no write there" }); } } return result; @@ -257,7 +266,7 @@ export class DeliveredWrites { /** Every delivered repository the account cannot write, as the forge says now, and why. */ async unwritable(): Promise<{ account: string; delivered: number; unwritable: { repository: string; permission: string; why: string }[]; merge_blocked: { repository: string; why: string }[] }> { - if (!this.heard) throw new Error(`which repositories build a registered module is not known yet: ${INDEXER_MODULES} has not been read`); + if (!this.heard) throw new Error(`which repositories build a registered module is not known yet: ${CATALOG_HALF} has no first view held`); const delivered = this.delivered(); const out: { repository: string; permission: string; why: string }[] = []; const blocked: { repository: string; why: string }[] = []; @@ -295,7 +304,7 @@ export function writeTools(writes: () => DeliveredWrites): Tool[] { return [ { name: "gitea_unwritable_repos", - description: "Every repository that builds a registered module (the indexer's modules, read over the bus) on which " + + description: "Every repository that builds a registered module (the catalog half's trunk revision, read over the bus) on which " + "the mesh's merging forge account (mesh-tools, which sets commit statuses and merges) cannot write, with the " + "permission the forge gives it there and why: such a pull request can be checked but never merged, and the " + "delivery gate does not ask its Approve. Empty when it writes every one. Its merge_blocked lists each repository of " + -- 2.54.0 From 0bf636826a5758ba70b9d8cde1efd45d807f2b4c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:32:33 +0100 Subject: [PATCH 012/243] GIT4: the seat git's branch answers a branch's head and whether a rule refuses a direct push and admits only merges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §5, §111, §310 branch Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/module.json | 3 ++- modules/gitea/seat.ts | 17 +++++++++++++++-- modules/gitea/test/guard.test.ts | 4 ++++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 7d95544e4..9af9a0c41 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -170,6 +170,7 @@ "scope": "mesh", "serves": [ "pull", + "branch", "open-pulls", "tags", "trees", @@ -251,7 +252,7 @@ "type": "string", "trusted": true, "runtime": true, - "default": "{\"publish\":[\"mesh-build-go\",\"mesh-build-node\"],\"publish-part\":[\"mesh-build-go\",\"mesh-build-node\"],\"remove\":[\"mesh-delivery-builder\"],\"versions\":[\"*\"],\"pull\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-gate\"],\"open-pulls\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"tags\":[\"mesh-delivery-forge-bridge\"],\"trees\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"file\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"contains\":[\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"statuses\":[\"mesh-delivery-gate\"],\"status\":[\"mesh-delivery-forge-bridge\"],\"comment\":[\"mesh-delivery-forge-bridge\"],\"view\":[\"mesh-delivery-forge-bridge\"],\"note\":[\"mesh-delivery-forge-bridge\"],\"merge-pull\":[\"mesh-delivery-forge-bridge\"]}", + "default": "{\"publish\":[\"mesh-build-go\",\"mesh-build-node\"],\"publish-part\":[\"mesh-build-go\",\"mesh-build-node\"],\"remove\":[\"mesh-delivery-builder\"],\"versions\":[\"*\"],\"pull\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-gate\"],\"branch\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\"],\"open-pulls\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"tags\":[\"mesh-delivery-forge-bridge\"],\"trees\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"file\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"contains\":[\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"statuses\":[\"mesh-delivery-gate\"],\"status\":[\"mesh-delivery-forge-bridge\"],\"comment\":[\"mesh-delivery-forge-bridge\"],\"view\":[\"mesh-delivery-forge-bridge\"],\"note\":[\"mesh-delivery-forge-bridge\"],\"merge-pull\":[\"mesh-delivery-forge-bridge\"]}", "description": "per verb of the seats the module holds, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); each holder reads the lines of its own verbs. The package seats' lines are design 51's (empty is that list for them); the seat git's are design 47's: its reads the parts that declare them in invokes, its acts (status, comment, view, note, merge-pull) the forge bridge alone, and a verb of git with no line is called by no one" } }, diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 4078216b8..4fac593f2 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -20,7 +20,7 @@ import { callerOf, refusal, parseVerbCallers, type VerbCallers } from "@novox/me import { config } from "@novox/mesh-sdk/runtime"; import { seat as GIT, verbs as GIT_VERBS, - type ContainsAnswer, type CommentAnswer, type ErrorKind, type FileAnswer, type MergePullAnswer, type Merging, type NoteAnswer, + type BranchAnswer, type ContainsAnswer, type CommentAnswer, type ErrorKind, type FileAnswer, type MergePullAnswer, type Merging, type NoteAnswer, type OpenPull, type OpenPullsAnswer, type Pull, type PullAnswer, type Status, type StatusAnswer, type StatusesAnswer, type TagsAnswer, type TreesAnswer, type ViewAnswer, } from "@novox/mesh-delivery-contracts-git"; @@ -170,6 +170,14 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] const pull = pullOf(full, p); return pull.state === "open" ? { pull, merging: await mergingOf(gitea, owner, repo, p) } : { pull }; }), + verb(GIT_VERBS.branch, "The seat git's branch: a branch's head, and whether a protection rule refuses a direct push and admits only merges.", + async (args): Promise => { + const { owner, repo, full } = repositoryOf(args.repository); + const branch = String(args.branch ?? ""); + // The head first: a branch the forge does not hold is its 404, the kind not-found, never an unprotected branch. + const head = await gitea.branchHead(owner, repo, branch); + return { repository: full, branch, head, protected: pushRefused(await gitea.appliedProtection(owner, repo, branch)) }; + }), verb(GIT_VERBS.openPulls, "The seat git's open-pulls: every open pull request of every repository, with the statuses its base requires and those its head has.", async (): Promise => { const pulls: OpenPull[] = (await gitea.openPulls()).map((o) => ({ @@ -273,6 +281,11 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] ]; } +/** Whether a protection rule refuses a direct push and admits only merges: the rule exists and takes no push. */ +export function pushRefused(rule: { enable_push?: boolean } | null): boolean { + return !!rule && rule.enable_push === false; +} + function holds(comments: GiteaComment[], id: string): boolean { return comments.some((c) => String(c.body ?? "").includes(actMarker(id))); } @@ -295,7 +308,7 @@ export async function mergePull(gitea: GiteaClient, args: Record [a.pull.state, a.pull["merge-commit"], a.merging], ["merged", OTHER, undefined]], ["pull", "unknown", { repository: "novox/app", number: 404 }, {}, (a) => [a.pull, a["error-kind"]], [undefined, "not-found"]], ["pull", "forge down", { repository: "novox/app", number: 9 }, { getPullRequest: down }, (a) => [a.pull, a["error-kind"]], [undefined, "failed"]], + ["branch", "a branch a rule guards", { repository: "novox/app", branch: "main" }, { branchHead: async () => SHA, appliedProtection: async () => ({ rule_name: "main", enable_push: false }) }, (a) => [a.head, a.protected], [SHA, true]], + ["branch", "a branch no rule guards", { repository: "novox/app", branch: "scratch" }, { branchHead: async () => SHA, appliedProtection: async () => null }, (a) => [a.head, a.protected], [SHA, false]], + ["branch", "a rule that takes a push", { repository: "novox/app", branch: "main" }, { branchHead: async () => SHA, appliedProtection: async () => ({ rule_name: "main", enable_push: true }) }, (a) => [a.protected], [false]], + ["branch", "no such branch", { repository: "novox/app", branch: "nope" }, { branchHead: gone }, (a) => [a.head, a["error-kind"]], [undefined, "not-found"]], ["open-pulls", "one", {}, {}, (a) => [a.pulls.length, a.pulls[0].required], [1, ["mesh/check"]]], ["open-pulls", "forge down", {}, { openPulls: down }, (a) => [a.pulls, a["error-kind"]], [undefined, "failed"]], ["tags", "a page", { repository: "novox/app", prefix: "go/" }, {}, (a) => [a.tags, a.page, a.bound, a.more], [["go/v0.1.1", "go/v0.1.0"], 1, 50, undefined]], -- 2.54.0 From 975f1aa8e83dcb467b8c8691c4900dbd3dfddb59 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:32:33 +0100 Subject: [PATCH 013/243] GIT5: the forge's push webhook of a branch says branch-moved with the new head on the seat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §11, §111, §331 branch-moved Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/seatevents.ts | 21 ++++++++++++++++----- modules/gitea/test/announce.test.ts | 8 +++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/modules/gitea/seatevents.ts b/modules/gitea/seatevents.ts index 8b859f846..4d49da269 100644 --- a/modules/gitea/seatevents.ts +++ b/modules/gitea/seatevents.ts @@ -10,9 +10,9 @@ import { seatPublish } from "@novox/mesh-sdk/runtime"; import { forgeDeleteSubject, forgePullSubject, forgeStatusSubject, pullClosedSubject, pullMergedSubject, pullUpdatedSubject, - repoWritableSubject, + repoWritableSubject, branchMovedSubject, type Change, type ForgeDeleteEvent, type ForgePullEvent, type ForgeStatusEvent, type PullClosedEvent, type PullMergedEvent, - type PullUpdatedEvent, type RepoWritableEvent, + type PullUpdatedEvent, type RepoWritableEvent, type BranchMovedEvent, } from "@novox/mesh-delivery-contracts-git"; /** The forge's pull request actions the seat's forge.pull says (the contract's word list). */ @@ -36,8 +36,18 @@ function changeOf(b: Body): Change | undefined { } /** The seat's subject and body for one of the module's announcements, or undefined when the seat says no such event. */ -export function seatEventOf(type: string, b: Body): { subject: string; body: unknown } | undefined { +export function seatEventOf(type: string, b: Body, now: () => Date = () => new Date()): { subject: string; body: unknown; id?: string } | undefined { switch (type) { + case "forge.push": { + // A branch moved, by a merge or an admitted push alike (design 47 §11): the forge's push webhook of a branch with a + // head after it. A tag's push, and a branch's deletion (the head is the zero hash), move no branch. + const ref = s(b.ref) ?? ""; + const head = s(b.after) ?? ""; + if (!ref.startsWith("refs/heads/") || !/^[0-9a-f]{40,64}$/.test(head) || /^0+$/.test(head)) return undefined; + const branch = ref.slice("refs/heads/".length); + const body: BranchMovedEvent = { repository: repositoryOf(b), branch, head, at: now().toISOString() }; + return { subject: branchMovedSubject, body, id: `moved:${body.repository}:${branch}@${head}` }; + } case "pull.updated": { const body: PullUpdatedEvent = { repository: repositoryOf(b), number: Number(b.number), base: String(b.base ?? ""), branch: String(b.head ?? ""), head: String(b.head_sha ?? ""), ...opt("base-head", s(b.base_sha)), @@ -82,7 +92,8 @@ export function seatEventOf(type: string, b: Body): { subject: string; body: unk return undefined; } -/** The module's own events the seat says instead: never said as the module's (its `emits` names none of them). */ +/** The module's own events the seat says instead: never said as the module's (its `emits` names none of them). `forge.push` + * stays the module's own too: the seat's branch-moved is said beside it. */ export const SEAT_ONLY = new Set(["pull.updated", "pull.merged", "pull.closed", "forge.status", "forge.delete", "repo.writable"]); export type Say = (subject: string, body: unknown, id?: string) => Promise; @@ -91,6 +102,6 @@ export type Say = (subject: string, body: unknown, id?: string) => Promise export async function sayOnSeat(type: string, body: Body, id = "", say: Say = seatPublish): Promise { const e = seatEventOf(type, body); if (!e) return false; - await say(e.subject, e.body, id); + await say(e.subject, e.body, id || e.id); return true; } diff --git a/modules/gitea/test/announce.test.ts b/modules/gitea/test/announce.test.ts index 7a3c95eb6..a40ff4a3d 100644 --- a/modules/gitea/test/announce.test.ts +++ b/modules/gitea/test/announce.test.ts @@ -45,13 +45,19 @@ test("an announcement is said on the seat git with its id, in the contract's bod ["repo.writable", { owner: "novox", repo: "app", account: "mesh-tools" }, "mesh.seat.git.event.repo-writable"], ["forge.pull-request.synchronized", { full_name: "novox/app", number: 7, head_sha: "aaa111" }, "mesh.seat.git.event.forge-pull"], ["forge.pull-request.labeled", { full_name: "novox/app", number: 7 }, undefined], - ["forge.push", { full_name: "novox/app" }, undefined], + ["forge.push", { full_name: "novox/app", ref: "refs/tags/v1", after: "a".repeat(40) }, undefined], + ["forge.push", { full_name: "novox/app", ref: "refs/heads/gone", after: "0".repeat(40) }, undefined], + ["forge.push", { full_name: "novox/app", ref: "refs/heads/feat/x", after: "a".repeat(40) }, "mesh.seat.git.event.branch-moved"], ]; for (const [type, body, subject] of rows) assert.equal(seatEventOf(type, body)?.subject, subject, type); assert.deepEqual(seatEventOf("pull.updated", updated)?.body, { repository: "novox/app", number: 7, base: "main", branch: "", head: "aaa111", "base-head": "ttt333" }); assert.equal(seatEventOf("forge.pull-request.edited", { full_name: "novox/app", number: 7, body: "x" })?.body && "body" in (seatEventOf("forge.pull-request.edited", { full_name: "novox/app", number: 7, body: "x" })!.body as object), false); + // A push of a branch (a merge and an admitted push alike) says branch-moved with the new head, one id per move. + const moved = seatEventOf("forge.push", { full_name: "novox/app", ref: "refs/heads/main", after: "b".repeat(40) }, () => new Date("2026-11-11T10:00:00Z")); + assert.deepEqual(moved?.body, { repository: "novox/app", branch: "main", head: "b".repeat(40), at: "2026-11-11T10:00:00.000Z" }); + assert.equal(moved?.id, `moved:novox/app:main@${"b".repeat(40)}`); assert.deepEqual([...SEAT_ONLY].sort(), ["forge.delete", "forge.status", "pull.closed", "pull.merged", "pull.updated", "repo.writable"]); }); -- 2.54.0 From a00429bc80dbf3e739f45141f4f0c5c7f1066c50 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:36:37 +0100 Subject: [PATCH 014/243] GIT6, GIT9: merge-pull merges the head the caller names and asks no Approve; it refuses every merge until the first registrations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit merge-pull answers its refused-kind (head-moved, unprotected, git-refused, no-registrations); an already merged pull request at that head is answered performed. The failed-steps refusal (policy-failed) is GIT7, stopped on a design question. GIT9: registrations.ts learns the first registrations from the seat mesh-catalog's verb registrations at the start and its event registration-changed; the code artifact needs mesh-delivery-assigner:program. Builds: design 47 G §38, §129, §314, §95, §96, §130, §353, §432 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/index.ts | 11 +++++-- modules/gitea/module.json | 7 ++++- modules/gitea/registrations.ts | 52 ++++++++++++++++++++++++++++++++ modules/gitea/seat.ts | 22 +++++++++----- modules/gitea/test/guard.test.ts | 43 ++++++++++++++++++++------ 5 files changed, 115 insertions(+), 20 deletions(-) create mode 100644 modules/gitea/registrations.ts diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 2f927b0c4..37c897c3a 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -42,11 +42,12 @@ import { state } from "@novox/mesh-sdk/state"; import { on } from "@novox/mesh-sdk/events"; import { broker } from "@novox/mesh-sdk/messaging"; import { STATUS_STATE, keepStatus, statusTools, viewWaitsFinding, type Follower, type Status } from "@novox/mesh-sdk/views"; -import type { CatalogRevisionEvent, RevisionInput } from "@novox/mesh-delivery-contracts-mesh-catalog"; +import type { CatalogRevisionEvent, RegistrationChangedEvent, RevisionInput } from "@novox/mesh-delivery-contracts-mesh-catalog"; import { CATALOG_REVISION_EVENT, CatalogView, findingValue, unansweredFinding } from "./catalog.js"; import { announce, keepWatermark, readWatermark } from "./announce.js"; import { SEAT_ONLY, sayOnSeat } from "./seatevents.js"; import { FORGE_CONTAINER, seatTools } from "./seat.js"; +import { REGISTRATION_CHANGED_EVENT, Registrations } from "./registrations.js"; import { run } from "./delivery.js"; import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { keepMainRules, mainRulesStep } from "./mainrules.js"; @@ -390,13 +391,15 @@ const writes = (): DeliveredWrites => (writesOnce ??= new DeliveredWrites(GiteaA // module's state first-views, which gitea_first_views reads; gitea_first_view_again asks it again by hand (mesh-sdk views). const firstViews = state(STATUS_STATE); const followers: Follower[] = []; +// The first registrations switch merge-pull on (registrations.ts; design 47 §95). +const registrations = new Registrations((verb, input) => broker().request(verb, input), (line) => console.log(`[gitea] ${line}`)); // gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). The seat git's verbs are // served here, beside the one writer of a head's statuses: the seat's status takes its turn on the head's chain, after the // announcement's pending and a trunk's move, so a recheck's pending never lands over a later verdict (issue 456). registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...statusTools("gitea", "gitea_first_views", "gitea_first_view_again", followers, firstViews, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), - ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, statuses: gates }) : [])]); + ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, statuses: gates, registered: () => registrations.written }) : [])]); /** What the forge's event asks of the announcer at once (issue 729, 1422): see watch.ts. */ const actOnEvent = (client: GiteaClient, event: ForgeEvent): Promise => actOn(event, { @@ -533,6 +536,10 @@ if (gitea) { // What is no catalog-revision event is said and taken: asked again, it would be no more readable. view.hear(event.body).catch((err) => console.error(`[gitea] ${err instanceof Error ? err.message : err}`)))); void view.follower.start().then(() => w.viewed()); + void bindUntilHeld(REGISTRATION_CHANGED_EVENT, () => on(REGISTRATION_CHANGED_EVENT, async (event) => { + try { registrations.hear(event.body); } catch (err) { console.error(`[gitea] ${err instanceof Error ? err.message : err}`); } + })); + void registrations.start(); } catch (err) { console.error(`[gitea] the merging account is not given write on the delivered repositories: no site administrator's login — ${err instanceof Error ? err.message : err}`); } diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 9af9a0c41..d6bdf1e8a 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -499,6 +499,9 @@ "name": "code", "kind": "bundle", "language": "typescript", + "needs": [ + "mesh-delivery-assigner:program" + ], "entrypoints": [ "index.js", "tools/index.js" @@ -644,10 +647,12 @@ ], "invokes": [ "seat:mesh-catalog.revision", + "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", "mesh-delivery-builder.builder_versions" ], "consumes": [ - "mesh-catalog.catalog-revision" + "mesh-catalog.catalog-revision", + "mesh-catalog.registration-changed" ] } diff --git a/modules/gitea/registrations.ts b/modules/gitea/registrations.ts new file mode 100644 index 000000000..82a46d634 --- /dev/null +++ b/modules/gitea/registrations.ts @@ -0,0 +1,52 @@ +// **The first registrations switch the merge on** (novox/hq design 47 §95, §353): until the catalog half has said its first +// `registration-changed`, the seat git's `merge-pull` refuses every merge naming the missing registrations, because a merge +// made with no Approve could reach the old assigner, which writes wants from the trunk with no pipeline. The module learns it +// two ways, both through the seat `mesh-catalog`: its verb `registrations` asked at the start (a restart after the first +// registrations were written), and its event `registration-changed` (the first being written, or heard again from the +// seat's kept stream). Neither is the holder's state or a module named. + +import { + seat as CATALOG, verbs as CATALOG_VERBS, events as CATALOG_EVENTS, + type RegistrationChangedEvent, type RegistrationsAnswer, type RegistrationsInput, +} from "@novox/mesh-delivery-contracts-mesh-catalog"; + +/** The seat verb asked, as the manifest's `invokes` names it. */ +export const REGISTRATIONS_VERB = `seat:${CATALOG}.${CATALOG_VERBS.registrations}`; +/** The seat event heard, as the manifest's `consumes` names it. */ +export const REGISTRATION_CHANGED_EVENT = `${CATALOG}.${CATALOG_EVENTS.registrationChanged}`; + +export class Registrations { + private first = false; + constructor(private readonly ask: (verb: string, input: RegistrationsInput) => Promise, + private readonly log: (line: string) => void = () => {}) {} + + /** Whether the first registrations are written: the one fact the merge switches on. */ + get written(): boolean { + return this.first; + } + + /** One `registration-changed`: the first registration written, however many were before this module started. */ + hear(ev: RegistrationChangedEvent | undefined): void { + if (!ev || typeof ev.module !== "string" || ev.module === "") { + throw new Error(`not a ${REGISTRATION_CHANGED_EVENT} event: ${JSON.stringify(ev ?? null).slice(0, 200)}`); + } + if (!this.first) this.log(`the first registration is written (${ev.module}); merge-pull merges from here`); + this.first = true; + } + + /** The start's first view: any registration held is the first written. A failed ask is said and left to the event. */ + async start(): Promise { + try { + let a: unknown = await this.ask(REGISTRATIONS_VERB, {}); + if (typeof a === "string") a = JSON.parse(a); + const r = (a as RegistrationsAnswer | undefined)?.registrations; + if (!Array.isArray(r)) throw new Error(`${REGISTRATIONS_VERB} answered no registrations: ${JSON.stringify(a ?? null).slice(0, 200)}`); + if (r.length > 0) { + this.first = true; + this.log(`${r.length} registrations are held; merge-pull merges`); + } + } catch (err) { + this.log(`the registrations could not be read: ${err instanceof Error ? err.message : String(err)}; merge-pull waits for registration-changed`); + } + } +} diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 4fac593f2..33c56dbeb 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -55,6 +55,8 @@ export interface SeatDeps { run: Runner; container: string; statuses: HeadStatuses; + /** Whether the first registrations were written (registrations.ts): until then merge-pull refuses every merge (design 47 §95). */ + registered: () => boolean; /** The config value `callers` as the runtime answers it; the runtime's own config read by default. */ callers?: () => Promise; } @@ -277,7 +279,7 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] return said.added ? { id, performed: true } : { id, held: true, performed: true }; }), verb(GIT_VERBS.mergePull, "The seat git's merge-pull: an act of the forge bridge, a pull request merged at the head the caller names; it judges no check.", - async (args): Promise => mergePull(gitea, args)), + async (args): Promise => mergePull(gitea, args, deps.registered)), ]; } @@ -291,31 +293,35 @@ function holds(comments: GiteaComment[], id: string): boolean { } /** merge-pull (design 47 §314): only the head the caller names, only into a base that takes no push, no check judged. */ -export async function mergePull(gitea: GiteaClient, args: Record): Promise { +export async function mergePull(gitea: GiteaClient, args: Record, registered: () => boolean): Promise { const id = idOf(args.id); const { owner, repo, full } = repositoryOf(args.repository); const number = numberOf(args.number); const head = commitOf("head", args.head); - const no = (why: string): MergePullAnswer => ({ id, refused: `${full}#${number} is not merged: ${why}` }); + const no = (why: string, kind: NonNullable): MergePullAnswer => + ({ id, refused: `${full}#${number} is not merged: ${why}`, "refused-kind": kind }); + // Until the first registrations are written no merge is made: the old assigner would write a want from the trunk with no + // pipeline (design 47 §95). It names what is missing. + if (!registered()) return no("no module is registered yet: the catalog has said no registration-changed, so no merge is made until the first registrations are written", "no-registrations"); const pull = await gitea.getPullRequest(owner, repo, number); if (pull.merged) { // Already merged at that head: performed (the act said again after its answer was lost); at another, refused. return pull.head_sha === head ? { id, performed: true, held: true, ...(pull.merge_commit_sha ? { "merge-commit": pull.merge_commit_sha } : {}) } - : no(`it was merged at ${String(pull.head_sha).slice(0, 12)}, not the ${head.slice(0, 12)} the caller names`); + : no(`it was merged at ${String(pull.head_sha).slice(0, 12)}, not the ${head.slice(0, 12)} the caller names`, "head-moved"); } - if (pull.state !== "open") return no("it is closed"); + if (pull.state !== "open") return no("it is closed", "git-refused"); if (pull.head_sha !== head) { - return no(`its head is ${String(pull.head_sha).slice(0, 12)}, not the ${head.slice(0, 12)} the caller names`); + return no(`its head is ${String(pull.head_sha).slice(0, 12)}, not the ${head.slice(0, 12)} the caller names`, "head-moved"); } const rule = await gitea.appliedProtection(owner, repo, String(pull.base ?? "")); if (!pushRefused(rule)) { - return no(`its base ${pull.base} is guarded by no protection rule that refuses a push`); + return no(`its base ${pull.base} is guarded by no protection rule that refuses a push`, "unprotected"); } try { await gitea.mergePullRequest(owner, repo, number, "merge", true, head); } catch (err) { const f = failure(err); - if (f["error-kind"] === "refused") return no(`the forge refused the merge: ${f.error}`); + if (f["error-kind"] === "refused") return no(`the forge refused the merge: ${f.error}`, "git-refused"); throw err; } const merged = await gitea.getPullRequest(owner, repo, number); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 833870a15..a066ee62d 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -439,6 +439,7 @@ test("the seat git's verbs judge their caller, and merge-pull merges the Approve const { seatTools, gitCallers } = await import("../dist/seat.js"); const { runAs } = await import("@novox/mesh-sdk/callers"); const bridge = "ace.mesh-delivery-forge-bridge"; + let registered = true; const list = JSON.stringify({ publish: ["mesh-build-go"], "merge-pull": ["mesh-delivery-forge-bridge"], pull: ["mesh-delivery-forge-bridge", "mesh-delivery-gate"], status: ["mesh-delivery-forge-bridge"] }); assert.deepEqual(Object.keys(gitCallers(list)).sort(), ["merge-pull", "pull", "status"], "only the seat git's lines are its"); @@ -456,29 +457,31 @@ test("the seat git's verbs judge their caller, and merge-pull merges the Approve }; const gates = new GateWriter(slow); const seat = seatTools(Object.assign(Object.create(client), { setCommitStatus: slow.setCommitStatus }), - { run: async () => ({ stdout: "", code: 0 }), container: "gitea", callers: async () => list, statuses: gates }); + { run: async () => ({ stdout: "", code: 0 }), container: "gitea", callers: async () => list, statuses: gates, registered: () => registered }); const call = (verb: string, args: Record, caller = bridge) => runAs({ "mesh/caller": caller }, args, () => seat.find((t: any) => t.name === `git.${verb}`)!.run(args)) as Promise; const merge = (extra: Record = {}, caller?: string) => call("merge-pull", { id: "act-1", repository: "novox/mesh-catalog", number: 7, head: REVIEWED, ...extra }, caller); - const rows: { name: string; set: () => void; caller?: string; extra?: Record; want: RegExp | "merged" | "held"; loud?: boolean }[] = [ + const rows: { name: string; set: () => void; caller?: string; extra?: Record; want: RegExp | "merged" | "held"; loud?: boolean; kind?: string }[] = [ + { name: "no registration written yet", set: () => { registered = false; }, want: /no module is registered yet/, kind: "no-registrations" }, { name: "a caller not on the list", set: () => {}, caller: "ace.mesh-delivery-checker", want: /refused for ace.mesh-delivery-checker/ }, { name: "an unproven caller", set: () => {}, caller: "", want: /no caller the tool runner proved/ }, - { name: "a head that moved", set: () => { state.head = MOVED; }, want: /its head is 3c4d5e6f7a8b, not the abc1234def56/ }, - { name: "a base no rule guards", set: () => { state.rules = []; }, want: /guarded by no protection rule/ }, - { name: "a base that takes a push", set: () => { state.rules = [{ rule_name: "main", enable_push: true }]; }, want: /guarded by no protection rule/ }, + { name: "a head that moved", set: () => { state.head = MOVED; }, want: /its head is 3c4d5e6f7a8b, not the abc1234def56/, kind: "head-moved" }, + { name: "a base no rule guards", set: () => { state.rules = []; }, want: /guarded by no protection rule/, kind: "unprotected" }, + { name: "a base whose protection was removed after the last branch-moved", set: () => { state.rules = [protectedMain()]; state.rules = []; }, want: /guarded by no protection rule/, kind: "unprotected" }, + { name: "a base that takes a push", set: () => { state.rules = [{ rule_name: "main", enable_push: true }]; }, want: /guarded by no protection rule/, kind: "unprotected" }, { name: "a failed check", set: () => { state.statuses = [{ context: "mesh/check", state: "failure", description: "fail" }]; }, want: "merged" }, { name: "no status at all", set: () => { state.statuses = []; }, want: "merged" }, { name: "a check of another trunk", set: () => { state.statuses = judgedAt(MOVED); }, want: "merged" }, { name: "the id said again after the merge", set: () => { state.merged = true; }, want: "held" }, { name: "a pull request merged at another head", set: () => { state.merged = true; state.head = MOVED; }, want: /merged at 3c4d5e6f7a8b, not the abc1234def56/ }, - { name: "a closed pull request", set: () => { state.prState = "closed"; }, want: /it is closed/ }, + { name: "a closed pull request", set: () => { state.prState = "closed"; }, want: /it is closed/, kind: "git-refused" }, { name: "a fast-forward", set: () => { state.parents = [REVIEWED]; }, want: "merged" }, { name: "the merge read back fails", set: () => { state.commitsFail = 502; }, want: "merged" }, { name: "the forge merged another head", set: () => { state.landed = MOVED; }, want: /WAS merged .*the forge merged 3c4d5e6f7a8b, not the abc1234def56/, loud: true }, ]; for (const r of rows) { - state.merged = false; state.head = REVIEWED; state.statuses = judgedAt(TRUNK); state.sent = []; + registered = true; state.merged = false; state.head = REVIEWED; state.statuses = judgedAt(TRUNK); state.sent = []; state.rules = [protectedMain()]; state.prState = "open"; state.parents = null; state.commitsFail = 0; state.landed = ""; r.set(); const got = await merge(r.extra, r.caller); @@ -491,6 +494,7 @@ test("the seat git's verbs judge their caller, and merge-pull merges the Approve assert.deepEqual(writes(), [], `${r.name}: nothing merged again`); } else { assert.match(String(got.refused ?? got.error), r.want, r.name); + if (r.kind) assert.equal(got["refused-kind"], r.kind, r.name); assert.deepEqual(writes(), r.loud ? ["POST /repos/novox/mesh-catalog/pulls/7/merge"] : [], `${r.name}: what was merged`); } } @@ -550,11 +554,11 @@ test("every read and act of the seat git answers its contract, and an error is a }; const run = async (_f: string, args: string[]) => args.includes("show") || args.includes("--ref") && !args.includes("append") ? { stdout: noteShown, code: noteShown ? 0 : 1 } : (noteWritten++, { stdout: "", code: 0 }); - const seat = seatTools(forge as any, { run: run as any, container: "gitea", callers: async () => JSON.stringify({}), statuses: { status: async () => {} } }); + const seat = seatTools(forge as any, { run: run as any, container: "gitea", callers: async () => JSON.stringify({}), statuses: { status: async () => {} }, registered: () => true }); const everyone = JSON.stringify(Object.fromEntries(seat.map((t: any) => [t.name.slice(4), ["*"]]))); const call = (verb: string, args: Record, over: Record = {}) => { const t = seat.find((x: any) => x.name === `git.${verb}`)!; - const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", callers: async () => everyone, statuses: { status: async () => {} } }); + const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", callers: async () => everyone, statuses: { status: async () => {} }, registered: () => true }); return runAs({ "mesh/caller": "ace.x" }, args, () => wrapped.find((x: any) => x.name === t.name)!.run(args)) as Promise; }; const rows: [string, string, Record, Record, (a: any) => unknown, unknown][] = [ @@ -598,3 +602,24 @@ test("every read and act of the seat git answers its contract, and an error is a const note = await call("note", { id: "n1", repository: "novox/app", commit: SHA, line: "worded differently" }); assert.deepEqual([note.performed, note.held, noteWritten], [true, true, 0]); }); + +// The first registrations switch merge-pull on (design 47 §95, §353): none held and none heard, it waits; a registration held +// at the start, or the first registration-changed, switches it, and an event that is none is an error. +test("the first registrations are learned from the seat mesh-catalog's verb at the start and from its event", async () => { + const { Registrations } = await import("../dist/registrations.js"); + const said: string[] = []; + const reg = (answer: unknown) => new Registrations(async () => { if (answer instanceof Error) throw answer; return answer; }, (l) => said.push(l)); + const none = reg({ registrations: [] }); + await none.start(); + assert.equal(none.written, false, "an empty answer is no first registration"); + none.hear({ module: "mesh-sdk", moved: ["watch"], version: 1, warrant: "q-1" }); + assert.equal(none.written, true, "the event switches it"); + const held = reg(JSON.stringify({ registrations: [{ module: "mesh-sdk" }] })); + await held.start(); + assert.equal(held.written, true, "a registration held at the start"); + const down = reg(new Error("no responders")); + await down.start(); + assert.equal(down.written, false); + assert.match(said.join("\n"), /could not be read: no responders/, "a failed first view is said, never taken as none silently"); + assert.throws(() => down.hear(undefined as any), /not a mesh-catalog.registration-changed event/); +}); -- 2.54.0 From 5daf3a333ae3c8c538f99522d7aeadb710a35f0d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:37:36 +0100 Subject: [PATCH 015/243] GIT11: the seat git's tag writes go/vMAJOR.MINOR.PATCH at the commit a released version was built from MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tag at that commit is answered performed; at another commit, refused (a tag is never moved). The builder alone calls it: the bus refuses any other account from the definition's callers. Builds: design 47 G §370 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/client.ts | 18 ++++++++++++++++++ modules/gitea/module.json | 5 +++-- modules/gitea/seat.ts | 16 +++++++++++++++- modules/gitea/test/guard.test.ts | 5 +++++ 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 42898d71c..78648b3fb 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -496,6 +496,24 @@ export class GiteaClient { return list.map((t) => String(t?.name ?? "")).filter((n) => n !== ""); } + /** The commit a tag names, or null when the repository has no such tag (the forge's 404 only). */ + async tagCommit(owner: string, repo: string, tag: string): Promise { + try { + const t = await this.request(`${repoPath(owner, repo)}/tags/${pathSegments("tag", tag)}`); + const sha = String(t?.commit?.sha ?? t?.id ?? ""); + if (!/^[0-9a-f]{40}$/.test(sha)) throw new Error(`the forge named no commit for ${owner}/${repo}'s tag ${tag}`); + return sha; + } catch (err) { + if (/: 404 /.test(err instanceof Error ? err.message : "")) return null; + throw err; + } + } + + /** Write a lightweight tag at a commit (design 47 §370): what the seat git's `tag` does. */ + async createTag(owner: string, repo: string, tag: string, commit: string): Promise { + await this.request(`${repoPath(owner, repo)}/tags`, { method: "POST", body: JSON.stringify({ tag_name: tag, target: commit }) }); + } + /** A commit's parents, by sha, in order: what a merge commit says it joined (novox/hq issue 381). An answer * that names no parents is no answer, never an empty list read as a root commit. */ async commitParents(owner: string, repo: string, sha: string): Promise { diff --git a/modules/gitea/module.json b/modules/gitea/module.json index d6bdf1e8a..dd462ecbe 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -181,7 +181,8 @@ "comment", "view", "note", - "merge-pull" + "merge-pull", + "tag" ] } ], @@ -252,7 +253,7 @@ "type": "string", "trusted": true, "runtime": true, - "default": "{\"publish\":[\"mesh-build-go\",\"mesh-build-node\"],\"publish-part\":[\"mesh-build-go\",\"mesh-build-node\"],\"remove\":[\"mesh-delivery-builder\"],\"versions\":[\"*\"],\"pull\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-gate\"],\"branch\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\"],\"open-pulls\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"tags\":[\"mesh-delivery-forge-bridge\"],\"trees\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"file\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"contains\":[\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"statuses\":[\"mesh-delivery-gate\"],\"status\":[\"mesh-delivery-forge-bridge\"],\"comment\":[\"mesh-delivery-forge-bridge\"],\"view\":[\"mesh-delivery-forge-bridge\"],\"note\":[\"mesh-delivery-forge-bridge\"],\"merge-pull\":[\"mesh-delivery-forge-bridge\"]}", + "default": "{\"publish\":[\"mesh-build-go\",\"mesh-build-node\"],\"publish-part\":[\"mesh-build-go\",\"mesh-build-node\"],\"remove\":[\"mesh-delivery-builder\"],\"versions\":[\"*\"],\"pull\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-gate\"],\"branch\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\"],\"open-pulls\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"tags\":[\"mesh-delivery-forge-bridge\"],\"trees\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"file\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"contains\":[\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"statuses\":[\"mesh-delivery-gate\"],\"status\":[\"mesh-delivery-forge-bridge\"],\"comment\":[\"mesh-delivery-forge-bridge\"],\"view\":[\"mesh-delivery-forge-bridge\"],\"note\":[\"mesh-delivery-forge-bridge\"],\"merge-pull\":[\"mesh-delivery-forge-bridge\"],\"tag\":[\"mesh-delivery-builder\"]}", "description": "per verb of the seats the module holds, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); each holder reads the lines of its own verbs. The package seats' lines are design 51's (empty is that list for them); the seat git's are design 47's: its reads the parts that declare them in invokes, its acts (status, comment, view, note, merge-pull) the forge bridge alone, and a verb of git with no line is called by no one" } }, diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 33c56dbeb..5715098d3 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -22,7 +22,7 @@ import { seat as GIT, verbs as GIT_VERBS, type BranchAnswer, type ContainsAnswer, type CommentAnswer, type ErrorKind, type FileAnswer, type MergePullAnswer, type Merging, type NoteAnswer, type OpenPull, type OpenPullsAnswer, type Pull, type PullAnswer, type Status, type StatusAnswer, type StatusesAnswer, - type TagsAnswer, type TreesAnswer, type ViewAnswer, + type TagAnswer, type TagsAnswer, type TreesAnswer, type ViewAnswer, } from "@novox/mesh-delivery-contracts-git"; import type { CommitStatus, GiteaClient, GiteaComment, GiteaPull } from "./client.js"; import { MAX_FILE_BYTES } from "./client.js"; @@ -95,6 +95,8 @@ export function repositoryOf(raw: unknown): { owner: string; repo: string; full: } const SHA = /^[0-9a-f]{40}$/; +/** The git tag of a released Go version (design 51; design 47 §370). */ +const GO_TAG = /^go\/v\d+\.\d+\.\d+$/; const STATES = new Set(["pending", "success", "failure", "error", "warning"]); function commitOf(name: string, raw: unknown): string { const sha = String(raw ?? "").trim(); @@ -280,6 +282,18 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] }), verb(GIT_VERBS.mergePull, "The seat git's merge-pull: an act of the forge bridge, a pull request merged at the head the caller names; it judges no check.", async (args): Promise => mergePull(gitea, args, deps.registered)), + verb(GIT_VERBS.tag, "The seat git's tag: the git tag go/vMAJOR.MINOR.PATCH of a released version, written at the commit its build was made from; a tag that is at that commit is performed.", + async (args): Promise => { + const { owner, repo } = repositoryOf(args.repository); + const tag = String(args.tag ?? ""); + if (!GO_TAG.test(tag)) throw new Refused(`${JSON.stringify(tag)} is not go/vMAJOR.MINOR.PATCH`); + const commit = commitOf("commit", args.commit); + const at = await gitea.tagCommit(owner, repo, tag); + if (at === commit) return { tag, performed: true }; + if (at !== null) throw new Refused(`${tag} is at ${at.slice(0, 12)} already, not at ${commit.slice(0, 12)}; a tag is never moved`); + await gitea.createTag(owner, repo, tag, commit); + return { tag, performed: true }; + }), ]; } diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index a066ee62d..1474a1aec 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -536,6 +536,7 @@ test("every read and act of the seat git answers its contract, and an error is a const gone = () => { throw new Error("Gitea API GET /repos/novox/app/pulls/9: 404 Not Found"); }; const down = () => { throw new Error("Gitea API GET /repos/novox/app/x: 502 Bad Gateway"); }; const pr = { number: 9, title: "t", state: "open", merged: false, head: "feat/x", head_sha: SHA, base: "main", user: "jochen", html_url: "http://f/9", mergeable: true }; + const tagged: string[] = []; let comments: { id: number; body: string }[] = [], noteShown = "", added = 0, noteWritten = 0; const forge: Record = { getPullRequest: async (_o: string, _r: string, n: number) => (n === 9 ? pr : gone()), @@ -583,6 +584,10 @@ test("every read and act of the seat git answers its contract, and an error is a ["contains", "forge down", { repository: "novox/app", commit: SHA, in: OTHER }, { compare: down }, (a) => [a.contains, a["error-kind"]], [undefined, "failed"]], ["statuses", "newest of each", { repository: "novox/app", commit: SHA }, {}, (a) => [a.combined, a.statuses.map((s: any) => s.context)], ["success", ["mesh/check"]]], ["statuses", "forge down", { repository: "novox/app", commit: SHA }, { combinedStatus: down }, (a) => [a.statuses, a["error-kind"]], [undefined, "failed"]], + ["tag", "written", { repository: "novox/app", tag: "go/v1.2.3", commit: SHA }, { tagCommit: async () => null, createTag: async (_o: string, _r: string, tag: string, c: string) => { tagged.push(`${tag}@${c}`); } }, (a) => [a.performed, tagged.splice(0)], [true, [`go/v1.2.3@${SHA}`]]], + ["tag", "already at that commit", { repository: "novox/app", tag: "go/v1.2.3", commit: SHA }, { tagCommit: async () => SHA, createTag: async () => { throw new Error("written again"); } }, (a) => [a.performed, a.error], [true, undefined]], + ["tag", "at another commit is never moved", { repository: "novox/app", tag: "go/v1.2.3", commit: SHA }, { tagCommit: async () => OTHER }, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], + ["tag", "no Go version", { repository: "novox/app", tag: "v1", commit: SHA }, {}, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], ["status", "refused state", { id: "s1", repository: "novox/app", commit: SHA, context: "c", state: "bogus" }, {}, (a) => [a.performed, a["error-kind"]], [undefined, "refused"]], ["comment", "first time", { id: "c1", repository: "novox/app", number: 9, text: "hi" }, {}, (a) => [a.performed, a.held, added], [true, undefined, 1]], ["comment", "forge down", { id: "c2", repository: "novox/app", number: 9, text: "hi" }, { listComments: down }, (a) => [a.performed, a["error-kind"]], [undefined, "failed"]], -- 2.54.0 From 312d824e4fc156b2a3d051183d2e717bb38098f4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:43:18 +0100 Subject: [PATCH 016/243] GIT13: the forge module keeps no trusted callers; who may call a verb is the seat definition's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The config value callers, the package seats' list of callers with its watch, and the seat git's caller judgement are gone: the grants part grants by each definition's callers and the bus refuses the rest. Builds: design 47 G §68, §127, §313, §321 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/README.md | 2 +- .../cmd/package-registry/retention_test.go | 2 +- modules/gitea/cmd/package-registry/seats.go | 6 +- .../whole_integration_test.go | 18 +-- modules/gitea/internal/packageseats/holder.go | 117 +----------------- .../internal/packageseats/holder_test.go | 37 +----- modules/gitea/module.json | 7 -- modules/gitea/seat.ts | 36 +----- modules/gitea/test/guard.test.ts | 20 +-- 9 files changed, 17 insertions(+), 228 deletions(-) diff --git a/modules/gitea/README.md b/modules/gitea/README.md index 119e7c92d..c946a02b0 100644 --- a/modules/gitea/README.md +++ b/modules/gitea/README.md @@ -197,7 +197,7 @@ the module's own object store `incoming`, its transfers in the state `transfers` side), `versions` and `remove` (a tagged version, by the builder). Before anything is uploaded the holder asks the builder whether it numbered the version with that digest; a package goes up as npm's publish sends it, a Go module's zip to the Go upload. Every outcome is said on the seat's events (`published`, `refused`, `removed`), a Go package's -with its module path. Who may call each verb is the trusted config value `callers` (empty: design 51's list). +with its module path. Who may call each verb is the seat definition's `callers`, which the grants part grants by; the holder keeps no list. ## The package registry's retention (novox/hq ADR 0251 §4, to-be 51) diff --git a/modules/gitea/cmd/package-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go index 93d77ace9..10116919d 100644 --- a/modules/gitea/cmd/package-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -488,7 +488,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Errorf("transfers has options (a lifetime): %s", req) } // The retention asks the builder's versions; the forge's other bundles ask the catalog half and record a re-read. - if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-hand-acts.hand-act,mesh-delivery-builder.builder_versions" { + if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-catalog.registrations,seat:mesh-hand-acts.hand-act,mesh-delivery-builder.builder_versions" { t.Errorf("the manifest invokes %v", m.Invokes) } if len(m.Tools) != 0 { diff --git a/modules/gitea/cmd/package-registry/seats.go b/modules/gitea/cmd/package-registry/seats.go index cb4bce4df..5b409e603 100644 --- a/modules/gitea/cmd/package-registry/seats.go +++ b/modules/gitea/cmd/package-registry/seats.go @@ -276,11 +276,7 @@ var boundCap = parts.BusCap(packageseats.IncomingObjects) // busHolder is the holder over the runtime's bus state, the builder's numbering and the forge. func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) { - callers, err := packageseats.ConfiguredCallers() - if err != nil { - return nil, err - } - return &packageseats.Holder{Callers: callers, Objects: parts.BusObjects(packageseats.IncomingObjects), + return &packageseats.Holder{Objects: parts.BusObjects(packageseats.IncomingObjects), Transfers: parts.BusTransfers[packageseats.Opening](packageseats.TransfersState), Builder: builds.BuilderVersions{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding, Cap: boundCap, Record: recordAct, Idle: parts.BusArm(packageseats.IdleState)}, nil diff --git a/modules/gitea/cmd/package-registry/whole_integration_test.go b/modules/gitea/cmd/package-registry/whole_integration_test.go index 64ef99a8c..6c766e8af 100644 --- a/modules/gitea/cmd/package-registry/whole_integration_test.go +++ b/modules/gitea/cmd/package-registry/whole_integration_test.go @@ -6,7 +6,7 @@ package main // the built package-registry bundle, played over its real stdin and stdout as the node's tool runtime plays it, its // `incoming` an object store and its `transfers` and `findings` key-value buckets of a nats-server of this test alone // (so the publish in parts runs through mesh-sdk's parts.BusObjects and BusTransfers, ruling 4 of the lead's 14:0x), the -// builder's builder_versions answered on mesh/ask, the config value `callers` watched on mesh/provision.watch, and the forge an +// builder's builder_versions answered on mesh/ask, and the forge an // httptest server that takes reads only with the tools' token and writes only with the packages account's. // // A Go package the builder numbered goes up in two chunks, is uploaded whole as the packages account and says @@ -113,7 +113,6 @@ func (p *player) handle(m wire) { Data []byte `json:"data"` Subject string `json:"subject"` Body json.RawMessage `json:"body"` - Kind string `json:"kind"` TTL int `json:"ttl_seconds"` } _ = json.Unmarshal(m.Params, &a) @@ -123,12 +122,6 @@ func (p *player) handle(m wire) { _ = json.Unmarshal(a.Body, &q) d, ok := p.numbered[q.Name+"@"+q.Version] p.answer(m.ID, map[string]any{"recorded": ok && d == q.Digest, "why": "not numbered with that digest"}, nil) - case m.Method == "mesh/provision.watch" && a.Kind == "config": - // The watch of `callers`, as the runtime answers it: the value now as one change (mesh/provision, - // named by the watch's id) handed over before the watch is answered, here unset, so design 51's list holds. - p.send(map[string]any{"id": "change-" + watchID(m.ID), "method": "mesh/provision", "params": map[string]any{ - "watch": watchID(m.ID), "change": map[string]any{"kind": "config", "key": a.Key}}}) - p.answer(m.ID, map[string]any{}, nil) case m.Method == "mesh/seat.publish": p.answer(m.ID, map[string]any{}, p.nc.Publish(a.Subject, a.Body)) case m.Method == "mesh/publish": @@ -142,15 +135,6 @@ func (p *player) handle(m wire) { } } -// watchID is the watch's id as a string, the form the change names it by. -func watchID(raw json.RawMessage) string { - var id string - if json.Unmarshal(raw, &id) != nil { - return string(raw) - } - return id -} - func (p *player) objects(m wire, local, name string, data []byte) { os, err := p.js.ObjectStore(p.ctx, "gitea_"+local) if err != nil { diff --git a/modules/gitea/internal/packageseats/holder.go b/modules/gitea/internal/packageseats/holder.go index 70c98c080..7b03101fb 100644 --- a/modules/gitea/internal/packageseats/holder.go +++ b/modules/gitea/internal/packageseats/holder.go @@ -2,21 +2,18 @@ // `go-package-registry` (novox/hq design 51 *The four registry seats*, ADR 0605, ADR 0611). A build module publishes // a package through `publish` and `publish-part`: the package's one file lands in the forge module's own object store // `incoming`, is hashed, checked against the builder's numbering and then uploaded into the forge's package registry -// with the module's own write side. The receiving side is mesh-sdk's seat-neutral parts.Receiver and the caller list -// mesh-sdk's VerbCallers (ADR 0627 §2): this package binds both to the two seats and to the forge. +// with the module's own write side. The receiving side is mesh-sdk's seat-neutral parts.Receiver: this package binds it to the two seats and to the +// forge. Who may call a verb is the seat definition's `callers`, which the grants part grants by (design 47 §68, §321). package packageseats import ( - "encoding/json" "errors" "fmt" "strings" - "sync" "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-delivery/sdk/parts" - stdio "git.novox.be/novox/mesh-sdk/go" ) // The seats the forge module holds. @@ -67,103 +64,6 @@ const ( ReasonRetention = "retention" ) -// Callers is the trusted config value `callers`, read as mesh-sdk's list per verb. -type Callers = stdio.VerbCallers - -// DefaultCallers is design 51's list for the package seats: publish and publish-part the two build modules that -// publish packages; remove the builder (a tagged version whose branch ended); versions anyone on the mesh, as the -// registry's reads are anonymous. -func DefaultCallers() Callers { - builds := []string{"mesh-build-go", "mesh-build-node"} - return Callers{VerbPublish: builds, VerbPublishPart: builds, VerbRemove: {"mesh-delivery-builder"}, - VerbVersions: {stdio.AnyCaller}} -} - -// ConfiguredCallers is the config value `callers` as mesh-config resolved it, its lines of the package seats' verbs; unset, -// or with none of them, is DefaultCallers. It is held from a watch of the value and replaced on each change, so a call -// never asks the runtime (issue 1966). The same value carries the lines of the other seat the module holds (git, served -// by its TypeScript side), so each holder takes its own verbs' lines and neither refuses the other's. -func ConfiguredCallers() (Callers, error) { return configured() } - -var ( - watchMu sync.Mutex - heldMu sync.Mutex - heldList Callers - heldBad error -) - -// configured reads the held list; the first call starts the watch, and a change that cannot be read keeps the list last -// read, said and never widened. -func configured() (Callers, error) { - if l, err := read(); l != nil || err != nil { - return l, err - } - watchMu.Lock() - defer watchMu.Unlock() - if l, err := read(); l != nil || err != nil { - return l, err - } - if err := stdio.WatchConfig("callers", func(c stdio.ProvisionChange) error { - heldMu.Lock() - defer heldMu.Unlock() - if c.Refused != "" { - heldBad = fmt.Errorf("the config value callers is not answered: %s", c.Refused) - return nil - } - l, err := OwnCallers(c.Value) - if err != nil { - heldBad = fmt.Errorf("the config value callers cannot be read: %w", err) - return nil - } - heldList, heldBad = l, nil - return nil - }); err != nil { - return nil, fmt.Errorf("the config value callers cannot be read: %w", err) - } - l, err := read() - if l == nil && err == nil { - err = errors.New("the config value callers cannot be read: the runtime handed over no value") - } - return l, err -} - -func read() (Callers, error) { - heldMu.Lock() - defer heldMu.Unlock() - if heldList != nil { - return heldList, nil - } - return nil, heldBad -} - -// OwnCallers reads the package seats' lines of the config value as the runtime answered it: a JSON object of verb to -// callers, or that object written as a string. A value that cannot be read is an error, never the defaults. -func OwnCallers(raw json.RawMessage) (Callers, error) { - text := strings.TrimSpace(string(raw)) - var quoted string - if json.Unmarshal(raw, "ed) == nil { - text = strings.TrimSpace(quoted) - } - if text == "" || text == "null" { - return DefaultCallers(), nil - } - var all map[string]json.RawMessage - if err := json.Unmarshal([]byte(text), &all); err != nil { - return nil, fmt.Errorf("callers is not a JSON object of verb to callers: %w", err) - } - own := map[string]json.RawMessage{} - for _, verb := range Verbs { - if line, ok := all[verb]; ok { - own[verb] = line - } - } - if len(own) == 0 { - return DefaultCallers(), nil - } - b, _ := json.Marshal(own) - return stdio.ParseVerbCallers(string(b), Verbs, DefaultCallers()) -} - // Publish is a publish as the seats' contract names it (mesh-contracts npmpackageregistry and gopackageregistry share // one shape): what the tools bundle reads from either seat's PublishInput. type Publish struct { @@ -227,7 +127,6 @@ type Store interface { // Holder answers the verbs of both seats. type Holder struct { - Callers Callers Objects parts.Objects Transfers parts.Transfers[Opening] Builder Builder @@ -316,9 +215,6 @@ func (h *Holder) answer(transfer string, r parts.Received) Answer { // Publish is `publish` on seat for the proved caller: a package is one file, its digest the file's sha256. func (h *Holder) Publish(seat, caller string, p Publish) (Answer, error) { - if err := h.Callers.Allowed(VerbPublish, caller); err != nil { - return Answer{}, err - } if err := checkPublish(seat, p); err != nil { return Answer{}, err } @@ -349,9 +245,6 @@ func (h *Holder) Publish(seat, caller string, p Publish) (Answer, error) { // PublishPart is `publish-part`: one chunk, from the caller that opened the transfer alone. func (h *Holder) PublishPart(seat, caller, transfer, part string, sequence int, last bool, data []byte) (Answer, error) { - if err := h.Callers.Allowed(VerbPublishPart, caller); err != nil { - return Answer{}, err - } got, err := h.receiver().Take(key(seat, transfer), caller, part, sequence, last, data, func(t Transfer) error { if t.Shape.Seat != seat { return fmt.Errorf("transfer %s is %s's, not %s's", transfer, t.Shape.Seat, seat) @@ -366,9 +259,6 @@ func (h *Holder) PublishPart(seat, caller, transfer, part string, sequence int, // Versions is `versions`: every version the seat holds of name, newest first. func (h *Holder) Versions(seat, caller, name string) ([]Version, error) { - if err := h.Callers.Allowed(VerbVersions, caller); err != nil { - return nil, err - } if _, ok := Seats[seat]; !ok { return nil, fmt.Errorf("versions: the forge holds no seat %s", seat) } @@ -385,9 +275,6 @@ func (h *Holder) Versions(seat, caller, name string) ([]Version, error) { // Remove is `remove`: the builder removes a tagged version (its branch ended); an untagged version goes only by the // forge's own retention, never by a call. A version already gone is said removed all the same, so a retry ends. func (h *Holder) Remove(seat, caller, name, version, reason string) (Outcome, error) { - if err := h.Callers.Allowed(VerbRemove, caller); err != nil { - return Outcome{}, err - } if _, ok := Seats[seat]; !ok { return Outcome{}, fmt.Errorf("remove: the forge holds no seat %s", seat) } diff --git a/modules/gitea/internal/packageseats/holder_test.go b/modules/gitea/internal/packageseats/holder_test.go index 91e062cdc..63ebf4de8 100644 --- a/modules/gitea/internal/packageseats/holder_test.go +++ b/modules/gitea/internal/packageseats/holder_test.go @@ -10,7 +10,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" "git.novox.be/novox/mesh-delivery/sdk/parts" - stdio "git.novox.be/novox/mesh-sdk/go" ) type memObjects map[string][]byte @@ -102,7 +101,7 @@ type bed struct { func newBed(numbered bool) *bed { b := &bed{forge: &forge{files: map[string]map[string][]byte{}}, findings: map[string]bool{}} - b.h = &Holder{Callers: DefaultCallers(), Objects: memObjects{}, Transfers: memTransfers{}, Builder: builder{ok: numbered}, + b.h = &Holder{Objects: memObjects{}, Transfers: memTransfers{}, Builder: builder{ok: numbered}, Store: b.forge, Now: func() time.Time { return time.Unix(1e9, 0) }, Say: func(seat, event string, o Outcome) error { b.said = append(b.said, said{seat, event, o}); return nil }, Finding: func(kind, subject, _ string, raised bool) { b.findings[kind+" "+subject] = raised }} @@ -212,14 +211,6 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) { do func(b *bed) error want string }{ - {"a caller not on publish's list", func(b *bed) error { - _, err := b.h.Publish(NpmSeat, "ace.claude-code", pub("x", "0.1.0", file)) - return err - }, "not on the holder's caller list"}, - {"an unproven caller", func(b *bed) error { - _, err := b.h.Publish(NpmSeat, stdio.Unproven, pub("x", "0.1.0", file)) - return err - }, "no caller the tool runner proved"}, {"a seat the forge does not hold", func(b *bed) error { _, err := b.h.Publish("cargo-package-registry", goBuild, pub("x", "0.1.0", file)) return err @@ -244,7 +235,6 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) { _, err := b.h.PublishPart(GoSeat, nodeBuild, "t-x", digest(file), 0, true, file) return err }, "not open"}, - {"remove by a build module", func(b *bed) error { _, err := b.h.Remove(GoSeat, goBuild, "x", "0.1.1-feat.1", "tag-gone"); return err }, "not on the holder's caller list"}, {"remove of a version with no tag", func(b *bed) error { _, err := b.h.Remove(GoSeat, "novox.mesh-delivery-builder", "x", "0.1.1", "tag-gone") return err @@ -304,29 +294,4 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) { } }) - t.Run("the callers value: its own verbs' lines, the other seat's left to it, unset or none of its own the defaults", func(t *testing.T) { - manifest := `"{\"publish\":[\"mesh-build-go\"],\"remove\":[\"mesh-delivery-builder\"],\"pull\":[\"mesh-delivery-gate\"]}"` - for _, row := range []struct { - raw string - publish []string - fails bool - }{ - {raw: manifest, publish: []string{"mesh-build-go"}}, - {raw: `{"pull":["mesh-delivery-gate"]}`, publish: DefaultCallers()[VerbPublish]}, - {raw: `""`, publish: DefaultCallers()[VerbPublish]}, - {raw: `{"publish":"mesh-build-go"}`, fails: true}, - {raw: `"not json"`, fails: true}, - } { - got, err := OwnCallers([]byte(row.raw)) - if row.fails != (err != nil) { - t.Fatalf("%s: err %v", row.raw, err) - } - if !row.fails && strings.Join(got[VerbPublish], ",") != strings.Join(row.publish, ",") { - t.Errorf("%s: publish %v, want %v", row.raw, got[VerbPublish], row.publish) - } - if _, theirs := got["pull"]; theirs { - t.Errorf("%s: the seat git's line is not the package seats'", row.raw) - } - } - }) } diff --git a/modules/gitea/module.json b/modules/gitea/module.json index dd462ecbe..02c8fcd63 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -248,13 +248,6 @@ "default": [ "jschoubben" ] - }, - "callers": { - "type": "string", - "trusted": true, - "runtime": true, - "default": "{\"publish\":[\"mesh-build-go\",\"mesh-build-node\"],\"publish-part\":[\"mesh-build-go\",\"mesh-build-node\"],\"remove\":[\"mesh-delivery-builder\"],\"versions\":[\"*\"],\"pull\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-gate\"],\"branch\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\"],\"open-pulls\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"tags\":[\"mesh-delivery-forge-bridge\"],\"trees\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"file\":[\"mesh-delivery-forge-bridge\",\"mesh-delivery-checker\",\"mesh-delivery-indexer\"],\"contains\":[\"mesh-delivery-indexer\",\"mesh-delivery-gate\"],\"statuses\":[\"mesh-delivery-gate\"],\"status\":[\"mesh-delivery-forge-bridge\"],\"comment\":[\"mesh-delivery-forge-bridge\"],\"view\":[\"mesh-delivery-forge-bridge\"],\"note\":[\"mesh-delivery-forge-bridge\"],\"merge-pull\":[\"mesh-delivery-forge-bridge\"],\"tag\":[\"mesh-delivery-builder\"]}", - "description": "per verb of the seats the module holds, the callers that may call it (JSON object of verb to callers; * every proved caller, operator the operator's session); each holder reads the lines of its own verbs. The package seats' lines are design 51's (empty is that list for them); the seat git's are design 47's: its reads the parts that declare them in invokes, its acts (status, comment, view, note, merge-pull) the forge bridge alone, and a verb of git with no line is called by no one" } }, "facts": { diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 5715098d3..8ba0d9725 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -2,10 +2,8 @@ // delivery part makes of the forge is a verb of the seat, served here as a tool `git.` against the seat's generated // contract, and no part names this module's own tools. // -// **Who may call which verb** is the holder's trusted config value `callers` (ADR 0627 §2), the one value the module's -// package seats read too: each holder takes the lines of the verbs it serves, so neither refuses the other's. Unset, no -// one may call a verb of `git`: the list is the manifest's, never a list in this code. The caller is the one the tool -// runner proved in the call's `_meta` (mesh-sdk `callers.callerOf`), never an argument. +// **Who may call which verb** is the seat definition's `callers`, which the grants part grants by: the bus refuses a caller a +// definition does not name (design 47 §68, §313, §321), so this holder judges no caller and keeps no list of its own. // // **A failed read answers an error, never an empty answer**: the forge's 404 is the kind `not-found`, its refusal (another // 4xx, or a refusal of this holder) the kind `refused`, a forge that failed or did not answer the kind `failed`; an empty list or an absent file is answered @@ -16,8 +14,6 @@ // (design 47 §314): only into a base whose protection refuses a push, and only that head. import type { ToolDefinition } from "@novox/mesh-sdk/tools"; -import { callerOf, refusal, parseVerbCallers, type VerbCallers } from "@novox/mesh-sdk/callers"; -import { config } from "@novox/mesh-sdk/runtime"; import { seat as GIT, verbs as GIT_VERBS, type BranchAnswer, type ContainsAnswer, type CommentAnswer, type ErrorKind, type FileAnswer, type MergePullAnswer, type Merging, type NoteAnswer, @@ -33,9 +29,6 @@ import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js"; /** The verbs of the seat, in its definition's order. */ export const VERBS: readonly string[] = Object.values(GIT_VERBS); -/** The config value whose lines per verb say who may call it. */ -export const CALLERS_KEY = "callers"; - /** How many tags a page of `tags` holds. */ export const TAGS_PAGE = 50; @@ -50,31 +43,13 @@ export interface HeadStatuses { } /** What the holder reaches beside the forge's API: git in the forge's container (the note), the head's one writer of its - * statuses, and the callers' list. */ + * statuses, and whether the first registrations are written. */ export interface SeatDeps { run: Runner; container: string; statuses: HeadStatuses; /** Whether the first registrations were written (registrations.ts): until then merge-pull refuses every merge (design 47 §95). */ registered: () => boolean; - /** The config value `callers` as the runtime answers it; the runtime's own config read by default. */ - callers?: () => Promise; -} - -/** The lines of `raw` (the config value, an object or that object as a string) for the seat's verbs; unset is none. */ -export function gitCallers(raw: unknown): VerbCallers { - if (raw === undefined || raw === null || raw === "") return {}; - let all: unknown = raw; - if (typeof raw === "string") { - try { - all = JSON.parse(raw); - } catch (err) { - throw new Error(`callers is not a JSON object of verb to callers: ${err instanceof Error ? err.message : String(err)}`); - } - } - if (!all || typeof all !== "object" || Array.isArray(all)) throw new Error("callers is not a JSON object of verb to callers"); - const mine = Object.fromEntries(Object.entries(all as Record).filter(([verb]) => VERBS.includes(verb))); - return parseVerbCallers(JSON.stringify(mine), VERBS, {}); } /** A failure as the contract says it: the forge's or this holder's refusal, or a forge that failed. */ @@ -146,17 +121,14 @@ async function mergingOf(gitea: GiteaClient, owner: string, repo: string, p: Git return { mergeable: p.mergeable, protected: !!rule, required, writes }; } -/** The seat's verbs as tools `git.`, each judging its caller first and answering in the seat's contract. */ +/** The seat's verbs as tools `git.`, each answering in the seat's contract. */ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] { - const listOf = deps.callers ?? (() => config(CALLERS_KEY)); const verb = (name: string, description: string, act: (args: Record) => Promise): ToolDefinition => ({ name: `${GIT}.${name}`, description, input: {}, run: async (args: Record) => { try { - const why = refusal(gitCallers(await listOf()), name, callerOf(args)); - if (why) return failure(new Refused(why.message)); return await act(args); } catch (err) { return failure(err); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index 1474a1aec..d939bcfb9 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -431,19 +431,14 @@ test("the two new tools are named for what mesh_search is asked, and gitea_api s assert.ok("refused" in judgeApiCall("PATCH", "/repos/a/b/pulls/7", { base: "x" })); }); -// **The seat git's verbs judge their caller, and merge-pull judges the Approve's head and no check** (design 47 *The seat -// git*, ADR 0661 §3): a caller the trusted list does not name is refused before the forge is asked; merge-pull merges the +// **merge-pull judges the head it is asked at and no check** (design 47 *The seat git*, §314): merge-pull merges the // head it is named into a base that takes no push, whatever its statuses say, and an id said again after the merge is // answered performed. -test("the seat git's verbs judge their caller, and merge-pull merges the Approve's head and judges no check", async () => { - const { seatTools, gitCallers } = await import("../dist/seat.js"); +test("merge-pull merges the head the caller names and judges no check", async () => { + const { seatTools } = await import("../dist/seat.js"); const { runAs } = await import("@novox/mesh-sdk/callers"); const bridge = "ace.mesh-delivery-forge-bridge"; let registered = true; - const list = JSON.stringify({ publish: ["mesh-build-go"], "merge-pull": ["mesh-delivery-forge-bridge"], - pull: ["mesh-delivery-forge-bridge", "mesh-delivery-gate"], status: ["mesh-delivery-forge-bridge"] }); - assert.deepEqual(Object.keys(gitCallers(list)).sort(), ["merge-pull", "pull", "status"], "only the seat git's lines are its"); - assert.deepEqual(gitCallers(undefined), {}, "unset, no one may call"); // The head's one writer of its statuses (gate.ts), over a forge whose first write is still in flight when the next is asked. const { GateWriter } = await import("../dist/gate.js"); const landed: string[] = []; @@ -457,15 +452,13 @@ test("the seat git's verbs judge their caller, and merge-pull merges the Approve }; const gates = new GateWriter(slow); const seat = seatTools(Object.assign(Object.create(client), { setCommitStatus: slow.setCommitStatus }), - { run: async () => ({ stdout: "", code: 0 }), container: "gitea", callers: async () => list, statuses: gates, registered: () => registered }); + { run: async () => ({ stdout: "", code: 0 }), container: "gitea", statuses: gates, registered: () => registered }); const call = (verb: string, args: Record, caller = bridge) => runAs({ "mesh/caller": caller }, args, () => seat.find((t: any) => t.name === `git.${verb}`)!.run(args)) as Promise; const merge = (extra: Record = {}, caller?: string) => call("merge-pull", { id: "act-1", repository: "novox/mesh-catalog", number: 7, head: REVIEWED, ...extra }, caller); const rows: { name: string; set: () => void; caller?: string; extra?: Record; want: RegExp | "merged" | "held"; loud?: boolean; kind?: string }[] = [ { name: "no registration written yet", set: () => { registered = false; }, want: /no module is registered yet/, kind: "no-registrations" }, - { name: "a caller not on the list", set: () => {}, caller: "ace.mesh-delivery-checker", want: /refused for ace.mesh-delivery-checker/ }, - { name: "an unproven caller", set: () => {}, caller: "", want: /no caller the tool runner proved/ }, { name: "a head that moved", set: () => { state.head = MOVED; }, want: /its head is 3c4d5e6f7a8b, not the abc1234def56/, kind: "head-moved" }, { name: "a base no rule guards", set: () => { state.rules = []; }, want: /guarded by no protection rule/, kind: "unprotected" }, { name: "a base whose protection was removed after the last branch-moved", set: () => { state.rules = [protectedMain()]; state.rules = []; }, want: /guarded by no protection rule/, kind: "unprotected" }, @@ -555,11 +548,10 @@ test("every read and act of the seat git answers its contract, and an error is a }; const run = async (_f: string, args: string[]) => args.includes("show") || args.includes("--ref") && !args.includes("append") ? { stdout: noteShown, code: noteShown ? 0 : 1 } : (noteWritten++, { stdout: "", code: 0 }); - const seat = seatTools(forge as any, { run: run as any, container: "gitea", callers: async () => JSON.stringify({}), statuses: { status: async () => {} }, registered: () => true }); - const everyone = JSON.stringify(Object.fromEntries(seat.map((t: any) => [t.name.slice(4), ["*"]]))); + const seat = seatTools(forge as any, { run: run as any, container: "gitea", statuses: { status: async () => {} }, registered: () => true }); const call = (verb: string, args: Record, over: Record = {}) => { const t = seat.find((x: any) => x.name === `git.${verb}`)!; - const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", callers: async () => everyone, statuses: { status: async () => {} }, registered: () => true }); + const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", statuses: { status: async () => {} }, registered: () => true }); return runAs({ "mesh/caller": "ace.x" }, args, () => wrapped.find((x: any) => x.name === t.name)!.run(args)) as Promise; }; const rows: [string, string, Record, Record, (a: any) => unknown, unknown][] = [ -- 2.54.0 From e6aab55008cb7b56848b76889da937bf7cf123de Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 11:44:59 +0100 Subject: [PATCH 017/243] GIT15: the forge module guesses no base branch for a pull request the forge gave none MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base "main" fallbacks of the open-pulls read and the announcer are gone (the indexer's module name and tool went with GIT10). The manifest's version is MAN1's, written with the fill's source it needs. Builds: design 47 G §7, §428 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/client.ts | 7 ++++--- modules/gitea/index.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/modules/gitea/client.ts b/modules/gitea/client.ts index 78648b3fb..d7402f378 100644 --- a/modules/gitea/client.ts +++ b/modules/gitea/client.ts @@ -413,9 +413,10 @@ export class GiteaClient { const pull = await this.getPullRequest(found.owner, found.name, found.number); if (pull.state !== "open" || !pull.head_sha) continue; // The base's protection as the forge applies it to that branch (appliedProtection): readable without administration. - const base = `${full}:${pull.base || "main"}`; - if (!rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base || "main").catch(() => undefined)); - const rule = rules.get(base); + // A pull request the forge gave no base is not guessed one (design 47 §7): its protection is unknown. + const base = `${full}:${pull.base ?? ""}`; + if (pull.base && !rules.has(base)) rules.set(base, await this.appliedProtection(found.owner, found.name, pull.base).catch(() => undefined)); + const rule = pull.base ? rules.get(base) : undefined; const checks = await this.headChecks(found.owner, found.name, pull.head_sha); const labels = found.labels ?? []; out.push({ owner: found.owner, repo: found.name, number: pull.number, base: pull.base ?? "", head: pull.head, head_sha: pull.head_sha, diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index 37c897c3a..d5ff40722 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -275,7 +275,7 @@ async function announceHead(client: GiteaClient, repo: RepoOf, pull: OpenOf): Pr // where it is: in the build queue from now, until a build machine takes it (novox/hq issue 446). Only where the base // still requires mesh/merge-gate (issue 1301): a main that requires mesh/check is the checker's; // a rule that cannot be read is said pending, as before. - const rule = await client.appliedProtection(repo.owner, repo.name, String(pull.base || "main")).catch(() => undefined); + const rule = pull.base ? await client.appliedProtection(repo.owner, repo.name, String(pull.base)).catch(() => undefined) : undefined; if (writesGate(requiredOf(rule))) await gates.pending(repo.owner, repo.name, sha); return true; } -- 2.54.0 From b8c3f5f09a4c7e8718215dce942ff151be38bffc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:05:00 +0100 Subject: [PATCH 018/243] COND1: the keeper hears findings as events and reads no observer's state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The keeper takes finding-written and finding-cleared from each observer's stream (`*.finding-written`, `*.finding-cleared`, `*.findings-stated` in consumes); its reads of every `findings`, the views.findings consumes and the read_findings invokes go. node-engine's name is a local constant since mesh-sdk's buscompose lost it. Builds: design 47 G §383, §386 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/events.go | 136 ++++++++++++++++++ .../cmd/mesh-conditions/keeper.go | 5 +- .../cmd/mesh-conditions/keeper_test.go | 39 +++++ .../cmd/mesh-conditions/main.go | 18 +-- .../cmd/mesh-conditions/manifest_test.go | 19 +-- .../mesh-conditions/whole_integration_test.go | 71 ++++++--- modules/mesh-conditions/module.json | 28 +--- 7 files changed, 246 insertions(+), 70 deletions(-) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/events.go diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go new file mode 100644 index 000000000..3fbb7fbfd --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -0,0 +1,136 @@ +package main + +// The keeper hears each observer's findings as events on the observer's own stream (design 47 G §383): it reads no +// observer's state. `finding-written` raises or changes the condition of its key, `finding-cleared` clears it. The +// observer and the node are the event's subject (its envelope's key and node); the bus's grants keep another account +// from publishing there, and the keeper refuses a body that names another observer. + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// The events an observer says on its stream (design 47 G §383). +const ( + EventFindingWritten = "finding-written" + EventFindingCleared = "finding-cleared" +) + +// emitterOf is the module an event's envelope key `.` names. +func emitterOf(key, event string) (string, bool) { + m, ok := strings.CutSuffix(key, "."+event) + return m, ok && m != "" && !strings.Contains(m, ".") +} + +// FindingEvent is one `finding-written` or `finding-cleared` heard from an observer on a node. +func (k *Keeper) FindingEvent(e stdio.Envelope) error { + if observer, ok := emitterOf(e.Key, EventFindingWritten); ok { + return k.FindingWritten(observer, e.Node, e.Body) + } + if observer, ok := emitterOf(e.Key, EventFindingCleared); ok { + return k.FindingCleared(observer, e.Node, e.Body) + } + return nil +} + +// isSource says whether the keeper takes findings from the module. +func (k *Keeper) isSource(observer string) bool { return slices.Contains(Observers, observer) } + +// FindingWritten raises or changes the condition of the finding an observer says on a node. A body that is not a finding +// the keeper can raise, or names another observer, is said in the journal and changes nothing. +func (k *Keeper) FindingWritten(observer, node string, body []byte) error { + k.mu.Lock() + k.init() + k.mu.Unlock() + if !k.isSource(observer) { + k.Logf("%s says finding-written on %s, and is no source of findings: left out", observer, node) + return nil + } + var f seat.Finding + if err := json.Unmarshal(body, &f); err != nil { + k.Logf("%s's finding-written on %s is not readable: %v", observer, node, err) + return nil + } + if f.Observer != observer { + k.Logf("%s's finding-written on %s names the observer %q: refused", observer, node, f.Observer) + return nil + } + if f.Scope != f.Subject.Scope || f.ID != f.Subject.ID { + k.Logf("%s's finding-written on %s names the scope %q and id %q, and its subject %q and %q: refused", observer, node, + f.Scope, f.ID, f.Subject.Scope, f.Subject.ID) + return nil + } + v := storedFinding(f) + value, err := json.Marshal(v) + if err != nil { + return err + } + return k.Finding(observer, v.Key, value, false) +} + +// FindingCleared clears the condition of the finding an observer no longer observes. +func (k *Keeper) FindingCleared(observer, node string, body []byte) error { + if !k.isSource(observer) { + k.Logf("%s says finding-cleared on %s, and is no source of findings: left out", observer, node) + return nil + } + var c seat.FindingCleared + if err := json.Unmarshal(body, &c); err != nil { + k.Logf("%s's finding-cleared on %s is not readable: %v", observer, node, err) + return nil + } + k.mu.Lock() + k.init() + key := cc.Key(c.Scope, c.ID, c.Kind) + if _, ok := k.findings[observer][key]; !ok { + // A finding whose key ends in its ref rather than its kind. + for other, f := range k.findings[observer] { + if f.Subject.Scope == c.Scope && f.Subject.ID == c.ID && f.Kind == c.Kind { + key = other + } + } + } + k.mu.Unlock() + return k.Finding(observer, key, nil, true) +} + +// storedFinding is the event's finding as the keeper holds one. +func storedFinding(f seat.Finding) seat.FindingsByKeyValue { + token := f.Ref + if token == "" { + token = f.Kind + } + evidence := f.Evidence + if evidence == nil { + evidence = []seat.Evidence{} + } + return seat.FindingsByKeyValue{V: cc.FindingVersion, Key: cc.Key(f.Scope, f.ID, token), Kind: f.Kind, Subject: f.Subject, + Severity: f.Severity, Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, Needs: f.Needs, + Actions: f.Actions, Evidence: evidence, Source: f.Source, Resolver: f.Resolver, Observer: f.Observer, Owner: f.Owner, + Ref: f.Ref, Run: f.Run, WaitingOn: f.WaitingOn, At: f.At} +} + +// hear subscribes to the observers' events once the runtime serves, asked again until it takes each. +func hear(k *Keeper) { + for _, event := range []string{EventFindingWritten, EventFindingCleared} { + pattern := "*." + event + go func() { + <-stdio.Served() + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + err := stdio.Subscribe(pattern, k.FindingEvent) + if err == nil { + return + } + logf("%s", fmt.Sprintf("%s is not heard yet (%v); asked again in %s", pattern, err, wait)) + time.Sleep(wait) + } + }() + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 006aac7aa..1a5341e32 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -20,7 +20,6 @@ import ( assignerclient "git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient" mesh "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/motion" cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" @@ -36,6 +35,8 @@ const ( healersModule = "mesh-healers" // The holders of the assignments and the catalog, whose views and findings the keeper follows (the manifest test holds // these equal to the manifest's `consumes` and `invokes`). + // engineModule is the node-engine, an observer of §387 (design 47 G §387 names mesh-host). + engineModule = "mesh-host" assignerModule = "mesh-delivery-assigner" indexerModule = "mesh-delivery-indexer" ) @@ -69,7 +70,7 @@ var Observers = []string{watchmanModule, selfCheckModule, healersModule, assigne "node-tools", "claude-code", // The node-engine's: a node that has read no seat list five minutes after its start, seat-list-unread (ADR 0636 §1; // issue 1966: written on ace and raised by nobody). - buscompose.EngineModule} + engineModule} // The other states it reads, as `.`. const ( diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 80b893b05..cfa1e6f96 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -163,6 +163,22 @@ func finding(t *testing.T, observer, scope, id, kind, source string, sev string, return f.Key, raw } +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + raw, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return raw +} + +// eventFinding is the machine g14's silent finding as an observer says it in finding-written. +func eventFinding(observer string, at time.Time) seat.Finding { + return seat.Finding{Scope: cc.ScopeMachine, ID: "g14", Kind: "silent", Subject: seat.Subject{Scope: cc.ScopeMachine, ID: "g14"}, + Severity: cc.Warning, Summary: "silent of g14", Headline: "silent", Explanation: "It is silent.", Source: "S1", + Observer: observer, At: at, Since: at, Evidence: []seat.Evidence{{At: at, Said: "silent"}}} +} + // ADR 0564 clauses 2, 3, 7, 8 (issue 1952 C2, C3): a finding raises, changes and clears its condition; a blind observer // clears nothing; the healers' attempts reach the condition; a reopening within ten // minutes is the same fault; a silence is a transition in history. @@ -229,6 +245,29 @@ func TestTheKeeperRaisesChangesAndClearsFromFindings(t *testing.T) { } return "" }}, + // Design 47 G §383: the same findings heard as events on the observer's stream. + {name: "finding-cleared clears", do: func() error { + return r.k.FindingCleared(watchman, "ace", mustJSON(t, seat.FindingCleared{At: r.now, ID: "g14", Kind: "silent", Scope: cc.ScopeMachine, Why: "back"})) + }, said: "cleared machine.g14.silent", check: func(_ Condition, open bool) string { + if open { + return "still open" + } + return "" + }}, + {name: "finding-written raises", do: func() error { + return r.k.FindingWritten(watchman, "ace", mustJSON(t, eventFinding(watchman, r.now))) + }, said: "reopened machine.g14.silent"}, + {name: "a body naming another observer changes nothing", do: func() error { + return r.k.FindingWritten(watchman, "ace", mustJSON(t, eventFinding(selfCheck, r.now))) + }}, + {name: "a module that is no source changes nothing", do: func() error { + return r.k.FindingWritten("stranger", "ace", mustJSON(t, eventFinding("stranger", r.now))) + }}, + {name: "changed words change the condition", do: func() error { + f := eventFinding(watchman, r.now) + f.Needs = "a cable" + return r.k.FindingWritten(watchman, "ace", mustJSON(t, f)) + }, said: "words machine.g14.silent"}, } for _, s := range steps { if err := s.do(); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index d57315678..cbec7f8c2 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -1,8 +1,8 @@ // mesh-conditions: the condition keeper, holder of the seat mesh-conditions (novox/hq ADR 0564 clauses 1-3, 9, 13). A // Go bundle served by a runtime of its own, on its own bus account (`runs-as`), so that its account alone writes its // states: `conditions` (one key per open condition), `history` (every transition, ninety days), `takeover` (the key the -// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It watches every observer's -// `findings`, the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the +// readers follow), `held` and `bounds` (what a delivery holds, and for how long). It hears every observer's +// findings as events (design 47 G §383), watches the healers' `attempts`, the assigner's wants, the nodes' haves and the controller's lease and, until the // lease write, its `conditions`; it says every transition on its seat with the controller's body. stdout is the MCP // channel; what this module says, it says on stderr. package main @@ -18,7 +18,6 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" - seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -122,11 +121,7 @@ func Keep(k *Keeper, v *views.Reader) { watch(k, v, AttemptsRead, healersModule+" attempts", func(c stdio.StateChange) error { return said(k.Attempt(c.Key, c.Value, c.Op != "put")) }) - for _, o := range Observers { - watch(k, v, o+"."+seat.StateFindings, o, func(c stdio.StateChange) error { - return said(k.Finding(o, c.Key, c.Value, c.Op != "put")) - }) - } + hear(k) watch(k, v, BoundsState, "", func(c stdio.StateChange) error { if c.Op == stdio.OpExpire { return said(k.BoundPassed(c.Key)) @@ -143,19 +138,14 @@ func Keep(k *Keeper, v *views.Reader) { const ( WantsTool = assignerModule + ".assigner_read_wants" AssignmentsTool = assignerModule + ".assigner_read_assignments" - FindingsTool = assignerModule + ".assigner_read_findings" - // IndexerFindingsTool is the indexer's read tool of its findings: the indexer's state, too, is asked of its owner. - IndexerFindingsTool = indexerModule + ".indexer_read_findings" ) -// NewViews is the keeper's pictures of the assigner's wants, assignments and findings and of the indexer's findings. A first view it cannot get past its +// NewViews is the keeper's pictures of the assigner's wants and assignments (the findings come as events, design 47 G §383). A first view it cannot get past its // bound is a finding of its own, taken in as any observer's: the keeper raises the condition itself. func NewViews(k *Keeper) *views.Reader { v := &views.Reader{Module: Module, Logf: logf, Findings: ownFindings{k}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} v.Follow(assignerModule, "wants", WantsTool) v.Follow(assignerModule, "assignments", AssignmentsTool) - v.Follow(assignerModule, seat.StateFindings, FindingsTool) - v.Follow(indexerModule, seat.StateFindings, IndexerFindingsTool) return v } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 3750c0d33..fb4a6fe85 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" @@ -32,19 +33,19 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { !slices.Equal(Verbs, catalogue.VerbNames(def.Serves)) { t.Errorf("module %s runs as %s and claims %+v; the def serves %v", m.Module, m.RunsAs, m.Claims, catalogue.VerbNames(def.Serves)) } - // The assigner's wants, assignments and findings, and the indexer's findings, are asked once of their read tools and - // followed by their events, not read (ADR 0615 §1). + // The assigner's wants and assignments are asked once of their read tools and followed by their events (ADR 0615 §1); + // the observers' findings are heard as events and no `findings` is read (design 47 G §383). reads := []string{HavesRead, AttemptsRead} - for _, o := range Observers { - if o != assignerModule && o != indexerModule { - reads = append(reads, o+"."+seat.StateFindings) - } - } - if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, FindingsTool, IndexerFindingsTool, "seat:mesh-hand-acts.hand-act"}) || + if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act"}) || !slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments", - "mesh-delivery-assigner.views.findings", "mesh-delivery-indexer.views.findings"}) { + "*.finding-written", "*.finding-cleared", "*.findings-stated"}) { t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes) } + for _, r := range m.Reads { + if strings.HasSuffix(r, "."+seat.StateFindings) { + t.Errorf("the keeper reads %s, and hears findings as events", r) + } + } slices.Sort(reads) got := slices.Clone(m.Reads) slices.Sort(got) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go index 7abe372d3..36292d779 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -51,6 +51,28 @@ type player struct { in io.Writer next int pending map[string]chan wire + // subscribed is each pattern the keeper asked its events for. + subscribed chan string +} + +// event hands the keeper one event as the runtime does, answered once its handlers took it. +func (p *player) event(key, node string, body any) error { + p.mu.Lock() + p.next++ + id := fmt.Sprintf("e-%d", p.next) + ch := make(chan wire, 1) + p.pending[id] = ch + p.mu.Unlock() + p.send(map[string]any{"id": id, "method": "mesh/event", "params": map[string]any{"envelope": map[string]any{"key": key, "node": node, "body": body}}}) + select { + case m := <-ch: + if m.Error != nil { + return errors.New(m.Error.Message) + } + return nil + case <-time.After(30 * time.Second): + return fmt.Errorf("%s was not taken in 30s", key) + } } func (p *player) send(m map[string]any) { @@ -197,7 +219,15 @@ func (p *player) handle(m wire) { case "mesh/seat.publish": p.answer(m.ID, map[string]any{"sequence": 1}, p.nc.Publish(a.Subject, a.Body)) case "mesh/subscribe": + var sub struct { + Pattern string `json:"pattern"` + } + _ = json.Unmarshal(m.Params, &sub) p.answer(m.ID, map[string]any{}, nil) + select { + case p.subscribed <- sub.Pattern: + default: + } default: p.answer(m.ID, nil, errors.New("nats: no responders available for request")) } @@ -242,8 +272,8 @@ func (p *player) call(name string, args map[string]any) (json.RawMessage, error) } } -// 1952/C6: an observer's finding raises a condition on the bus, said with the controller's body, and its deletion clears -// it. +// 1952/C6, design 47 G §383: an observer's finding-written raises a condition on the bus, said with the controller's body, +// and its finding-cleared clears it. func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) @@ -264,9 +294,6 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { } buckets := []string{"mesh-controller_conditions", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_assignments", "mesh-healers_attempts"} - for _, o := range Observers { - buckets = append(buckets, o+"_findings") - } for _, s := range []string{ConditionsState, HistoryState, HeldState, BoundsState} { buckets = append(buckets, Module+"_"+s) } @@ -288,25 +315,29 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = stdin.Close(); cancel(); _ = cmd.Wait() }) - p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}} + p := &player{ctx: ctx, nc: nc, js: js, in: stdin, pending: map[string]chan wire{}, subscribed: make(chan string, 32)} go p.serve(stdout) // The first call serves; the watches start after it. if _, err := p.call(InputsTool, map[string]any{}); err != nil { t.Fatal(err) } - time.Sleep(time.Second) - at := time.Now().UTC() - f := seat.FindingsByKeyValue{Key: "machine.g14.silent", Kind: "silent", Subject: seat.Subject{Scope: cc.ScopeMachine, ID: "g14", - Machine: "g14"}, Severity: cc.Warning, Summary: "g14 is silent", Headline: "g14 is offline", - Explanation: "g14 said nothing for three minutes.", Resolved: "g14 is back", Source: "S1", - Observer: watchmanModule, At: at, Evidence: []seat.Evidence{{At: at, Said: "no presence for 3m"}}} - raw, err := encodeFinding(f) - if err != nil { - t.Fatal(err) + heard := map[string]bool{} + for deadline := time.After(20 * time.Second); !(heard["*."+EventFindingWritten] && heard["*."+EventFindingCleared]); { + select { + case got := <-p.subscribed: + heard[got] = true + case <-deadline: + t.Fatalf("the keeper asked for %v, not both finding events", heard) + } } - findings, _ := js.KeyValue(ctx, watchmanModule+"_findings") - if _, err := findings.Put(ctx, f.Key, raw); err != nil { + at := time.Now().UTC() + f := seat.Finding{Scope: cc.ScopeMachine, ID: "g14", Kind: "silent", Subject: seat.Subject{Scope: cc.ScopeMachine, ID: "g14", + Machine: "g14"}, Severity: cc.Warning, Summary: "g14 is silent", Headline: "g14 is offline", + Explanation: "g14 said nothing for three minutes.", Source: "S1", Observer: watchmanModule, At: at, Since: at, + Evidence: []seat.Evidence{{At: at, Said: "no presence for 3m"}}} + key := cc.Key(f.Scope, f.ID, f.Kind) + if err := p.event(watchmanModule+"."+EventFindingWritten, "ace", f); err != nil { t.Fatal(err) } msg, err := events.NextMsg(10 * time.Second) @@ -315,14 +346,14 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { } var e Event if err := json.Unmarshal(msg.Data, &e); err != nil || msg.Subject != "mesh.seat.mesh-conditions.event.condition-raised" || - e.Key != f.Key || e.Change != cc.ChangeRaised || e.Show != cc.ShowKeeper(f.Key) || e.Headline != f.Headline { + e.Key != key || e.Change != cc.ChangeRaised || e.Show != cc.ShowKeeper(key) || e.Headline != f.Headline { t.Fatalf("said %s on %s", msg.Data, msg.Subject) } listed, err := p.call(Seat+".conditions", map[string]any{}) if err != nil || !strings.Contains(string(listed), `"key":"machine.g14.silent"`) { t.Fatalf("conditions answered %s, %v", listed, err) } - if err := findings.Delete(ctx, f.Key); err != nil { + if err := p.event(watchmanModule+"."+EventFindingCleared, "ace", seat.FindingCleared{At: at, ID: f.ID, Kind: f.Kind, Scope: f.Scope, Why: "back"}); err != nil { t.Fatal(err) } msg, err = events.NextMsg(10 * time.Second) @@ -330,7 +361,7 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { t.Fatalf("no condition-cleared was said: %v", err) } conditions, _ := js.KeyValue(ctx, Module+"_"+ConditionsState) - if _, err := conditions.Get(ctx, f.Key); !errors.Is(err, jetstream.ErrKeyNotFound) { + if _, err := conditions.Get(ctx, key); !errors.Is(err, jetstream.ErrKeyNotFound) { t.Errorf("the cleared condition is still kept: %v", err) } } diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index f6803f865..7f1a0c112 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -14,42 +14,20 @@ } ], "reads": [ - "mesh-delivery-watchman.findings", - "mesh-self-check.findings", - "mesh-healers.findings", - "messenger.findings", - "mesh-provisioner.findings", - "artifact-store-tools.findings", - "mesh-config.findings", - "mesh-vault.findings", - "mesh-watcher.findings", - "minio.findings", - "nfs-server.findings", - "gitea.findings", - "mesh-build-go.findings", - "mesh-delivery-builder.findings", - "mesh-delivery-checker.findings", - "mesh-delivery-teller.findings", - "mesh-delivery-view.findings", - "mesh-enrolment.findings", - "node-tools.findings", - "claude-code.findings", - "mesh-host.findings", "mesh-healers.attempts", "node-engine.have" ], "invokes": [ "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-assigner.assigner_read_assignments", - "mesh-delivery-assigner.assigner_read_findings", - "mesh-delivery-indexer.indexer_read_findings", "seat:mesh-hand-acts.hand-act" ], "consumes": [ "mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments", - "mesh-delivery-assigner.views.findings", - "mesh-delivery-indexer.views.findings" + "*.finding-written", + "*.finding-cleared", + "*.findings-stated" ], "state": [ "conditions", -- 2.54.0 From b9a224dc99348203c944d50af8cd042505ebbbef Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:05:11 +0100 Subject: [PATCH 019/243] NATS2: the bus module plans each sender's sent store, 256 MiB, 7 days, refusing when full MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The store is made by Keeper.Sends, kept when a verdict asks nothing of it, and a full one says `full`. Which module sends is NATS3's input (the catalog half); this makes the store. Builds: design 47 G §59, §137, §344, §477 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/nats/internal/keeper/jetstream.go | 2 +- modules/nats/internal/keeper/keeper.go | 21 +++++++++++++++- modules/nats/internal/keeper/keeper_test.go | 27 +++++++++++++++++++++ modules/nats/internal/keeper/plan.go | 23 ++++++++++++++++++ 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/modules/nats/internal/keeper/jetstream.go b/modules/nats/internal/keeper/jetstream.go index 53e960db9..31e4ab8d8 100644 --- a/modules/nats/internal/keeper/jetstream.go +++ b/modules/nats/internal/keeper/jetstream.go @@ -74,7 +74,7 @@ func (b JetStream) Create(ctx context.Context, o Object) error { LimitMarkerTTL: limitMarker(o.Settings.KeyTTL)}) case KindObjects: _, err = b.JS.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{Bucket: o.Name, MaxBytes: o.Settings.MaxBytes, - Storage: jetstream.FileStorage, Replicas: 1}) + TTL: time.Duration(o.Settings.MaxAgeSeconds) * time.Second, Storage: jetstream.FileStorage, Replicas: 1}) default: _, err = b.JS.CreateStream(ctx, streamConfig(o)) } diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 962ae0ea4..073671dd4 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -392,7 +392,8 @@ func (k *Keeper) Heard(a Accepted) { } for _, local := range sortedKeys(held.Made) { m := held.Made[local] - if asked[local] || m.Orphaned { + if asked[local] || m.Orphaned || local == SentLocal { + // The sent store is no manifest's ask: it is made by Sends and goes only by drop (§59, §344). continue } if !m.Created { @@ -411,6 +412,24 @@ func (k *Keeper) Heard(a Accepted) { k.write(held) } +// Sends makes module's sent store where the bus lacks it (§59, §344): called when the catalog half and the seat list +// say the module serves a verb or emits an event, and again on the change that gives a holder its first verb or event. +// A module that no longer sends keeps its store: an object in it expires by age, and the store goes only by drop. +func (k *Keeper) Sends(module string) { + held, _, found := k.asked(module) + if !found { + held = Asked{V: 1, Module: module, At: k.now()} + } + if held.Made == nil { + held.Made = map[string]Made{} + } + m := held.Made[SentLocal] + m.Plan, m.Orphaned = SentPlanOf(module), false + k.ensure(&held, &m) + held.Made[SentLocal] = m + k.write(held) +} + // makeEntry makes one asked entry where the bus lacks it, compares it where it holds it, and says what it did. func (k *Keeper) makeEntry(held *Asked, e catalogue.BusStateEntry) { module := held.Module diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 7c7f3735a..735ed9a4d 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -254,6 +254,8 @@ func TestEachTemplateIsPlannedUnderTheModulesNames(t *testing.T) { binding: "stream mesh-knowledge_reviews on mesh.bus.mesh-knowledge.reviews"}, {name: "a local name holding the separator", entry: entry("my_drafts", catalogue.StorageKeyValue, catalogue.TemplateState, nil), refused: `the local name "my_drafts" is empty or holds "_"`}, + {name: "a local name that is the sent store's", entry: entry("sent-objects", catalogue.StorageObjectStore, catalogue.TemplateObjects, map[string]string{"max-mib": "1"}), + refused: "the bus module's own for the sent store mesh-knowledge_sent-objects"}, {name: "objects with no cap", entry: entry("files", catalogue.StorageObjectStore, catalogue.TemplateObjects, nil), refused: "gives no max-mib"}, {name: "a bare stream with no when-full", entry: entry("raw", catalogue.StorageStream, "", nil), @@ -347,6 +349,31 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { } } }}, + {"a sender gets its sent store: 256 MiB, 7 days, refusing when full, and it is kept when a verdict asks nothing of it", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { + k.Sends("m") + k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 3, BusState: []catalogue.BusStateEntry{drafts}}) + k.Sends("m") + if want := "[OBJ_m_sent-objects KV_m_drafts]"; fmt.Sprint(bus.created) != want { + t.Fatalf("created %v, want %s", bus.created, want) + } + if s := bus.held["OBJ_m_sent-objects"]; s.MaxBytes != 256*MiB || s.MaxAgeSeconds != 7*24*3600 || s.Discard != DiscardNew { + t.Errorf("the sent store is %+v, want 256 MiB, 7 days, refusing when full", s) + } + if want := "[created m sent-objects created m drafts]"; fmt.Sprint(out.events) != want { + t.Errorf("said %v, want %s", out.events, want) + } + if b := out.bodies[0].Binding; b.ObjectStore != "m_sent-objects" { + t.Errorf("binding %+v names no sent store", b) + } + bus.fill["OBJ_m_sent-objects"] = 200 * uint64(MiB) + k.FillPass() + if got := out.events[len(out.events)-1]; got != "full m sent-objects" { + t.Errorf("a full sent store said %q, want full", got) + } + if held := k.Held()["m"]; held.Made[SentLocal].Orphaned || len(bus.deleted) != 0 { + t.Errorf("the sent store was orphaned or removed: %+v %v", held.Made[SentLocal], bus.deleted) + } + }}, {"a replay not newer is not applied", func(t *testing.T, k *Keeper, bus *fakeBus, out *said) { k.Heard(Accepted{Module: "m", Digest: "d2", Revision: 9, BusState: []catalogue.BusStateEntry{drafts}}) k.Heard(Accepted{Module: "m", Digest: "d1", Revision: 4, BusState: nil}) diff --git a/modules/nats/internal/keeper/plan.go b/modules/nats/internal/keeper/plan.go index 419331328..080d61755 100644 --- a/modules/nats/internal/keeper/plan.go +++ b/modules/nats/internal/keeper/plan.go @@ -40,6 +40,14 @@ const ( EventsDays = 7 ) +// The sent store of a module that sends (design 47 G §59, §477): the local name its bus name `_sent-objects` is made +// of, its cap and the age after which the bus expires an object in it. +const ( + SentLocal = "sent-objects" + SentCapMiB = 256 + SentAgeDays = 7 +) + // Settings are one bus object's settings, as asked and as held: what check compares. type Settings struct { // MaxBytes is its cap in bytes; MaxAgeSeconds its age, 0 for none. @@ -109,6 +117,9 @@ func StreamSubject(module, local string) string { return "mesh.bus." + module + // type and template that do not go together, an option of the wrong shape. The judge refuses each of these first // (ADR 0595 §8); this is the keeper's own guard, so a manifest judged by an older judge is refused, never half-made. func PlanOf(module string, e catalogue.BusStateEntry) (Plan, error) { + if e.Name == SentLocal { + return Plan{}, fmt.Errorf("the local name %q is the bus module's own for the sent store %s (design 47 G §59)", e.Name, BusName(module, SentLocal)) + } if e.Name == "" || strings.Contains(e.Name, Separator) || strings.ContainsAny(e.Name, ".*> ") { return Plan{}, fmt.Errorf("the local name %q is empty or holds %q, a dot, a wildcard or a space: the mesh names it %s on the bus, and two modules' names must never meet", e.Name, Separator, BusName(module, "")) } @@ -242,6 +253,18 @@ func PlanOf(module string, e catalogue.BusStateEntry) (Plan, error) { return p, nil } +// SentPlanOf is the plan of module's sent store (§59): an object store `_sent-objects` of 256 MiB (§477) that +// refuses when full and keeps an object 7 days, after which the bus expires it. It is transport and not kept data, so +// its expiry is no removal that needs a person's word. It is made with no manifest field (§344). +func SentPlanOf(module string) Plan { + name := BusName(module, SentLocal) + e := catalogue.BusStateEntry{Name: SentLocal, Storage: catalogue.StorageObjectStore, Template: catalogue.TemplateObjects, + Options: map[string]json.RawMessage{catalogue.OptionCapMiB: json.RawMessage(strconv.Itoa(SentCapMiB))}} + return Plan{Module: module, Local: SentLocal, Storage: e.Storage, Template: e.Template, Entry: e, + Binding: membership.BusBinding{Local: SentLocal, Storage: e.Storage, Template: e.Template, ObjectStore: name}, + Objects: []Object{{Kind: KindObjects, Name: name, Settings: Settings{MaxBytes: SentCapMiB * MiB, MaxAgeSeconds: days(SentAgeDays), Discard: DiscardNew}}}} +} + func stateSettings(o options) (Settings, error) { history, err := o.int(catalogue.OptionPastValues, 1) if err != nil { -- 2.54.0 From bbd9f638a03ac60f4476d1acc158d55ea9bf59c8 Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 19:56:32 +0100 Subject: [PATCH 020/243] Move the restore of a bus archive into bussnap so the keeper restores a kept data too (milestone G, W14) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../nats/cmd/mesh-nats-snapshot/bussnap.go | 2 + .../nats/cmd/mesh-nats-snapshot/restore.go | 114 ---------------- modules/nats/internal/bussnap/restore.go | 126 ++++++++++++++++++ 3 files changed, 128 insertions(+), 114 deletions(-) create mode 100644 modules/nats/internal/bussnap/restore.go diff --git a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go index e4d8a8f59..444dd213f 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go +++ b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go @@ -14,6 +14,8 @@ var ( StreamNames = bussnap.StreamNames request = bussnap.Request truncate = bussnap.Truncate + RestoreAll = bussnap.RestoreAll + heldTo = bussnap.HeldTo ) const ( diff --git a/modules/nats/cmd/mesh-nats-snapshot/restore.go b/modules/nats/cmd/mesh-nats-snapshot/restore.go index 5d5866094..f98bbc536 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/restore.go +++ b/modules/nats/cmd/mesh-nats-snapshot/restore.go @@ -4,10 +4,8 @@ import ( "context" "crypto/rand" "encoding/hex" - "encoding/json" "errors" "fmt" - "io" "net" "os" "os/exec" @@ -20,118 +18,6 @@ import ( "nats-tools/internal/busarchive" ) -// restoreChunk is what is sent at a time; the server answers each before the next goes. -const restoreChunk = 64 * 1024 - -// RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds -// each restored stream's state to the manifest. **Nothing is restored over anything**: a server that -// already holds any of the streams is refused before the first is sent. -func RestoreAll(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, log func(string, ...any)) error { - var present []string - for _, s := range a.Manifest.Streams { - err := request(ctx, nc, apiStreamInfo+s.Name, nil, nil) - var api *apiError - switch { - case err == nil: - present = append(present, s.Name) - case errors.As(err, &api) && api.Code == 404: - default: - return err - } - } - if len(present) > 0 { - return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+ - "store (--into) and swap it in with the bus stopped", strings.Join(present, ", ")) - } - for _, s := range a.Manifest.Streams { - started := time.Now() - if err := restoreStream(ctx, nc, a, s); err != nil { - return fmt.Errorf("stream %s: %w", s.Name, err) - } - log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds()) - } - return nil -} - -func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error { - meta, archive, err := a.Open(s) - if err != nil { - return err - } - defer archive.Close() - var req map[string]json.RawMessage - if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { - return errors.New("its meta is not a stream's configuration and state") - } - var resp struct { - DeliverSubject string `json:"deliver_subject"` - } - if err := request(ctx, nc, apiStreamRestore+s.Name, req, &resp); err != nil { - return err - } - if resp.DeliverSubject == "" { - return errors.New("the server gave nowhere to send the archive") - } - buf := make([]byte, restoreChunk) - for { - n, err := archive.Read(buf) - if n > 0 { - cctx, cancel := context.WithTimeout(ctx, 30*time.Second) - reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n]) - cancel() - if rerr != nil { - return fmt.Errorf("sending the archive: %w", rerr) - } - if len(reply.Data) > 0 { - return fmt.Errorf("the server refused the archive: %s", truncate(reply.Data)) - } - } - if err == io.EOF { - break - } - if err != nil { - return err - } - } - // The end: an empty message, answered once the server has rebuilt the stream from what it was - // sent — which for a large stream takes a while. - fctx, cancel := context.WithTimeout(ctx, 30*time.Minute) - defer cancel() - final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil) - if err != nil { - return fmt.Errorf("finishing the restore: %w", err) - } - var created struct { - Error *apiError `json:"error"` - State streamState `json:"state"` - } - if err := json.Unmarshal(final.Data, &created); err != nil { - return fmt.Errorf("the server's last answer is not JSON: %q", truncate(final.Data)) - } - if created.Error != nil { - return created.Error - } - return heldTo(created.State, s) -} - -// heldTo is whether a restored stream holds at least what its snapshot said it held. -// -// **At least, not exactly — found by the restore test, not assumed.** The state the server answers a -// snapshot request with is taken when the snapshot starts; the stream goes on taking messages while -// its blocks are read out, and a block read a moment later carries some of what was appended in that -// moment. So a stream written during its snapshot restores with a later last sequence than the -// manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of -// the messages in that tail some may be there and some not. Every message up to the manifest's last -// sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message -// by message. What must never happen is a restored stream that ends before the manifest said it did. -func heldTo(restored streamState, s busarchive.StreamEntry) error { - if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) { - return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", - restored.Messages, restored.LastSeq, s.Messages, s.LastSeq) - } - return nil -} - // RestoreOffline fills a new store directory with the archive's streams, by a server of this // program's own: nats-server, the same binary the bus runs, on loopback, with the bus's one account // and a user that exists only for this run. It is stopped when the streams are in. What it leaves diff --git a/modules/nats/internal/bussnap/restore.go b/modules/nats/internal/bussnap/restore.go new file mode 100644 index 000000000..484ddf22e --- /dev/null +++ b/modules/nats/internal/bussnap/restore.go @@ -0,0 +1,126 @@ +package bussnap + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + "time" + + "github.com/nats-io/nats.go" + "nats-tools/internal/busarchive" +) + +// restoreChunk is what is sent at a time; the server answers each before the next goes. +const restoreChunk = 64 * 1024 + +// RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds +// each restored stream's state to the manifest. **Nothing is restored over anything**: a server that +// already holds any of the streams is refused before the first is sent. +func RestoreAll(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, log func(string, ...any)) error { + var present []string + for _, s := range a.Manifest.Streams { + err := Request(ctx, nc, APIStreamInfo+s.Name, nil, nil) + var api *APIError + switch { + case err == nil: + present = append(present, s.Name) + case errors.As(err, &api) && api.Code == 404: + default: + return err + } + } + if len(present) > 0 { + return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+ + "store (--into) and swap it in with the bus stopped", strings.Join(present, ", ")) + } + for _, s := range a.Manifest.Streams { + started := time.Now() + if err := restoreStream(ctx, nc, a, s); err != nil { + return fmt.Errorf("stream %s: %w", s.Name, err) + } + log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds()) + } + return nil +} + +func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error { + meta, archive, err := a.Open(s) + if err != nil { + return err + } + defer archive.Close() + var req map[string]json.RawMessage + if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { + return errors.New("its meta is not a stream's configuration and state") + } + var resp struct { + DeliverSubject string `json:"deliver_subject"` + } + if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil { + return err + } + if resp.DeliverSubject == "" { + return errors.New("the server gave nowhere to send the archive") + } + buf := make([]byte, restoreChunk) + for { + n, err := archive.Read(buf) + if n > 0 { + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n]) + cancel() + if rerr != nil { + return fmt.Errorf("sending the archive: %w", rerr) + } + if len(reply.Data) > 0 { + return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data)) + } + } + if err == io.EOF { + break + } + if err != nil { + return err + } + } + // The end: an empty message, answered once the server has rebuilt the stream from what it was + // sent — which for a large stream takes a while. + fctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil) + if err != nil { + return fmt.Errorf("finishing the restore: %w", err) + } + var created struct { + Error *APIError `json:"error"` + State StreamState `json:"state"` + } + if err := json.Unmarshal(final.Data, &created); err != nil { + return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data)) + } + if created.Error != nil { + return created.Error + } + return HeldTo(created.State, s) +} + +// HeldTo is whether a restored stream holds at least what its snapshot said it held. +// +// **At least, not exactly — found by the restore test, not assumed.** The state the server answers a +// snapshot request with is taken when the snapshot starts; the stream goes on taking messages while +// its blocks are read out, and a block read a moment later carries some of what was appended in that +// moment. So a stream written during its snapshot restores with a later last sequence than the +// manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of +// the messages in that tail some may be there and some not. Every message up to the manifest's last +// sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message +// by message. What must never happen is a restored stream that ends before the manifest said it did. +func HeldTo(restored StreamState, s busarchive.StreamEntry) error { + if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) { + return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", + restored.Messages, restored.LastSeq, s.Messages, s.LastSeq) + } + return nil +} -- 2.54.0 From c2e017a4c1c4e664070336241562c477dac019cd Mon Sep 17 00:00:00 2001 From: jochen Date: Mon, 9 Nov 2026 20:00:36 +0100 Subject: [PATCH 021/243] The bus module takes and restores a restore point of a module's kept data, beside the live data (milestone G, W14.2) Seat verbs snapshot-kept, restore-kept, discard-aside and aside, handed to the keeper; state kept; tool nats_kept. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/nats/README.md | 12 + modules/nats/cmd/nats-tools/keep.go | 6 +- .../cmd/nats-tools/kept_integration_test.go | 153 +++++++ modules/nats/cmd/nats-tools/keptools.go | 171 ++++++++ modules/nats/cmd/nats-tools/keptools_test.go | 77 ++++ modules/nats/cmd/nats-tools/main.go | 9 +- modules/nats/cmd/nats-tools/manifest_test.go | 14 +- modules/nats/internal/keeper/keeper.go | 3 + modules/nats/internal/keeper/kept.go | 382 ++++++++++++++++++ .../nats/internal/keeper/kept_jetstream.go | 128 ++++++ modules/nats/internal/keeper/kept_test.go | 298 ++++++++++++++ modules/nats/internal/keeper/requests.go | 32 ++ modules/nats/module.json | 16 +- 13 files changed, 1287 insertions(+), 14 deletions(-) create mode 100644 modules/nats/cmd/nats-tools/kept_integration_test.go create mode 100644 modules/nats/cmd/nats-tools/keptools.go create mode 100644 modules/nats/cmd/nats-tools/keptools_test.go create mode 100644 modules/nats/internal/keeper/kept.go create mode 100644 modules/nats/internal/keeper/kept_jetstream.go create mode 100644 modules/nats/internal/keeper/kept_test.go diff --git a/modules/nats/README.md b/modules/nats/README.md index 104e79e20..ec6b5700f 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -136,6 +136,18 @@ a put of the assigner (novox/hq ADR 0546, ADR 0565 clauses 8 and 9). In this ord tested) and the manifest's shape (programs mounted read-only, named in `restart-on`, no image built from a Dockerfile). +## A module's kept data, backed up and restored with its delivery (milestone G) + +The keeper takes, on its own account, a restore point of one module's kept data on the bus before the module moves across a +change of that kept data's `form`, and restores it beside the live data when a want takes the module back (design 47, *A +module's kept data on the bus*). Its seat verbs, served by the tools and handed to the keeper as requests: +`mesh-broker.snapshot-kept` (the data as the server's snapshot, the module's consumers' positions and the catch-up tool, in +one directory under the keeper's state), `mesh-broker.restore-kept` (the live data is snapshotted into a set-aside +directory under a name of its own and only then removed; the restore point is put in its place and the consumers on other +streams are set back), `mesh-broker.discard-aside` (only on a call that says why, recorded first) and `mesh-broker.aside`; +the tool `nats_kept` reads the restore points and the set-aside data from the keeper's state `kept`. The backup holder +keeps the restore points; it calls these verbs through the seat. + ## Files it does not know Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 6576c1284..57c852c18 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -236,8 +236,10 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer defer nc.Close() recorder := meshhandactsclient.Recorder{Ask: conn.Call} k := &keeper.Keeper{ - Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))}, - Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), + Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))}, + Kept: keeper.KeptJetStream{NC: nc, JS: js, Logf: logger.Printf}, + KeptDir: getenv("MESH_NATS_SNAPSHOTS_DIR"), + Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), Act: func(what, why, cause string) error { _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) return err diff --git a/modules/nats/cmd/nats-tools/kept_integration_test.go b/modules/nats/cmd/nats-tools/kept_integration_test.go new file mode 100644 index 000000000..8f1c0523c --- /dev/null +++ b/modules/nats/cmd/nats-tools/kept_integration_test.go @@ -0,0 +1,153 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "nats-tools/internal/bussnap" + "nats-tools/internal/keeper" +) + +// mapStore is the keeper's own state in memory. +type mapStore struct { + mu sync.Mutex + kv map[string][]byte +} + +func (s *mapStore) Get(state, key string) ([]byte, uint64, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + v, ok := s.kv[state+"/"+key] + return v, 1, ok, nil +} +func (s *mapStore) Put(state, key string, v []byte) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.kv[state+"/"+key] = v + return 1, nil +} +func (s *mapStore) Delete(state, key string) error { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.kv, state+"/"+key) + return nil +} +func (s *mapStore) Keys(string) ([]string, error) { return nil, nil } +func (s *mapStore) Create(state, key string, v []byte) (uint64, error) { + return 0, errors.New("not used") +} + +// A module's kept data on the real bus is backed up and restored beside the live data (design 47, "A module's kept data on +// the bus, backed up and restored with its delivery"): the bucket as the server's own snapshot, the module's consumer on +// another stream at its position; after a change, a restore brings the bucket and the position back, and the live bucket +// is kept whole in the data set aside. +func TestAKeptDataIsRestoredBesideTheLiveDataOnTheRealBus(t *testing.T) { + s, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(5 * time.Second) { + t.Fatal("the server did not start") + } + defer s.Shutdown() + nc, err := nats.Connect(s.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer nc.Close() + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx := context.Background() + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "idx_modules"}) + if err != nil { + t.Fatal(err) + } + forge, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "forge", Subjects: []string{"forge.>"}}) + if err != nil { + t.Fatal(err) + } + for i := 0; i < 10; i++ { + if _, err := js.Publish(ctx, "forge.merged", []byte("m")); err != nil { + t.Fatal(err) + } + } + cons, err := forge.CreateConsumer(ctx, jetstream.ConsumerConfig{Durable: "ace_idx", AckPolicy: jetstream.AckExplicitPolicy}) + if err != nil { + t.Fatal(err) + } + batch, err := cons.Fetch(4) + if err != nil { + t.Fatal(err) + } + for m := range batch.Messages() { + _ = m.DoubleAck(ctx) + } + if _, err := kv.PutString(ctx, "ace", "fingerprint-1"); err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + var acts []string + k := &keeper.Keeper{Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, dir)}, Store: &mapStore{kv: map[string][]byte{}}, + Kept: keeper.KeptJetStream{NC: nc, JS: js}, KeptDir: dir, + Act: func(what, why, cause string) error { acts = append(acts, what); return nil }} + take := &keeper.Request{Verb: keeper.VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Why: "before form 1"} + if _, err := k.SnapshotKept(ctx, take); err != nil { + t.Fatal(err) + } + // The form-1 build rewrites the kept data, and its consumer moves on. + if _, err := kv.PutString(ctx, "ace", "fingerprint-2-form-1"); err != nil { + t.Fatal(err) + } + batch, err = cons.Fetch(5) + if err != nil { + t.Fatal(err) + } + for m := range batch.Messages() { + _ = m.DoubleAck(ctx) + } + + restore := &keeper.Request{Verb: keeper.VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: take.Dir, Why: "revert"} + if _, err := k.RestoreKept(ctx, restore); err != nil { + t.Fatal(err) + } + kv, err = js.KeyValue(ctx, "idx_modules") + if err != nil { + t.Fatal(err) + } + if e, err := kv.Get(ctx, "ace"); err != nil || string(e.Value()) != "fingerprint-1" { + t.Errorf("the restored bucket holds %v (%v), want fingerprint-1", e, err) + } + if restore.CatchUp != "indexer_take_in" { + t.Errorf("catch-up %q", restore.CatchUp) + } + forge, _ = js.Stream(ctx, "forge") + ci, err := forge.Consumer(ctx, "ace_idx") + if err != nil || ci.CachedInfo().Config.OptStartSeq != 5 { + t.Errorf("the consumer is %+v (%v): it starts after the 4 messages it had acknowledged", ci.CachedInfo().Config, err) + } + kept, err := os.ReadDir(filepath.Join(dir, "aside")) + if err != nil || len(kept) != 1 { + t.Fatalf("aside holds %v (%v)", kept, err) + } + if _, err := os.Stat(filepath.Join(dir, "aside", kept[0].Name(), keeper.ArchiveFile)); err != nil { + t.Error("the live data set aside is not kept whole") + } + if len(acts) != 2 { + t.Errorf("hand acts %v: the restore point and the restore", acts) + } +} diff --git a/modules/nats/cmd/nats-tools/keptools.go b/modules/nats/cmd/nats-tools/keptools.go new file mode 100644 index 000000000..e5efeeede --- /dev/null +++ b/modules/nats/cmd/nats-tools/keptools.go @@ -0,0 +1,171 @@ +package main + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "nats-tools/internal/keeper" +) + +// keptWait is how long a call to take or restore a kept data waits for the keeper's end on the watch of its requests: under +// the runtime's own bound on a call, so a long snapshot is answered as "handed, not ended" and read again with its request. +const keptWait = 50 * time.Second + +// KeptHeld reads the keeper's state kept: the restore points it took and the data it set aside. +type KeptHeld func() ([]keeper.Point, []keeper.Aside, error) + +// KeptInState is the state kept through the runtime that launched this bundle. +func KeptInState() ([]keeper.Point, []keeper.Aside, error) { + st := stdio.State(keeper.StateKept) + keys, err := st.Keys() + if err != nil { + return nil, nil, err + } + sort.Strings(keys) + var points []keeper.Point + var aside []keeper.Aside + for _, key := range keys { + e, err := st.Get(key) + if err != nil || e == nil { + continue + } + switch { + case strings.HasPrefix(key, "point."): + var p keeper.Point + if json.Unmarshal(e.Value, &p) == nil { + points = append(points, p) + } + case strings.HasPrefix(key, "aside."): + var a keeper.Aside + if json.Unmarshal(e.Value, &a) == nil { + aside = append(aside, a) + } + } + } + return points, aside, nil +} + +func formOf(a map[string]any) int { + f, _ := a["form"].(float64) + return int(f) +} + +// keptTools are the seat mesh-broker's verbs for a module's kept data and the bus module's tool that reads what it holds: +// each verb hands its request to the keeper, which acts on its own account, and waits for the end on the watch of its +// requests (never a poll). Asked again with `request`, a verb reads the end of the request it made. +func keptTools(rs requestStore, held KeptHeld) []stdio.Tool { + entry := func(extra map[string]any) map[string]any { + in := map[string]any{"module": str("the module whose kept data it is, e.g. mesh-delivery-indexer"), "local-name": str("the kept data's local name, e.g. modules"), + "form": num("the kept data's form (a manifest field; absent reads 0)"), "fingerprint": str("the fingerprint the restore point was taken before"), + "why": str("why, in plain words: kept with the act"), "request": str("a request this verb made before: its end is read, nothing new is asked")} + for k, v := range extra { + in[k] = v + } + return in + } + run := func(verb string, build func(a map[string]any, r *keeper.Request) error) func(a map[string]any) (any, error) { + return func(a map[string]any) (any, error) { + name := "mesh-broker." + verb + if id := arg(a, "request"); id != "" { + r, err := rs.Get(id) + if err != nil || r == nil { + return nil, fmt.Errorf("no request %s is held (%v)", id, err) + } + if r.Done.IsZero() && rs.Ended != nil { + if got := rs.Ended(id, keptWait); got != nil { + return got, nil + } + if again, err := rs.Get(id); err == nil && again != nil { + r = again + } + } + return r, nil + } + r := keeper.Request{Verb: verb, Why: strings.TrimSpace(arg(a, "why")), Caller: stdio.CallerOf(a), At: rs.Now()} + if err := build(a, &r); err != nil { + return nil, err + } + if r.Why == "" { + return nil, fmt.Errorf("%s says why", name) + } + r.ID = requestID(r.At) + if err := rs.Put(r.ID, r); err != nil { + return nil, fmt.Errorf("the request cannot be handed to the keeper: %w", err) + } + if rs.Ended != nil { + if got := rs.Ended(r.ID, keptWait); got != nil { + return got, nil + } + } + return map[string]any{"request": r.ID, "state": "handed to the keeper, not ended yet", "read": fmt.Sprintf("%s with request %s", name, r.ID)}, nil + } + } + point := func(a map[string]any, r *keeper.Request) error { + r.Module, r.Local, r.Form, r.Fingerprint = arg(a, "module"), arg(a, "local-name"), formOf(a), arg(a, "fingerprint") + if r.Module == "" || r.Local == "" || r.Fingerprint == "" { + return fmt.Errorf("a restore point is of one kept data: a module, its local name and the fingerprint it is taken before") + } + return nil + } + readAside := func() (any, error) { + _, aside, err := held() + if err != nil { + return nil, err + } + if aside == nil { + aside = []keeper.Aside{} + } + return map[string]any{"aside": aside}, nil + } + return []stdio.Tool{ + {Name: "mesh-broker.snapshot-kept", + Description: "Take the restore point of one module's kept data on the bus (milestone G, design 47): the data whole as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool the newer manifest declares, kept by the bus module's keeper. Answers the directory the restore point is in. Handed to the keeper, recorded as a hand act.", + Input: entry(map[string]any{"catch-up": str("the tool a restore is followed by, from the newer manifest, kept in the restore point")}), + Run: run(keeper.VerbSnapshotKept, func(a map[string]any, r *keeper.Request) error { + r.CatchUp = arg(a, "catch-up") + return point(a, r) + })}, + {Name: "mesh-broker.restore-kept", + Description: "Restore one module's kept data on the bus from a restore point beside the live data, never over it (design 47): the live data is snapshotted into a set-aside directory under a name of its own and then removed from the bus, the restore point's streams are put in its place and the module's other consumers are set back to their positions. Answers what was set aside and the catch-up tool the restore point carries. Handed to the keeper, recorded as a hand act.", + Input: entry(map[string]any{"from": str("the directory the restore point was restored to, an absolute path")}), + Run: run(keeper.VerbRestoreKept, func(a map[string]any, r *keeper.Request) error { + r.From = arg(a, "from") + if r.From == "" { + return fmt.Errorf("a restore names the directory the restore point is in") + } + return point(a, r) + })}, + {Name: "mesh-broker.discard-aside", + Description: "Remove the data a restore set aside, by its id (from `aside`), only on a person's call: it says who and why, and the keeper records the hand act before it removes anything.", + Input: map[string]any{"id": str("the id of the data set aside"), "why": str("why, in plain words: kept with the act"), "request": str("a request made before: its end is read")}, + Run: run(keeper.VerbDiscardAside, func(a map[string]any, r *keeper.Request) error { + r.From = arg(a, "id") + if r.From == "" { + return fmt.Errorf("discard-aside names the id of the data set aside") + } + return nil + })}, + {Name: "mesh-broker.aside", Description: "The data restores set aside, each with its module, kept data, size, when, who and why: what a person decides to discard.", + Run: func(map[string]any) (any, error) { return readAside() }}, + {Name: "nats_kept", + Description: "What the bus module holds of its modules' kept data besides the live data: every restore point it took (module, kept data, form, the fingerprint it was taken before, its consumers' positions, the catch-up tool, its size) and the data a restore set aside. Read from the keeper's state kept.", + Run: func(map[string]any) (any, error) { + points, aside, err := held() + if err != nil { + return nil, err + } + if points == nil { + points = []keeper.Point{} + } + if aside == nil { + aside = []keeper.Aside{} + } + return map[string]any{"points": points, "aside": aside}, nil + }}, + } +} diff --git a/modules/nats/cmd/nats-tools/keptools_test.go b/modules/nats/cmd/nats-tools/keptools_test.go new file mode 100644 index 000000000..4205dc0f1 --- /dev/null +++ b/modules/nats/cmd/nats-tools/keptools_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + + "nats-tools/internal/keeper" +) + +// The seat mesh-broker's verbs for a module's kept data hand their request to the keeper as the proved caller, wait for +// its end on the watch of the requests, and say what a restore set aside (design 47, "A module's kept data on the bus"). +func TestTheKeptDataVerbsAreHandedToTheKeeperAndReadBack(t *testing.T) { + held := map[string]keeper.Request{} + rs := requestStore{ + Put: func(key string, r keeper.Request) error { held[key] = r; return nil }, + Get: func(key string) (*keeper.Request, error) { + r, ok := held[key] + if !ok { + return nil, nil + } + return &r, nil + }, + Now: func() time.Time { return time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) }, + // The keeper's end, as its watch hands it. + Ended: func(id string, within time.Duration) *keeper.Request { + r := held[id] + r.Done, r.Outcome, r.Dir = time.Unix(1, 0), "done", "/keeper/points/idx/modules/f0-abc" + held[id] = r + return &r + }, + } + aside := []keeper.Aside{{ID: "idx.modules.1", Module: "idx", Local: "modules", Bytes: 12}} + tools := map[string]stdio.Tool{} + for _, tl := range keptTools(rs, func() ([]keeper.Point, []keeper.Aside, error) { return nil, aside, nil }) { + tools[tl.Name] = tl + } + for _, verb := range KeptVerbs { + if _, ok := tools["mesh-broker."+verb]; !ok { + t.Errorf("the verb mesh-broker.%s is not served", verb) + } + } + out, err := tools["mesh-broker.snapshot-kept"].Run(map[string]any{"module": "idx", "local-name": "modules", "form": float64(1), + "fingerprint": "abc", "catch-up": "indexer_take_in", "why": "before form 1", "caller": "person.mallory"}) + if err != nil { + t.Fatal(err) + } + if r := out.(*keeper.Request); r.Verb != keeper.VerbSnapshotKept || r.Form != 1 || r.Fingerprint != "abc" || r.CatchUp != "indexer_take_in" || + r.Caller != stdio.Unproven || r.Dir == "" { + t.Errorf("answered %+v: the request for form 1 before abc, for the caller the runtime proved, with the directory the keeper answered", r) + } + for name, args := range map[string]map[string]any{ + "mesh-broker.snapshot-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc"}, + "mesh-broker.restore-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc", "why": "revert"}, + "mesh-broker.discard-aside": {"why": "x"}, + } { + if _, err := tools[name].Run(args); err == nil { + t.Errorf("%s with %v was handed to the keeper", name, args) + } + } + if out, err := tools["mesh-broker.restore-kept"].Run(map[string]any{"module": "idx", "local-name": "modules", "fingerprint": "abc", + "from": "/restored/f0-abc", "why": "revert"}); err != nil || out.(*keeper.Request).From != "/restored/f0-abc" { + t.Errorf("restore answered %+v (%v)", out, err) + } + read, err := tools["mesh-broker.aside"].Run(nil) + if err != nil || !strings.Contains(strings.ReplaceAll(strings.TrimSpace(toJSON(read)), " ", ""), `"id":"idx.modules.1"`) { + t.Errorf("aside answered %v (%v)", read, err) + } +} + +func toJSON(v any) string { + b, _ := json.Marshal(v) + return string(b) +} diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 034c5be3b..1e277fdd5 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -166,7 +166,10 @@ func main() { // bus holds against what was asked, and what no manifest asks for. Its acting verbs (create, apply, recreate, drop) are // the keeper's own work, on its own account alone (keeper.go), so no other account, the tool runner's included, holds // them. -var ReadingVerbs = []string{"list", "show", "check", "orphans"} +var ReadingVerbs = []string{"list", "show", "check", "orphans", "aside"} + +// KeptVerbs are the seat mesh-broker's verbs for a module's kept data (keptools.go): each handed to the keeper. +var KeptVerbs = []string{"snapshot-kept", "restore-kept", "discard-aside"} // Events are the seat mesh-broker's events (ADR 0595 §6, ADR 0620 §2): said on the seat's subjects on the shared events stream // and kept in full on the bus module's own stream broker-events (BROKER_EVENTS), which `history` reads (keep.go). @@ -190,5 +193,7 @@ func allTools() []stdio.Tool { out = append(out, brokerTools(HeldInState, jsz, StreamHistory(LiveBus()))...) out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) - return append(out, requestTools(RequestsInState())...) + requests := RequestsInState() + out = append(out, requestTools(requests)...) + return append(out, keptTools(requests, KeptInState)...) } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index 2394a9eb8..d6639fdf3 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -189,8 +189,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { if err := json.Unmarshal(raw, &nm); err != nil { t.Fatal(err) } - if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, ReadingVerbs) { - t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v alone", nm.Claims, ReadingVerbs) + if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, append(slices.Clone(ReadingVerbs), KeptVerbs...)) { + t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and the keeper-handed verbs %v alone", nm.Claims, ReadingVerbs, KeptVerbs) } if !slices.Equal(nm.Consumes, []string{"mesh-catalog.manifest-accepted"}) { t.Errorf("consumes %v: the seat mesh-catalog's manifest-accepted alone (ADR 0595 §4)", nm.Consumes) @@ -224,8 +224,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { } // spent, the warrants the keeper spent, is written by the keeper alone and only read by the tools (review round 2, // security F2: the tool runner's account, which writes requests, may never delete a spend record). - if !slices.Equal(states, []string{"asked", "requests", "spent"}) || !slices.Equal(readOnly, []string{"spent"}) { - t.Errorf("state is %v, read-only %v: asked and requests, old sections (ADR 0595 §10), and spent, read-only", states, readOnly) + if !slices.Equal(states, []string{"asked", "requests", "spent", "kept"}) || !slices.Equal(readOnly, []string{"spent"}) { + t.Errorf("state is %v, read-only %v: asked and requests, old sections (ADR 0595 §10), spent, read-only, and kept, the restore points taken", states, readOnly) } if _, ok := nm.OwnSecrets["broker-keeper"]; !ok { t.Error("own-secrets lack broker-keeper, the keeper's own account (ADR 0436 §1)") @@ -234,9 +234,9 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { bus, _ := keeper["bus"].(map[string]any) health, _ := keeper["health"].(map[string]any) if keeper["type"] != "process" || keeper["artifact"] != "tools" || fmt.Sprint(keeper["run"]) != "[./nats-tools keep]" || - fmt.Sprint(bus["state"]) != "[asked requests spent]" || + fmt.Sprint(bus["state"]) != "[asked requests spent kept]" || health["kind"] != "tool" || health["tool"] != "nats_keeper_health" { - t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked, requests and spent, judged by nats_keeper_health", keeper) + t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked, requests, spent and kept, judged by nats_keeper_health", keeper) } // Its bus declares keeps, the one key more ADR 0626 §1 allows it, read through the SDK as the grant composition reads it. var whole catalogue.Manifest @@ -263,7 +263,7 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("the manifest lists the tool %s, which the bundle does not serve", name) } } - for _, verb := range ReadingVerbs { + for _, verb := range append(slices.Clone(ReadingVerbs), KeptVerbs...) { if !served["mesh-broker."+verb] { t.Errorf("the bundle does not serve the seat verb mesh-broker.%s", verb) } diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index 073671dd4..bde7cd12e 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -219,6 +219,9 @@ type Keeper struct { Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) Now func() time.Time Logf func(string, ...any) + // Kept takes and restores a module's kept data, in KeptDir (kept.go); nil where the process has no such bus. + Kept KeptBus + KeptDir string // Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default. Raise, Clear int64 diff --git a/modules/nats/internal/keeper/kept.go b/modules/nats/internal/keeper/kept.go new file mode 100644 index 000000000..e4caf5a5e --- /dev/null +++ b/modules/nats/internal/keeper/kept.go @@ -0,0 +1,382 @@ +package keeper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "nats-tools/internal/busarchive" +) + +// A module's kept data on the bus, backed up before the module moves and restored beside the live data when a want takes +// it back (milestone G, design 47 "A module's kept data on the bus, backed up and restored with its delivery"). The +// keeper takes it on its own account, as it takes a snapshot before a recreate (ADR 0235 §5): the streams of one kept +// data as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool +// the newer manifest declares, in one directory the backup holder keeps as a restore point. A restore never goes over +// the live data: the live data is first snapshotted into a set-aside directory under a name of its own, and only then +// removed from the bus; only discard-aside removes the set-aside copy, on a person's call. + +const ( + // VerbSnapshotKept takes a restore point of one kept data; VerbRestoreKept restores one beside the live data. + VerbSnapshotKept = "snapshot-kept" + VerbRestoreKept = "restore-kept" + // VerbDiscardAside removes the data a restore set aside, by its id. + VerbDiscardAside = "discard-aside" +) + +// StateKept is the keeper's own state of the restore points it took and the data it set aside (never their bytes, which +// are files in the keeper's directory): `point...
.` and `aside.`. Read by the +// bus module's tool nats_kept and by the seat mesh-broker's `aside`. +const StateKept = "kept" + +// PointFile and ArchiveFile are the two files of a restore point's directory; AsideFile of a set-aside directory. +const ( + PointFile = "point.json" + ArchiveFile = "bus.tar" + AsideFile = "aside.json" +) + +// Position is where one durable consumer of the module stood on a stream: the stream sequence it had acknowledged up to, +// and its configuration as the server held it, so a restore can make it again at that place. +type Position struct { + Stream string `json:"stream"` + Consumer string `json:"consumer"` + AckFloor uint64 `json:"ack-floor"` + Config json.RawMessage `json:"config"` +} + +// KeptBus is what the keeper needs of the bus beyond Bus to take and restore a kept data. +type KeptBus interface { + // Existing answers which of the stream names the bus holds. + Existing(ctx context.Context, names []string) ([]string, error) + // Positions are the module's durable consumers on every stream (a consumer is the module's when it is named for it). + Positions(ctx context.Context, module string) ([]Position, error) + // SetPositions makes each consumer again from the place it stood. + SetPositions(ctx context.Context, ps []Position) error + // RestoreArchive puts every stream of an archive into the bus, which must hold none of them. + RestoreArchive(ctx context.Context, a *busarchive.Archive) error + // DeleteStreams removes streams by name. + DeleteStreams(ctx context.Context, names []string) error +} + +// Point is a restore point's description, its directory's point.json. +type Point struct { + Module string `json:"module"` + Local string `json:"local"` + Form int `json:"form"` + Fingerprint string `json:"fingerprint"` + CatchUp string `json:"catch-up,omitempty"` + Streams []string `json:"streams"` + Positions []Position `json:"positions"` + At time.Time `json:"at"` + Bytes int64 `json:"bytes"` +} + +// Aside is data a restore set aside: the live kept data as it was when the restore removed it. +type Aside struct { + ID string `json:"id"` + Module string `json:"module"` + Local string `json:"local"` + Streams []string `json:"streams"` + At time.Time `json:"at"` + Bytes int64 `json:"bytes"` + By string `json:"by"` + Why string `json:"why"` + Dir string `json:"dir"` +} + +// PointKey is a restore point's key in StateKept. +func PointKey(module, local string, form int, fingerprint string) string { + return fmt.Sprintf("point.%s.%s.%d.%s", module, local, form, fingerprint) +} + +// safeName is one path component made of what a module, a kept data and a fingerprint are made of. +func safeName(s string) bool { + return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00 *>.") +} + +func (k *Keeper) pointDir(module, local string, form int, fingerprint string) string { + return filepath.Join(k.KeptDir, "points", module, local, fmt.Sprintf("f%d-%s", form, fingerprint)) +} + +// streamsOf is the streams one kept data is: its plan's objects where the keeper made it, else the streams the bus holds +// under its bus name (a foundation part's own section, made by the grants part). +func (k *Keeper) streamsOf(ctx context.Context, module, local string) ([]string, error) { + if held, _, found := k.asked(module); found { + if m, made := held.Made[local]; made && len(m.Plan.Objects) > 0 { + var out []string + for _, o := range m.Plan.Objects { + out = append(out, o.Stream()) + } + return out, nil + } + } + name := BusName(module, local) + got, err := k.Kept.Existing(ctx, []string{"KV_" + name, "OBJ_" + name, name}) + if err != nil { + return nil, err + } + if len(got) == 0 { + return nil, fmt.Errorf("the bus holds no kept data %s of %s (looked for KV_%s, OBJ_%s and %s)", local, module, name, name, name) + } + return got, nil +} + +func (k *Keeper) keptReady(r *Request) error { + if k.Kept == nil || k.KeptDir == "" { + return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored") + } + if !safeName(r.Module) || !safeName(r.Local) || !safeName(r.Fingerprint) { + return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint) + } + return nil +} + +// SnapshotKept takes the restore point of one kept data: the file pair in a directory named for its form and fingerprint, +// written whole before its name is given, and its record in StateKept. It answers the directory. +func (k *Keeper) SnapshotKept(ctx context.Context, r *Request) (string, error) { + if err := k.keptReady(r); err != nil { + return "", err + } + streams, err := k.streamsOf(ctx, r.Module, r.Local) + if err != nil { + return "", err + } + // The positions first: a consumer that moves while the snapshot is taken is then a little behind its archive, and a + // restored consumer that is behind hears a message twice, never misses one. + positions, err := k.Kept.Positions(ctx, r.Module) + if err != nil { + return "", fmt.Errorf("the consumers' positions of %s cannot be read, so no restore point is taken: %w", r.Module, err) + } + tar, err := k.Bus.Snapshot(ctx, streams) + if err != nil { + return "", fmt.Errorf("the snapshot of %s's %s failed, so no restore point is taken: %w", r.Module, r.Local, err) + } + dir := k.pointDir(r.Module, r.Local, r.Form, r.Fingerprint) + if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil { + return "", err + } + partial := dir + ".partial" + _ = os.RemoveAll(partial) + if err := os.Mkdir(partial, 0o700); err != nil { + return "", err + } + if err := os.Rename(tar, filepath.Join(partial, ArchiveFile)); err != nil { + _ = os.RemoveAll(partial) + return "", fmt.Errorf("the snapshot cannot be kept as a restore point: %w", err) + } + st, _ := os.Stat(filepath.Join(partial, ArchiveFile)) + p := Point{Module: r.Module, Local: r.Local, Form: r.Form, Fingerprint: r.Fingerprint, CatchUp: r.CatchUp, + Streams: streams, Positions: positions, At: k.now()} + if st != nil { + p.Bytes = st.Size() + } + raw, _ := json.MarshalIndent(p, "", " ") + if err := os.WriteFile(filepath.Join(partial, PointFile), append(raw, '\n'), 0o600); err != nil { + _ = os.RemoveAll(partial) + return "", err + } + // A restore point taken again at the same form and fingerprint replaces the earlier one, whole. + _ = os.RemoveAll(dir) + if err := os.Rename(partial, dir); err != nil { + return "", err + } + val, _ := json.Marshal(p) + if _, err := k.Store.Put(StateKept, PointKey(r.Module, r.Local, r.Form, r.Fingerprint), val); err != nil { + k.logf("the record of the restore point %s cannot be written to its own %s: %v", dir, StateKept, err) + } + r.Dir = dir + k.record(r, fmt.Sprintf("took the restore point of %s's %s at form %d before %s (%d stream(s), %d consumer position(s), %d bytes)", + r.Module, r.Local, r.Form, r.Fingerprint, len(streams), len(positions), p.Bytes)) + return fmt.Sprintf("restore point of %s's %s at form %d before %s in %s", r.Module, r.Local, r.Form, r.Fingerprint, dir), nil +} + +// readPoint reads and checks a restore point's directory: its point.json names what the request names, its files are +// plain files (never a link), and its archive matches its own manifest. +func readPoint(dir string, r *Request) (*Point, *busarchive.Archive, error) { + for _, name := range []string{PointFile, ArchiveFile} { + fi, err := os.Lstat(filepath.Join(dir, name)) + if err != nil { + return nil, nil, fmt.Errorf("the restore point %s holds no %s: %w", dir, name, err) + } + if !fi.Mode().IsRegular() { + return nil, nil, fmt.Errorf("%s of the restore point %s is not a plain file", name, dir) + } + } + raw, err := os.ReadFile(filepath.Join(dir, PointFile)) + if err != nil { + return nil, nil, err + } + var p Point + if err := json.Unmarshal(raw, &p); err != nil { + return nil, nil, fmt.Errorf("%s of %s does not read: %w", PointFile, dir, err) + } + if p.Module != r.Module || p.Local != r.Local || p.Form != r.Form || p.Fingerprint != r.Fingerprint { + return nil, nil, fmt.Errorf("the directory %s holds the restore point of %s's %s at form %d before %s, not of %s's %s at form %d before %s", + dir, p.Module, p.Local, p.Form, p.Fingerprint, r.Module, r.Local, r.Form, r.Fingerprint) + } + f, err := os.Open(filepath.Join(dir, ArchiveFile)) + if err != nil { + return nil, nil, err + } + defer f.Close() + a, err := busarchive.ReadArchive(f, "") + if err != nil { + return nil, nil, fmt.Errorf("the archive of the restore point %s does not read: %w", dir, err) + } + if err := a.Verify(); err != nil { + a.Close() + return nil, nil, fmt.Errorf("the archive of the restore point %s does not match its own manifest, so nothing is restored: %w", dir, err) + } + return &p, a, nil +} + +// RestoreKept restores one kept data beside the live data: the live data is snapshotted into a set-aside directory first +// and only then removed from the bus, and the restore point's streams are put in its place; the consumers the module +// reads with on other streams are set back to their positions. It answers what was set aside, where. +func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) { + if err := k.keptReady(r); err != nil { + return "", err + } + if !filepath.IsAbs(r.From) { + return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From) + } + p, a, err := readPoint(filepath.Clean(r.From), r) + if err != nil { + return "", err + } + defer a.Close() + live, err := k.Kept.Existing(ctx, p.Streams) + if err != nil { + return "", err + } + id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405")) + var aside *Aside + if len(live) > 0 { + // Aside before anything is removed: a snapshot that fails leaves the live data where it is. + tar, err := k.Bus.Snapshot(ctx, live) + if err != nil { + return "", fmt.Errorf("the live %s of %s could not be set aside, so nothing was removed: %w", r.Local, r.Module, err) + } + dir := filepath.Join(k.KeptDir, "aside", id) + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", err + } + if err := os.Rename(tar, filepath.Join(dir, ArchiveFile)); err != nil { + return "", fmt.Errorf("the live data's snapshot cannot be kept: %w", err) + } + st, _ := os.Stat(filepath.Join(dir, ArchiveFile)) + aside = &Aside{ID: id, Module: r.Module, Local: r.Local, Streams: live, At: k.now(), By: r.Caller, Why: r.Why, Dir: dir} + if st != nil { + aside.Bytes = st.Size() + } + raw, _ := json.MarshalIndent(aside, "", " ") + if err := os.WriteFile(filepath.Join(dir, AsideFile), append(raw, '\n'), 0o600); err != nil { + return "", err + } + val, _ := json.Marshal(aside) + if _, err := k.Store.Put(StateKept, "aside."+id, val); err != nil { + return "", fmt.Errorf("what is set aside cannot be recorded, so nothing was removed: %w", err) + } + if err := k.Kept.DeleteStreams(ctx, live); err != nil { + return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err) + } + } + if err := k.Kept.RestoreArchive(ctx, a); err != nil { + msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err) + if aside != nil { + if back, rerr := k.putBack(ctx, aside); rerr != nil { + msg += fmt.Sprintf("; the live data, set aside in %s, could not be put back either: %v", aside.Dir, rerr) + } else { + msg += fmt.Sprintf("; the live data is back as it was (%s)", back) + } + } + return "", errors.New(msg) + } + // The consumers on streams the restore point did not carry (the snapshot of a stream carries its own consumers). + var apply []Position + for _, pos := range p.Positions { + if !slices.Contains(p.Streams, pos.Stream) { + apply = append(apply, pos) + } + } + if len(apply) > 0 { + if err := k.Kept.SetPositions(ctx, apply); err != nil { + return "", fmt.Errorf("the kept data is restored, but the consumers' positions could not be set back: %w", err) + } + } + r.Dir, r.CatchUp = "", p.CatchUp + where := "nothing was live, so nothing was set aside" + if aside != nil { + r.Dir = aside.Dir + where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir) + } + k.record(r, fmt.Sprintf("restored %s's %s from the restore point at form %d before %s beside the live data; %s", r.Module, r.Local, r.Form, r.Fingerprint, where)) + return fmt.Sprintf("restored %s's %s from its restore point; %s; %d consumer position(s) set back; catch-up tool %q", r.Module, r.Local, where, len(apply), p.CatchUp), nil +} + +// putBack restores set-aside data to the bus, after a restore failed. +func (k *Keeper) putBack(ctx context.Context, a *Aside) (string, error) { + f, err := os.Open(filepath.Join(a.Dir, ArchiveFile)) + if err != nil { + return "", err + } + defer f.Close() + arch, err := busarchive.ReadArchive(f, "") + if err != nil { + return "", err + } + defer arch.Close() + if err := k.Kept.DeleteStreams(ctx, a.Streams); err != nil { + return "", err + } + if err := k.Kept.RestoreArchive(ctx, arch); err != nil { + return "", err + } + return "from " + a.Dir, nil +} + +// DiscardAside removes the data a restore set aside, by its id (Request.From), on a person's call: the request says who +// and why, and is recorded as a hand act before the data goes. +func (k *Keeper) DiscardAside(r *Request) (string, error) { + if k.KeptDir == "" { + return "", errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR)") + } + id := strings.TrimSpace(r.From) + if !safeAsideID(id) { + return "", fmt.Errorf("%q is not the id of data set aside", id) + } + if strings.TrimSpace(r.Why) == "" { + return "", errors.New("discarding set-aside data says why") + } + raw, _, found, err := k.Store.Get(StateKept, "aside."+id) + if err != nil { + return "", err + } + if !found { + return "nothing done", fmt.Errorf("no data is set aside as %s", id) + } + var a Aside + _ = json.Unmarshal(raw, &a) + k.record(r, fmt.Sprintf("discarded the data set aside as %s (%s's %s, %d bytes)", id, a.Module, a.Local, a.Bytes)) + if r.Unrecorded != "" { + return "nothing done", fmt.Errorf("the hand act cannot be recorded, so the set-aside data stays: %s", r.Unrecorded) + } + if err := os.RemoveAll(filepath.Join(k.KeptDir, "aside", id)); err != nil { + return "partly done", err + } + if err := k.Store.Delete(StateKept, "aside."+id); err != nil { + return "partly done", err + } + return fmt.Sprintf("discarded the data set aside as %s", id), nil +} + +func safeAsideID(id string) bool { + return id != "" && !strings.ContainsAny(id, "/\\\x00 *>") && !strings.HasPrefix(id, ".") +} diff --git a/modules/nats/internal/keeper/kept_jetstream.go b/modules/nats/internal/keeper/kept_jetstream.go new file mode 100644 index 000000000..8a3042f0e --- /dev/null +++ b/modules/nats/internal/keeper/kept_jetstream.go @@ -0,0 +1,128 @@ +package keeper + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "nats-tools/internal/busarchive" + "nats-tools/internal/bussnap" +) + +// KeptJetStream is KeptBus on the keeper's own connection. +type KeptJetStream struct { + NC *nats.Conn + JS jetstream.JetStream + Logf func(string, ...any) +} + +// Existing answers which of the names are streams the bus holds. +func (b KeptJetStream) Existing(ctx context.Context, names []string) ([]string, error) { + var out []string + for _, n := range names { + _, err := b.JS.Stream(ctx, n) + switch { + case err == nil: + out = append(out, n) + case errors.Is(err, jetstream.ErrStreamNotFound): + default: + return nil, err + } + } + return out, nil +} + +// ConsumerOf is whether a durable consumer is the module's: named for it, alone or after the machine that runs it +// (`_`, process.Account.QueueConsumerName). +func ConsumerOf(consumer, module string) bool { + return consumer == module || strings.HasSuffix(consumer, "_"+module) +} + +// Positions are the module's durable consumers on every stream, each with the stream sequence it has acknowledged up to. +func (b KeptJetStream) Positions(ctx context.Context, module string) ([]Position, error) { + var out []Position + streams := b.JS.ListStreams(ctx) + var names []string + for s := range streams.Info() { + names = append(names, s.Config.Name) + } + if err := streams.Err(); err != nil { + return nil, err + } + for _, name := range names { + st, err := b.JS.Stream(ctx, name) + if err != nil { + return nil, err + } + list := st.ListConsumers(ctx) + for ci := range list.Info() { + if !ConsumerOf(ci.Name, module) { + continue + } + cfg, err := json.Marshal(ci.Config) + if err != nil { + return nil, err + } + out = append(out, Position{Stream: name, Consumer: ci.Name, AckFloor: ci.AckFloor.Stream, Config: cfg}) + } + if err := list.Err(); err != nil { + return nil, err + } + } + return out, nil +} + +// SetPositions makes each consumer again, from the sequence after the one it had acknowledged. A consumer's place cannot +// be edited, so it is removed and made with its own configuration and a new start. +func (b KeptJetStream) SetPositions(ctx context.Context, ps []Position) error { + for _, p := range ps { + st, err := b.JS.Stream(ctx, p.Stream) + if err != nil { + return fmt.Errorf("the stream %s of %s's position cannot be read: %w", p.Stream, p.Consumer, err) + } + var cfg jetstream.ConsumerConfig + if err := json.Unmarshal(p.Config, &cfg); err != nil { + return fmt.Errorf("the configuration of %s on %s does not read: %w", p.Consumer, p.Stream, err) + } + info, err := st.Info(ctx) + if err != nil { + return err + } + start := p.AckFloor + 1 + if start > info.State.LastSeq+1 { + start = info.State.LastSeq + 1 + } + cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = jetstream.DeliverByStartSequencePolicy, start, nil + if err := st.DeleteConsumer(ctx, p.Consumer); err != nil && !errors.Is(err, jetstream.ErrConsumerNotFound) { + return fmt.Errorf("%s on %s cannot be removed to be made again: %w", p.Consumer, p.Stream, err) + } + if _, err := st.CreateConsumer(ctx, cfg); err != nil { + return fmt.Errorf("%s on %s cannot be made again at %d: %w", p.Consumer, p.Stream, start, err) + } + } + return nil +} + +// RestoreArchive puts every stream of the archive into the bus, which must hold none of them. +func (b KeptJetStream) RestoreArchive(ctx context.Context, a *busarchive.Archive) error { + log := b.Logf + if log == nil { + log = func(string, ...any) {} + } + return bussnap.RestoreAll(ctx, b.NC, a, log) +} + +// DeleteStreams removes streams by name. +func (b KeptJetStream) DeleteStreams(ctx context.Context, names []string) error { + for _, n := range names { + if err := b.JS.DeleteStream(ctx, n); err != nil && !errors.Is(err, jetstream.ErrStreamNotFound) { + return fmt.Errorf("%s: %w", n, err) + } + } + return nil +} diff --git a/modules/nats/internal/keeper/kept_test.go b/modules/nats/internal/keeper/kept_test.go new file mode 100644 index 000000000..2c937daa8 --- /dev/null +++ b/modules/nats/internal/keeper/kept_test.go @@ -0,0 +1,298 @@ +package keeper + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/s2" + + "nats-tools/internal/busarchive" +) + +// tarBus is the fake bus whose snapshot is a real archive of the streams asked for, as the server's is. +type tarBus struct { + *fakeBus + dir string + n int +} + +func (b *tarBus) Snapshot(ctx context.Context, streams []string) (string, error) { + if _, err := b.fakeBus.Snapshot(ctx, streams); err != nil { + return "", err + } + b.n++ + path := filepath.Join(b.dir, fmt.Sprintf(".entry-%d.tar", b.n)) + return path, os.WriteFile(path, archiveOf(streams), 0o600) +} + +func sha(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } + +// archiveOf is a snapshot archive of the streams, each holding one block. +func archiveOf(streams []string) []byte { + var out bytes.Buffer + tw := tar.NewWriter(&out) + put := func(name string, b []byte) { + _ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))}) + _, _ = tw.Write(b) + } + m := busarchive.Manifest{Format: 1, Finished: time.Date(2026, 11, 9, 3, 0, 0, 0, time.UTC)} + files := map[string][]byte{} + for _, name := range streams { + var raw bytes.Buffer + itw := tar.NewWriter(&raw) + _ = itw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: 3}) + _, _ = itw.Write([]byte("abc")) + itw.Close() + var arc bytes.Buffer + w := s2.NewWriter(&arc) + _, _ = w.Write(raw.Bytes()) + w.Close() + meta := []byte(`{"config":{"name":"` + name + `"}}`) + e := busarchive.StreamEntry{Name: name, Meta: "streams/" + name + "/backup.json", MetaSHA256: sha(meta), + Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(arc.Len()), ArchiveSHA256: sha(arc.Bytes())} + m.Streams = append(m.Streams, e) + files[e.Meta], files[e.Archive] = meta, arc.Bytes() + } + raw, _ := json.Marshal(m) + put(busarchive.ManifestName, raw) + for _, s := range m.Streams { + put(s.Meta, files[s.Meta]) + put(s.Archive, files[s.Archive]) + } + tw.Close() + return out.Bytes() +} + +// keptBus is the fake bus's streams and consumers, as the server holds them. +type keptBus struct { + streams map[string]bool + positions []Position + set []Position + restored [][]string + refuse error +} + +func (b *keptBus) Existing(_ context.Context, names []string) ([]string, error) { + var out []string + for _, n := range names { + if b.streams[n] { + out = append(out, n) + } + } + return out, nil +} +func (b *keptBus) Positions(context.Context, string) ([]Position, error) { return b.positions, nil } +func (b *keptBus) SetPositions(_ context.Context, ps []Position) error { + b.set = append(b.set, ps...) + return nil +} +func (b *keptBus) RestoreArchive(_ context.Context, a *busarchive.Archive) error { + if err := b.refuse; err != nil { + b.refuse = nil // refused once, as a server that was busy + return err + } + var names []string + for _, s := range a.Manifest.Streams { + if b.streams[s.Name] { + return fmt.Errorf("the server already holds %s", s.Name) + } + names = append(names, s.Name) + } + for _, n := range names { + b.streams[n] = true + } + b.restored = append(b.restored, names) + return nil +} +func (b *keptBus) DeleteStreams(_ context.Context, names []string) error { + for _, n := range names { + delete(b.streams, n) + } + return nil +} + +func keptKeeper(t *testing.T) (*Keeper, *keptBus, *memStore, *[]string) { + t.Helper() + k, fb, store, _ := newKeeper(FirstView{}) + dir := t.TempDir() + kb := &keptBus{streams: map[string]bool{"KV_idx_modules": true}, + positions: []Position{{Stream: "forge", Consumer: "ace_idx", AckFloor: 41, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}, + {Stream: "KV_idx_modules", Consumer: "ace_idx", AckFloor: 7, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}}} + acts := &[]string{} + k.Bus, k.Kept, k.KeptDir = &tarBus{fakeBus: fb, dir: dir}, kb, dir + k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil } + return k, kb, store, acts +} + +func take(t *testing.T, k *Keeper) *Request { + t.Helper() + r := &Request{Verb: VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Caller: "ace.restic", Why: "before form 1"} + if _, err := k.SnapshotKept(context.Background(), r); err != nil { + t.Fatal(err) + } + return r +} + +// A module's kept data on the bus has a restore point before its module moves, and a restore never goes over the live +// data (design 47 "A module's kept data on the bus, backed up and restored with its delivery"). +func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) { + for _, row := range []struct { + name string + run func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) + }{ + {"a restore point holds the archive, the positions and the catch-up tool, keyed by form and fingerprint", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + r := take(t, k) + want := filepath.Join(k.KeptDir, "points", "idx", "modules", "f0-abc123") + if r.Dir != want { + t.Fatalf("restore point in %s, want %s", r.Dir, want) + } + raw, err := os.ReadFile(filepath.Join(want, PointFile)) + if err != nil { + t.Fatal(err) + } + var p Point + if err := json.Unmarshal(raw, &p); err != nil || p.CatchUp != "indexer_take_in" || len(p.Positions) != 2 || p.Streams[0] != "KV_idx_modules" || p.Bytes == 0 { + t.Errorf("point.json = %s (%v)", raw, err) + } + if _, err := os.Stat(filepath.Join(want, ArchiveFile)); err != nil { + t.Error(err) + } + if _, _, found, _ := store.Get(StateKept, PointKey("idx", "modules", 0, "abc123")); !found { + t.Error("the restore point is not recorded in the state kept") + } + if len(*acts) != 1 || !strings.Contains((*acts)[0], "took the restore point") { + t.Errorf("hand acts %v", *acts) + } + }}, + {"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"} + _, err := k.SnapshotKept(context.Background(), r) + if err == nil || !strings.Contains(err.Error(), "KV_idx_gone") { + t.Fatalf("err = %v", err) + } + if _, err := os.Stat(filepath.Join(k.KeptDir, "points")); err == nil { + t.Error("a directory was made for a snapshot that was not taken") + } + }}, + {"a name that is not one plain name is refused", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + for _, r := range []*Request{{Module: "../x", Local: "modules", Fingerprint: "a"}, {Module: "idx", Local: "mod/ules", Fingerprint: "a"}, {Module: "idx", Local: "modules", Fingerprint: ""}} { + if _, err := k.SnapshotKept(context.Background(), r); err == nil { + t.Errorf("%+v was taken", r) + } + } + }}, + {"a restore sets the live data aside whole, removes it, restores the archive and sets back the other consumers", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + p := take(t, k) + r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: p.Dir, Caller: "ace.restic", Why: "revert"} + if _, err := k.RestoreKept(context.Background(), r); err != nil { + t.Fatal(err) + } + if !kb.streams["KV_idx_modules"] || len(kb.restored) != 1 { + t.Errorf("streams %v restored %v", kb.streams, kb.restored) + } + if r.CatchUp != "indexer_take_in" { + t.Errorf("the catch-up tool is %q", r.CatchUp) + } + aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside")) + if err != nil || len(aside) != 1 { + t.Fatalf("aside: %v %v", aside, err) + } + if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", aside[0].Name(), ArchiveFile)); err != nil { + t.Error("the live data's archive is not kept") + } + if _, _, found, _ := store.Get(StateKept, "aside."+aside[0].Name()); !found { + t.Error("what is set aside is not recorded") + } + // Only the consumer on a stream the point did not carry is made again; a stream's own comes with its snapshot. + if len(kb.set) != 1 || kb.set[0].Stream != "forge" || kb.set[0].AckFloor != 41 { + t.Errorf("positions set back: %+v", kb.set) + } + }}, + {"a restore point of another fingerprint or a linked file is refused before anything is removed", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + p := take(t, k) + bad := *p + bad.Fingerprint = "other" + if _, err := k.RestoreKept(context.Background(), &bad); err == nil || !kb.streams["KV_idx_modules"] { + t.Errorf("err = %v, live %v", err, kb.streams) + } + link := filepath.Join(p.Dir, ArchiveFile) + real := filepath.Join(p.Dir, "real.tar") + if err := os.Rename(link, real); err != nil { + t.Fatal(err) + } + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} + if _, err := k.RestoreKept(context.Background(), r); err == nil || !strings.Contains(err.Error(), "plain file") || !kb.streams["KV_idx_modules"] { + t.Errorf("err = %v, live %v", err, kb.streams) + } + }}, + {"a live data that cannot be set aside stays where it is", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + p := take(t, k) + k.Bus.(*tarBus).fakeBus.refuse["snapshot"] = errors.New("the server refused") + r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} + if _, err := k.RestoreKept(context.Background(), r); err == nil || !kb.streams["KV_idx_modules"] || !strings.Contains(err.Error(), "nothing was removed") { + t.Errorf("err = %v, live %v", err, kb.streams) + } + }}, + {"a restore that fails puts the live data back and says so", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + p := take(t, k) + kb.refuse = errors.New("the server refused the archive") + r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} + _, err := k.RestoreKept(context.Background(), r) + if err == nil || !strings.Contains(err.Error(), "the live data is back as it was") { + t.Fatalf("err = %v", err) + } + }}, + {"set-aside data goes only on a call that says why and is recorded first", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + p := take(t, k) + if _, err := k.RestoreKept(context.Background(), &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}); err != nil { + t.Fatal(err) + } + entries, _ := os.ReadDir(filepath.Join(k.KeptDir, "aside")) + id := entries[0].Name() + if _, err := k.DiscardAside(&Request{From: id}); err == nil { + t.Error("discarded with no why") + } + if _, err := k.DiscardAside(&Request{From: "../points", Why: "x"}); err == nil { + t.Error("discarded a path") + } + k.Act = func(string, string, string) error { return errors.New("the log is down") } + r := &Request{From: id, Why: "the revert held", Caller: "ace.operator"} + k.record(r, "x") // sets Unrecorded as a failing log does + if _, err := k.DiscardAside(r); err == nil { + t.Error("discarded with the act unrecorded") + } + if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err != nil { + t.Error("the data went with its act unrecorded") + } + k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil } + if _, err := k.DiscardAside(&Request{From: id, Why: "the revert held", Caller: "ace.operator"}); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err == nil { + t.Error("still there") + } + if _, _, found, _ := store.Get(StateKept, "aside."+id); found { + t.Error("its record stays") + } + }}, + } { + t.Run(row.name, func(t *testing.T) { + k, kb, store, acts := keptKeeper(t) + row.run(t, k, kb, store, acts) + }) + } +} diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index d956a239e..f1bac747a 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -46,6 +46,13 @@ type Request struct { Snapshot string `json:"snapshot,omitempty"` // Waiting is why it waits, while it waits: the operator's warrant, or the keeper's first view. Waiting string `json:"waiting,omitempty"` + // Form, Fingerprint and CatchUp name a restore point (kept.go); From is the directory a restore reads, or the id of the + // data a discard removes; Dir is what the keeper answers: the restore point's directory, or the data set aside. + Form int `json:"form,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + CatchUp string `json:"catch-up,omitempty"` + From string `json:"from,omitempty"` + Dir string `json:"dir,omitempty"` // Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged. Unrecorded string `json:"unrecorded,omitempty"` } @@ -136,6 +143,10 @@ func (k *Keeper) act(r *Request) (string, error) { } return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil } + switch r.Verb { + case VerbSnapshotKept, VerbRestoreKept, VerbDiscardAside: + return k.actKept(r) + } if !k.Viewed() { return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant) } @@ -295,3 +306,24 @@ func (k *Keeper) record(r *Request, what string) { k.logf("the hand act %q could not be recorded: %v", what, err) } } + +// actKept acts on a request about a kept data's restore point. These need no first view or warrant: a snapshot changes +// nothing, a restore sets the live data aside whole before it removes it, and a discard names who and why (kept.go). +func (k *Keeper) actKept(r *Request) (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) + defer cancel() + var out string + var err error + switch r.Verb { + case VerbSnapshotKept: + out, err = k.SnapshotKept(ctx, r) + case VerbRestoreKept: + out, err = k.RestoreKept(ctx, r) + default: + out, err = k.DiscardAside(r) + } + if err != nil { + return "nothing done", err + } + return out, nil +} diff --git a/modules/nats/module.json b/modules/nats/module.json index ad9d08019..070b1a112 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -32,7 +32,11 @@ "list", "show", "check", - "orphans" + "orphans", + "aside", + "snapshot-kept", + "restore-kept", + "discard-aside" ] } ], @@ -98,6 +102,10 @@ "history": 1, "ttl-seconds": 86400, "read-only": true + }, + { + "name": "kept", + "history": 1 } ], "listens": [ @@ -133,7 +141,8 @@ "nats_drop", "nats_keeper_health", "nats_first_view_again", - "nats_requests" + "nats_requests", + "nats_kept" ], "guards": [ 8222 @@ -257,7 +266,8 @@ "state": [ "asked", "requests", - "spent" + "spent", + "kept" ], "asks": true, "keeps": true, -- 2.54.0 From a96fd44abb0a0d8be7a4a063c3ce689ca6f4382f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:07:49 +0100 Subject: [PATCH 022/243] COND3: the keeper sets its view of an observer from what it states MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit findings-stated raises what is new, keeps each pending key unchanged, clears every other condition of the observer on that node with its reason, clears a pending key past its bound (bounds state, key time to live), and for an observer held once takes the holder's set and later events only. The holder lookup is a hook (Keeper.Holder) that COND6 wires from the catalog; the view is kept in memory until COND2 keeps findings-views. Builds: design 47 G §384 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/events.go | 220 +++++++++++++++++- .../cmd/mesh-conditions/hold.go | 3 + .../cmd/mesh-conditions/keeper.go | 18 +- .../cmd/mesh-conditions/keeper_test.go | 83 +++++++ 4 files changed, 322 insertions(+), 2 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index 3fbb7fbfd..bba53987e 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -7,8 +7,10 @@ package main import ( "encoding/json" + "errors" "fmt" "slices" + "sort" "strings" "time" @@ -21,6 +23,7 @@ import ( const ( EventFindingWritten = "finding-written" EventFindingCleared = "finding-cleared" + EventFindingsStated = "findings-stated" ) // emitterOf is the module an event's envelope key `.` names. @@ -37,6 +40,9 @@ func (k *Keeper) FindingEvent(e stdio.Envelope) error { if observer, ok := emitterOf(e.Key, EventFindingCleared); ok { return k.FindingCleared(observer, e.Node, e.Body) } + if observer, ok := emitterOf(e.Key, EventFindingsStated); ok { + return k.FindingsStated(observer, e.Node, e.Body) + } return nil } @@ -67,11 +73,32 @@ func (k *Keeper) FindingWritten(observer, node string, body []byte) error { f.Scope, f.ID, f.Subject.Scope, f.Subject.ID) return nil } + if !k.accepts(observer, node, f.At) { + k.Logf("%s's finding-written on %s is not its holder's, or is older than the holder's last statement: left out", observer, node) + return nil + } + return k.take(observer, node, f) +} + +// take holds one finding an observer says on a node, and raises or changes its condition. +func (k *Keeper) take(observer, node string, f seat.Finding) error { v := storedFinding(f) value, err := json.Marshal(v) if err != nil { return err } + k.mu.Lock() + k.init() + if k.keyNode[observer] == nil { + k.keyNode[observer] = map[string]string{} + } + k.keyNode[observer][v.Key] = node + named := k.view(observer, node).Pending[v.Key] + delete(k.view(observer, node).Pending, v.Key) + k.mu.Unlock() + if !named.IsZero() { + _ = k.Store.Delete(BoundsState, pendingKey(observer, node, v.Key)) + } return k.Finding(observer, v.Key, value, false) } @@ -86,6 +113,10 @@ func (k *Keeper) FindingCleared(observer, node string, body []byte) error { k.Logf("%s's finding-cleared on %s is not readable: %v", observer, node, err) return nil } + if !k.accepts(observer, node, c.At) { + k.Logf("%s's finding-cleared on %s is not its holder's, or is older than the holder's last statement: left out", observer, node) + return nil + } k.mu.Lock() k.init() key := cc.Key(c.Scope, c.ID, c.Kind) @@ -119,7 +150,7 @@ func storedFinding(f seat.Finding) seat.FindingsByKeyValue { // hear subscribes to the observers' events once the runtime serves, asked again until it takes each. func hear(k *Keeper) { - for _, event := range []string{EventFindingWritten, EventFindingCleared} { + for _, event := range []string{EventFindingWritten, EventFindingCleared, EventFindingsStated} { pattern := "*." + event go func() { <-stdio.Served() @@ -134,3 +165,190 @@ func hear(k *Keeper) { }() } } + +// pendingPrefix starts the keys of the state `bounds` that are a pending key's bound. +const pendingPrefix = "pending=" + +func pendingKey(observer, node, key string) string { + return pendingPrefix + observer + "=" + node + "=" + key +} + +// notJudged is why a pending key that nothing named within its bound is cleared. +const notJudged = "not judged within its bound" + +// sourceView is what the keeper heard of one observer on one node. +type sourceView struct { + // At is the last findings-stated's at; Pending are its pending keys with the end of their bound. + At time.Time + Pending map[string]time.Time +} + +// view is the observer's view on the node, made when there is none. Called with mu held. +func (k *Keeper) view(observer, node string) *sourceView { + v := k.views[observer+"."+node] + if v == nil { + v = &sourceView{Pending: map[string]time.Time{}} + k.views[observer+"."+node] = v + } + return v +} + +// holderOf says whether an observer's events are taken from node alone, and so from which (design 47 G §384). +func (k *Keeper) holderOf(observer string) (string, bool) { + if k.Holder == nil { + return "", false + } + return k.Holder(observer) +} + +// accepts says whether an event of the observer on the node, said at at, is taken: for an observer held once, only the +// holder's, and only with an at later than the holder's last statement, so an old holder's late event changes nothing. +func (k *Keeper) accepts(observer, node string, at time.Time) bool { + holder, once := k.holderOf(observer) + if !once { + return true + } + if node != holder { + return false + } + k.mu.Lock() + defer k.mu.Unlock() + k.init() + v := k.views[observer+"."+node] + return v == nil || at.After(v.At) +} + +// FindingsStated sets the keeper's view of an observer on a node to the set it states (design 47 G §384): what is new is +// raised, each pending key keeps its condition unchanged, and every other condition of the observer on the node (of every +// node, for an observer held once) is cleared. +func (k *Keeper) FindingsStated(observer, node string, body []byte) error { + if !k.isSource(observer) { + k.Logf("%s says findings-stated on %s, and is no source of findings: left out", observer, node) + return nil + } + var s seat.FindingsStated + if err := json.Unmarshal(body, &s); err != nil { + k.Logf("%s's findings-stated on %s is not readable: %v", observer, node, err) + return nil + } + for _, f := range s.Findings { + if f.Observer != observer || f.Scope != f.Subject.Scope || f.ID != f.Subject.ID { + k.Logf("%s's findings-stated on %s names the observer %q for %s.%s: refused", observer, node, f.Observer, f.Scope, f.ID) + return nil + } + } + holder, once := k.holderOf(observer) + if once && node != holder { + k.Logf("%s's findings-stated on %s is not its holder's (%s): left out", observer, node, holder) + return nil + } + now := k.Now().UTC() + k.mu.Lock() + k.init() + v := k.view(observer, node) + if !v.At.IsZero() && !s.At.After(v.At) { + k.mu.Unlock() + k.Logf("%s's findings-stated on %s at %s is not newer than the one at %s: left out", observer, node, s.At, v.At) + return nil + } + v.At = s.At + named := map[string]bool{} + for _, f := range s.Findings { + named[storedFinding(f).Key] = true + } + oldPending := v.Pending + v.Pending = map[string]time.Time{} + type pend struct { + key string + deadline time.Time + } + var pending []pend + for _, p := range s.Pending { + bound, err := time.ParseDuration(p.Bound) + if err != nil { + bound = 0 + } + named[p.Key] = true + v.Pending[p.Key] = s.At.Add(bound) + pending = append(pending, pend{p.Key, s.At.Add(bound)}) + } + var drop []string + for key := range k.findings[observer] { + if !named[key] && (once || k.keyNode[observer][key] == node) { + drop = append(drop, key) + } + } + k.mu.Unlock() + sort.Strings(drop) + for key := range oldPending { + if _, still := v.Pending[key]; !still { + _ = k.Store.Delete(BoundsState, pendingKey(observer, node, key)) + } + } + var problems []string + for _, f := range s.Findings { + if err := k.take(observer, node, f); err != nil { + problems = append(problems, err.Error()) + } + } + why := "not stated when asked" + if s.Asked == "" { + why = "not stated at " + observer + "'s start" + } + for _, key := range drop { + k.mu.Lock() + k.whys[key] = why + k.mu.Unlock() + if err := k.Finding(observer, key, nil, true); err != nil { + problems = append(problems, err.Error()) + } + k.mu.Lock() + delete(k.whys, key) + delete(k.keyNode[observer], key) + k.mu.Unlock() + } + for _, p := range pending { + _ = k.Store.Delete(BoundsState, pendingKey(observer, node, p.key)) + ttl := p.deadline.Sub(now) + if ttl < time.Second { + if err := k.PendingPassed(pendingKey(observer, node, p.key)); err != nil { + problems = append(problems, err.Error()) + } + continue + } + if err := k.Store.CreateTTL(BoundsState, pendingKey(observer, node, p.key), map[string]any{"observer": observer, "node": node, + "key": p.key, "until": p.deadline}, ttl); err != nil && !errors.Is(err, stdio.ErrChanged) { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + +// PendingPassed is a pending key's bound passing: unless a finding or a later set named it since, its condition is cleared. +func (k *Keeper) PendingPassed(bound string) error { + rest := strings.TrimPrefix(bound, pendingPrefix) + observer, rest, ok1 := strings.Cut(rest, "=") + node, key, ok2 := strings.Cut(rest, "=") + if !ok1 || !ok2 { + return nil + } + k.mu.Lock() + k.init() + v := k.views[observer+"."+node] + if v == nil || v.Pending[key].IsZero() { + k.mu.Unlock() + return nil + } + delete(v.Pending, key) + k.whys[key] = notJudged + k.mu.Unlock() + err := k.Finding(observer, key, nil, true) + k.mu.Lock() + delete(k.whys, key) + delete(k.keyNode[observer], key) + k.mu.Unlock() + return err +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go index f34e5655d..f37b7da31 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/hold.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -229,6 +229,9 @@ func (k *Keeper) Assignments(key string, value []byte, deleted bool) { // BoundPassed is a delivery's bound key expiring: what it holds is raised, saying the delivery did not end. func (k *Keeper) BoundPassed(key string) error { + if strings.HasPrefix(key, pendingPrefix) { + return k.PendingPassed(key) + } node, module := splitNodeModule(key) k.mu.Lock() k.init() diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 1a5341e32..2ca8086a0 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -144,6 +144,14 @@ type Keeper struct { outOfHold map[string]string cleared map[string]clearedMemo seq uint64 + // views are, by `.`, what the keeper heard of that observer on that node (events.go); keyNode is the + // node each finding of an observer came from; whys is the reason a key's next clearing says. + views map[string]*sourceView + keyNode map[string]map[string]string + whys map[string]string + // Holder says, for an observer whose catalog entry claims only seats held once for the mesh, the node the seat list + // names as its holder (design 47 G §384); nil, or once false, is an observer on every node. + Holder func(observer string) (node string, once bool) } func (k *Keeper) init() { @@ -154,6 +162,7 @@ func (k *Keeper) init() { map[string]string{}, map[string]held{}, map[string]clearedMemo{} k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{} k.assignments = assignerclient.NewKeys() + k.views, k.keyNode, k.whys = map[string]*sourceView{}, map[string]map[string]string{}, map[string]string{} } if k.Now == nil { k.Now = time.Now @@ -341,7 +350,14 @@ func (k *Keeper) settleOnce(key string) error { if !open { return nil } - return k.clear(key, was, rev, now, "its finding went") + why := "its finding went" + k.mu.Lock() + if w, ok := k.whys[key]; ok { + why = w + delete(k.whys, key) + } + k.mu.Unlock() + return k.clear(key, was, rev, now, why) } if holding { return k.keepHeld(hold) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index cfa1e6f96..f0074bfc0 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -457,3 +457,86 @@ func TestTheKeeperReadsTheOldLayoutAndWritesTheSeatsLayout(t *testing.T) { t.Errorf("the event the seat hears: %+v", last) } } + +// Design 47 G §384: the keeper sets its view of an observer on a node to the set it states: what is new is raised, a +// pending key is kept unchanged, every other condition of the observer on that node is cleared with its reason, a pending key +// nothing named within its bound is cleared, and for an observer held once the holder's set replaces every node's. +func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { + r := newRig() + holder := "ace" + r.k.Holder = func(o string) (string, bool) { return holder, o == selfCheck } + f := func(observer, id string) seat.Finding { + g := eventFinding(observer, r.now) + g.ID, g.Subject.ID = id, id + return g + } + stated := func(observer, node, asked string, at time.Time, pend []seat.FindingsStatedPendingItem, fs ...seat.Finding) error { + return r.k.FindingsStated(observer, node, mustJSON(t, seat.FindingsStated{At: at, Asked: asked, Findings: fs, Pending: pend})) + } + why := func() string { + defer func() { r.said = nil }() + if len(r.said) == 0 || r.said[len(r.said)-1].Change != cc.ChangeCleared { + return "" + } + return r.said[len(r.said)-1].Why + } + t0 := r.now + steps := []struct { + name string + do func() error + open []string // the keys open after + why string + }{ + {"a start states two findings: both raised", func() error { + return stated(watchman, "ace", "", t0, nil, f(watchman, "a"), f(watchman, "b")) + }, []string{"machine.a.silent", "machine.b.silent"}, ""}, + {"a later set without b clears it, with its reason", func() error { + return stated(watchman, "ace", "ask-1", t0.Add(time.Minute), nil, f(watchman, "a")) + }, []string{"machine.a.silent"}, "not stated when asked"}, + {"a restart that states nothing clears the rest", func() error { + return stated(watchman, "ace", "", t0.Add(2*time.Minute), nil) + }, nil, "not stated at " + watchman + "'s start"}, + {"a pending key's condition stays open and unsaid", func() error { + _ = r.k.FindingWritten(watchman, "ace", mustJSON(t, f(watchman, "c"))) + r.said = nil + return stated(watchman, "ace", "", t0.Add(3*time.Minute), []seat.FindingsStatedPendingItem{{Key: "machine.c.silent", Since: t0, Bound: "5m"}}) + }, []string{"machine.c.silent"}, ""}, + {"a pending key nothing named within its bound is cleared", func() error { + return r.k.PendingPassed(pendingKey(watchman, "ace", "machine.c.silent")) + }, nil, notJudged}, + {"a set of another node clears nothing of ace", func() error { + _ = r.k.FindingWritten(watchman, "ace", mustJSON(t, f(watchman, "d"))) + r.said = nil + return stated(watchman, "g14", "", t0.Add(4*time.Minute), nil) + }, []string{"machine.d.silent"}, ""}, + {"an older set is left out", func() error { + return stated(watchman, "ace", "", t0.Add(-time.Hour), nil) + }, []string{"machine.d.silent"}, ""}, + {"a held-once observer's old holder raises nothing after the new holder's set", func() error { + if err := stated(selfCheck, "ace", "", t0.Add(5*time.Minute), nil, f(selfCheck, "e")); err != nil { + return err + } + holder = "g14" + if err := stated(selfCheck, "g14", "", t0.Add(6*time.Minute), nil, f(selfCheck, "g")); err != nil { + return err + } + return r.k.FindingWritten(selfCheck, "ace", mustJSON(t, func() seat.Finding { x := f(selfCheck, "h"); x.At = t0.Add(7 * time.Minute); return x }())) + }, []string{"machine.d.silent", "machine.g.silent"}, "not stated at " + selfCheck + "'s start"}, + } + for _, s := range steps { + if err := s.do(); err != nil { + t.Fatalf("%s: %v", s.name, err) + } + var open []string + for _, c := range func() []Condition { c, _ := r.k.Open(); return c }() { + open = append(open, c.Key) + } + sort.Strings(open) + if !slices.Equal(open, s.open) { + t.Errorf("%s: open %v, want %v", s.name, open, s.open) + } + if got := why(); got != s.why { + t.Errorf("%s: cleared because %q, want %q", s.name, got, s.why) + } + } +} -- 2.54.0 From b6884796e7fb87d7c4c66dc0d8077ada12f3dcdc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:08:05 +0100 Subject: [PATCH 023/243] NATS4: the bus module takes and restores a restore point of a module's kept data, beside the live data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verbs snapshot-kept and restore-kept (with aside-only and discard-aside) on the seat mesh-broker, handed to the keeper; taken over from the join's 0480e16f and c4e88eaa, without the verbs aside and discard-aside and the tool nats_kept, which no design line gives. Builds: design 47 G §296, §297, §302, §337, §405 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/nats/README.md | 5 +- modules/nats/cmd/nats-tools/keptools.go | 95 ++++---------------- modules/nats/cmd/nats-tools/keptools_test.go | 21 +++-- modules/nats/cmd/nats-tools/main.go | 6 +- modules/nats/internal/keeper/kept.go | 47 +++++++--- modules/nats/internal/keeper/kept_test.go | 13 +++ modules/nats/internal/keeper/requests.go | 6 +- modules/nats/module.json | 7 +- 8 files changed, 91 insertions(+), 109 deletions(-) diff --git a/modules/nats/README.md b/modules/nats/README.md index ec6b5700f..28e5b9a51 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -144,8 +144,9 @@ module's kept data on the bus*). Its seat verbs, served by the tools and handed `mesh-broker.snapshot-kept` (the data as the server's snapshot, the module's consumers' positions and the catch-up tool, in one directory under the keeper's state), `mesh-broker.restore-kept` (the live data is snapshotted into a set-aside directory under a name of its own and only then removed; the restore point is put in its place and the consumers on other -streams are set back), `mesh-broker.discard-aside` (only on a call that says why, recorded first) and `mesh-broker.aside`; -the tool `nats_kept` reads the restore points and the set-aside data from the keeper's state `kept`. The backup holder +streams are set back; with `aside-only` it sets the live data aside and restores nothing, with `discard-aside`, naming data +set aside, it removes that data on a call that says why, recorded first, and restores nothing). Both are called by the +holder of `node-backup` on the same node alone. The backup holder keeps the restore points; it calls these verbs through the seat. ## Files it does not know diff --git a/modules/nats/cmd/nats-tools/keptools.go b/modules/nats/cmd/nats-tools/keptools.go index e5efeeede..87545381a 100644 --- a/modules/nats/cmd/nats-tools/keptools.go +++ b/modules/nats/cmd/nats-tools/keptools.go @@ -1,9 +1,7 @@ package main import ( - "encoding/json" "fmt" - "sort" "strings" "time" @@ -16,49 +14,15 @@ import ( // the runtime's own bound on a call, so a long snapshot is answered as "handed, not ended" and read again with its request. const keptWait = 50 * time.Second -// KeptHeld reads the keeper's state kept: the restore points it took and the data it set aside. -type KeptHeld func() ([]keeper.Point, []keeper.Aside, error) - -// KeptInState is the state kept through the runtime that launched this bundle. -func KeptInState() ([]keeper.Point, []keeper.Aside, error) { - st := stdio.State(keeper.StateKept) - keys, err := st.Keys() - if err != nil { - return nil, nil, err - } - sort.Strings(keys) - var points []keeper.Point - var aside []keeper.Aside - for _, key := range keys { - e, err := st.Get(key) - if err != nil || e == nil { - continue - } - switch { - case strings.HasPrefix(key, "point."): - var p keeper.Point - if json.Unmarshal(e.Value, &p) == nil { - points = append(points, p) - } - case strings.HasPrefix(key, "aside."): - var a keeper.Aside - if json.Unmarshal(e.Value, &a) == nil { - aside = append(aside, a) - } - } - } - return points, aside, nil -} - func formOf(a map[string]any) int { f, _ := a["form"].(float64) return int(f) } -// keptTools are the seat mesh-broker's verbs for a module's kept data and the bus module's tool that reads what it holds: +// keptTools are the seat mesh-broker's verbs for a module's kept data (design 47 G §405): // each verb hands its request to the keeper, which acts on its own account, and waits for the end on the watch of its // requests (never a poll). Asked again with `request`, a verb reads the end of the request it made. -func keptTools(rs requestStore, held KeptHeld) []stdio.Tool { +func keptTools(rs requestStore) []stdio.Tool { entry := func(extra map[string]any) map[string]any { in := map[string]any{"module": str("the module whose kept data it is, e.g. mesh-delivery-indexer"), "local-name": str("the kept data's local name, e.g. modules"), "form": num("the kept data's form (a manifest field; absent reads 0)"), "fingerprint": str("the fingerprint the restore point was taken before"), @@ -112,16 +76,6 @@ func keptTools(rs requestStore, held KeptHeld) []stdio.Tool { } return nil } - readAside := func() (any, error) { - _, aside, err := held() - if err != nil { - return nil, err - } - if aside == nil { - aside = []keeper.Aside{} - } - return map[string]any{"aside": aside}, nil - } return []stdio.Tool{ {Name: "mesh-broker.snapshot-kept", Description: "Take the restore point of one module's kept data on the bus (milestone G, design 47): the data whole as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool the newer manifest declares, kept by the bus module's keeper. Answers the directory the restore point is in. Handed to the keeper, recorded as a hand act.", @@ -131,41 +85,28 @@ func keptTools(rs requestStore, held KeptHeld) []stdio.Tool { return point(a, r) })}, {Name: "mesh-broker.restore-kept", - Description: "Restore one module's kept data on the bus from a restore point beside the live data, never over it (design 47): the live data is snapshotted into a set-aside directory under a name of its own and then removed from the bus, the restore point's streams are put in its place and the module's other consumers are set back to their positions. Answers what was set aside and the catch-up tool the restore point carries. Handed to the keeper, recorded as a hand act.", - Input: entry(map[string]any{"from": str("the directory the restore point was restored to, an absolute path")}), + Description: "Restore one module's kept data on the bus from a restore point beside the live data, never over it (design 47 G §405): the live data is snapshotted into a set-aside directory under a name of its own and then removed from the bus, the restore point's streams are put in its place and the module's other consumers are set back to their positions. With `aside-only` it sets the live data aside and restores nothing; with `discard-aside`, naming data set aside, it removes that data and restores nothing. Answers what was set aside and the catch-up tool the restore point carries. Handed to the keeper, recorded as a hand act.", + Input: entry(map[string]any{"from": str("the directory the restore point was restored to, an absolute path"), + "aside-only": map[string]any{"type": "boolean", "description": "set the live data aside and restore nothing"}, + "discard-aside": str("the id of data set aside: remove it and restore nothing")}), Run: run(keeper.VerbRestoreKept, func(a map[string]any, r *keeper.Request) error { + if id := arg(a, "discard-aside"); id != "" { + r.Verb, r.From = keeper.VerbDiscardAside, id + return nil + } + if b, _ := a["aside-only"].(bool); b { + r.AsideOnly = true + r.Module, r.Local = arg(a, "module"), arg(a, "local-name") + if r.Module == "" || r.Local == "" { + return fmt.Errorf("aside-only is of one kept data: a module and its local name") + } + return nil + } r.From = arg(a, "from") if r.From == "" { return fmt.Errorf("a restore names the directory the restore point is in") } return point(a, r) })}, - {Name: "mesh-broker.discard-aside", - Description: "Remove the data a restore set aside, by its id (from `aside`), only on a person's call: it says who and why, and the keeper records the hand act before it removes anything.", - Input: map[string]any{"id": str("the id of the data set aside"), "why": str("why, in plain words: kept with the act"), "request": str("a request made before: its end is read")}, - Run: run(keeper.VerbDiscardAside, func(a map[string]any, r *keeper.Request) error { - r.From = arg(a, "id") - if r.From == "" { - return fmt.Errorf("discard-aside names the id of the data set aside") - } - return nil - })}, - {Name: "mesh-broker.aside", Description: "The data restores set aside, each with its module, kept data, size, when, who and why: what a person decides to discard.", - Run: func(map[string]any) (any, error) { return readAside() }}, - {Name: "nats_kept", - Description: "What the bus module holds of its modules' kept data besides the live data: every restore point it took (module, kept data, form, the fingerprint it was taken before, its consumers' positions, the catch-up tool, its size) and the data a restore set aside. Read from the keeper's state kept.", - Run: func(map[string]any) (any, error) { - points, aside, err := held() - if err != nil { - return nil, err - } - if points == nil { - points = []keeper.Point{} - } - if aside == nil { - aside = []keeper.Aside{} - } - return map[string]any{"points": points, "aside": aside}, nil - }}, } } diff --git a/modules/nats/cmd/nats-tools/keptools_test.go b/modules/nats/cmd/nats-tools/keptools_test.go index 4205dc0f1..a088cd704 100644 --- a/modules/nats/cmd/nats-tools/keptools_test.go +++ b/modules/nats/cmd/nats-tools/keptools_test.go @@ -33,9 +33,8 @@ func TestTheKeptDataVerbsAreHandedToTheKeeperAndReadBack(t *testing.T) { return &r }, } - aside := []keeper.Aside{{ID: "idx.modules.1", Module: "idx", Local: "modules", Bytes: 12}} tools := map[string]stdio.Tool{} - for _, tl := range keptTools(rs, func() ([]keeper.Point, []keeper.Aside, error) { return nil, aside, nil }) { + for _, tl := range keptTools(rs) { tools[tl.Name] = tl } for _, verb := range KeptVerbs { @@ -53,10 +52,11 @@ func TestTheKeptDataVerbsAreHandedToTheKeeperAndReadBack(t *testing.T) { t.Errorf("answered %+v: the request for form 1 before abc, for the caller the runtime proved, with the directory the keeper answered", r) } for name, args := range map[string]map[string]any{ - "mesh-broker.snapshot-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc"}, - "mesh-broker.restore-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc", "why": "revert"}, - "mesh-broker.discard-aside": {"why": "x"}, + "mesh-broker.snapshot-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc"}, + "mesh-broker.restore-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc", "why": "revert"}, + "mesh-broker.restore-kept#aside-only": {"aside-only": true, "why": "x"}, } { + name, _, _ = strings.Cut(name, "#") if _, err := tools[name].Run(args); err == nil { t.Errorf("%s with %v was handed to the keeper", name, args) } @@ -65,9 +65,14 @@ func TestTheKeptDataVerbsAreHandedToTheKeeperAndReadBack(t *testing.T) { "from": "/restored/f0-abc", "why": "revert"}); err != nil || out.(*keeper.Request).From != "/restored/f0-abc" { t.Errorf("restore answered %+v (%v)", out, err) } - read, err := tools["mesh-broker.aside"].Run(nil) - if err != nil || !strings.Contains(strings.ReplaceAll(strings.TrimSpace(toJSON(read)), " ", ""), `"id":"idx.modules.1"`) { - t.Errorf("aside answered %v (%v)", read, err) + // `aside-only` sets the live data aside and restores nothing; `discard-aside` removes the data set aside (§405). + if out, err := tools["mesh-broker.restore-kept"].Run(map[string]any{"module": "idx", "local-name": "modules", "aside-only": true, "why": "revert"}); err != nil || + !out.(*keeper.Request).AsideOnly || out.(*keeper.Request).Verb != keeper.VerbRestoreKept { + t.Errorf("aside-only answered %+v (%v)", out, err) + } + if out, err := tools["mesh-broker.restore-kept"].Run(map[string]any{"discard-aside": "idx.modules.1", "why": "operator's call"}); err != nil || + out.(*keeper.Request).Verb != keeper.VerbDiscardAside || out.(*keeper.Request).From != "idx.modules.1" { + t.Errorf("discard-aside answered %+v (%v)", out, err) } } diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index 1e277fdd5..d9f614485 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -166,10 +166,10 @@ func main() { // bus holds against what was asked, and what no manifest asks for. Its acting verbs (create, apply, recreate, drop) are // the keeper's own work, on its own account alone (keeper.go), so no other account, the tool runner's included, holds // them. -var ReadingVerbs = []string{"list", "show", "check", "orphans", "aside"} +var ReadingVerbs = []string{"list", "show", "check", "orphans"} // KeptVerbs are the seat mesh-broker's verbs for a module's kept data (keptools.go): each handed to the keeper. -var KeptVerbs = []string{"snapshot-kept", "restore-kept", "discard-aside"} +var KeptVerbs = []string{"snapshot-kept", "restore-kept"} // Events are the seat mesh-broker's events (ADR 0595 §6, ADR 0620 §2): said on the seat's subjects on the shared events stream // and kept in full on the bus module's own stream broker-events (BROKER_EVENTS), which `history` reads (keep.go). @@ -195,5 +195,5 @@ func allTools() []stdio.Tool { out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) requests := RequestsInState() out = append(out, requestTools(requests)...) - return append(out, keptTools(requests, KeptInState)...) + return append(out, keptTools(requests)...) } diff --git a/modules/nats/internal/keeper/kept.go b/modules/nats/internal/keeper/kept.go index e4caf5a5e..945f71ae4 100644 --- a/modules/nats/internal/keeper/kept.go +++ b/modules/nats/internal/keeper/kept.go @@ -132,7 +132,7 @@ func (k *Keeper) keptReady(r *Request) error { if k.Kept == nil || k.KeptDir == "" { return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored") } - if !safeName(r.Module) || !safeName(r.Local) || !safeName(r.Fingerprint) { + if !safeName(r.Module) || !safeName(r.Local) || (!r.AsideOnly && !safeName(r.Fingerprint)) { return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint) } return nil @@ -244,17 +244,30 @@ func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) { if err := k.keptReady(r); err != nil { return "", err } - if !filepath.IsAbs(r.From) { - return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From) - } - p, a, err := readPoint(filepath.Clean(r.From), r) - if err != nil { - return "", err - } - defer a.Close() - live, err := k.Kept.Existing(ctx, p.Streams) - if err != nil { - return "", err + var p *Point + var a *busarchive.Archive + var live []string + if r.AsideOnly { + // `aside-only` (design 47 G §405): the live data is set aside and nothing is restored. + streams, err := k.streamsOf(ctx, r.Module, r.Local) + if err != nil { + return "", err + } + live = streams + } else { + if !filepath.IsAbs(r.From) { + return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From) + } + var err error + p, a, err = readPoint(filepath.Clean(r.From), r) + if err != nil { + return "", err + } + defer a.Close() + live, err = k.Kept.Existing(ctx, p.Streams) + if err != nil { + return "", err + } } id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405")) var aside *Aside @@ -288,6 +301,16 @@ func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) { return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err) } } + if r.AsideOnly { + r.Dir = "" + where := "nothing was live, so nothing was set aside" + if aside != nil { + r.Dir = aside.Dir + where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir) + } + k.record(r, fmt.Sprintf("set aside %s's %s and restored nothing; %s", r.Module, r.Local, where)) + return fmt.Sprintf("set aside %s's %s; %s", r.Module, r.Local, where), nil + } if err := k.Kept.RestoreArchive(ctx, a); err != nil { msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err) if aside != nil { diff --git a/modules/nats/internal/keeper/kept_test.go b/modules/nats/internal/keeper/kept_test.go index 2c937daa8..95e0d2b12 100644 --- a/modules/nats/internal/keeper/kept_test.go +++ b/modules/nats/internal/keeper/kept_test.go @@ -175,6 +175,19 @@ func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) { t.Errorf("hand acts %v", *acts) } }}, + {"aside-only sets the live data aside whole and restores nothing", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { + r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", AsideOnly: true, Caller: "ace.restic", Why: "newer build stopped"} + if _, err := k.RestoreKept(context.Background(), r); err != nil { + t.Fatal(err) + } + if kb.streams["KV_idx_modules"] || len(kb.restored) != 0 { + t.Errorf("streams %v restored %v: the live data is removed and nothing restored", kb.streams, kb.restored) + } + aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside")) + if err != nil || len(aside) != 1 || r.Dir == "" { + t.Fatalf("aside: %v %v dir %q", aside, err, r.Dir) + } + }}, {"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"} _, err := k.SnapshotKept(context.Background(), r) diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index f1bac747a..b7993c83d 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -51,8 +51,10 @@ type Request struct { Form int `json:"form,omitempty"` Fingerprint string `json:"fingerprint,omitempty"` CatchUp string `json:"catch-up,omitempty"` - From string `json:"from,omitempty"` - Dir string `json:"dir,omitempty"` + // AsideOnly is the argument `aside-only` of restore-kept: the live data is set aside and nothing is restored. + AsideOnly bool `json:"aside-only,omitempty"` + From string `json:"from,omitempty"` + Dir string `json:"dir,omitempty"` // Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged. Unrecorded string `json:"unrecorded,omitempty"` } diff --git a/modules/nats/module.json b/modules/nats/module.json index 070b1a112..f239c2e49 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -33,10 +33,8 @@ "show", "check", "orphans", - "aside", "snapshot-kept", - "restore-kept", - "discard-aside" + "restore-kept" ] } ], @@ -141,8 +139,7 @@ "nats_drop", "nats_keeper_health", "nats_first_view_again", - "nats_requests", - "nats_kept" + "nats_requests" ], "guards": [ 8222 -- 2.54.0 From ca02942a296336cba86e29673076f116e5f208f9 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:10:08 +0100 Subject: [PATCH 024/243] COND4: the keeper's ask, resync naming a module, findings-unstated and the findings view tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resync writes the hand-act log, says findings-asked for every observer or the one named and settles what it holds; an observer silent five minutes after an ask (bounds state, key time to live), its machine not away, is the keeper's finding findings-unstated of scope module, cleared by its next findings-stated; conditions_findings_views answers each observer and node. The nodes asked are those heard (the kept list of COND2 is stopped on a question). Builds: design 47 G §390 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/asks.go | 194 ++++++++++++++++++ .../cmd/mesh-conditions/events.go | 17 +- .../cmd/mesh-conditions/hold.go | 3 + .../cmd/mesh-conditions/keeper.go | 6 + .../cmd/mesh-conditions/keeper_test.go | 33 +++ .../cmd/mesh-conditions/manifest_test.go | 2 +- .../cmd/mesh-conditions/tools.go | 12 +- 7 files changed, 259 insertions(+), 8 deletions(-) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/asks.go diff --git a/modules/mesh-conditions/cmd/mesh-conditions/asks.go b/modules/mesh-conditions/cmd/mesh-conditions/asks.go new file mode 100644 index 000000000..a380abd87 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/asks.go @@ -0,0 +1,194 @@ +package main + +// The keeper's ask and what follows it (design 47 G §390): `findings-asked` on its seat asks the observers to state their +// findings again; one that states nothing within five minutes, its machine not away, is the keeper's finding +// `findings-unstated`, cleared by its next `findings-stated`. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "time" + + cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// AskedWithin is how long an observer has to state its findings after an ask. +const AskedWithin = 5 * time.Minute + +// The kind of the keeper's finding of an observer that stated nothing, and the prefix of its ask's key in `bounds`. +const ( + KindFindingsUnstated = "findings-unstated" + askedPrefix = "asked=" + // FindingsViewsTool reads what the keeper holds of each observer and node. + FindingsViewsTool = "conditions_findings_views" +) + +func askedKey(observer, node, ask string) string { + return askedPrefix + observer + "=" + node + "=" + ask +} + +func sayAsk(body any) error { + return stdio.SeatPublish(seat.SubjectFindingsAsked, body, "") +} + +func recordHandAct(what, why, verb string) error { + _, err := meshhandactsclient.Recorder{Ask: stdio.Ask}.Record(meshhandactsclient.Request{What: what, Why: why, Verb: verb}) + return err +} + +// machineAway says the watchman holds the machine away. +func (k *Keeper) machineAway(node string) bool { + k.mu.Lock() + defer k.mu.Unlock() + return k.away[node] +} + +// Ask says `findings-asked` for module alone, or for every observer when module is empty, and starts the bound of each +// observer on a machine not away. It answers the ask's id. +func (k *Keeper) Ask(by, module string) (string, error) { + now := k.Now().UTC() + k.mu.Lock() + k.init() + k.seq++ + ask := fmt.Sprintf("%d-%d", now.UnixNano(), k.seq) + var wait [][2]string + for _, v := range k.views { + if v.observer == "" || module != "" && v.observer != module || k.away[v.node] { + continue + } + v.Asks[ask] = now.Add(AskedWithin) + wait = append(wait, [2]string{v.observer, v.node}) + } + k.mu.Unlock() + body := map[string]any{"ask": ask, "at": now, "by": by} + if module != "" { + body["module"] = module + } + publish := k.Publish + if publish == nil { + publish = func(_ string, body any) error { return sayAsk(body) } + } + if err := publish(seat.EventFindingsAsked, body); err != nil { + return "", fmt.Errorf("asking the observers to state their findings: %w", err) + } + var problems []string + for _, w := range wait { + if err := k.Store.CreateTTL(BoundsState, askedKey(w[0], w[1], ask), map[string]any{"observer": w[0], "node": w[1], "ask": ask, + "until": now.Add(AskedWithin)}, AskedWithin); err != nil && !errors.Is(err, stdio.ErrChanged) { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return ask, errors.New(strings.Join(problems, "; ")) + } + return ask, nil +} + +// ResyncVerb is `resync`: it writes the call in the hand-act log, asks the observers (or the one named) to state their +// findings, and settles every key the keeper holds. +func (k *Keeper) ResyncVerb(by, module string) (map[string]any, error) { + what := "asked every observer to state its findings again" + if module != "" { + what = "asked " + module + " to state its findings again" + } + record := k.Record + if record == nil { + record = recordHandAct + } + if err := record(what, "resync, asked by "+by, seat.Seat+".resync"); err != nil { + return nil, fmt.Errorf("the hand act was not recorded, so nothing was asked: %w", err) + } + ask, err := k.Ask(by, module) + if err != nil { + return nil, err + } + n, err := k.Resync() + return map[string]any{"ask": ask, "settled": n}, err +} + +// AskPassed is an ask's five minutes ending: an observer that stated nothing since, its machine not away, is the finding +// findings-unstated. +func (k *Keeper) AskPassed(bound string) error { + observer, rest, ok1 := strings.Cut(strings.TrimPrefix(bound, askedPrefix), "=") + node, ask, ok2 := strings.Cut(rest, "=") + if !ok1 || !ok2 { + return nil + } + k.mu.Lock() + k.init() + v := k.views[observer+"."+node] + if v == nil { + k.mu.Unlock() + return nil + } + _, open := v.Asks[ask] + delete(v.Asks, ask) + away := k.away[node] + k.mu.Unlock() + if !open || away { + return nil + } + now := k.Now().UTC() + f := seat.FindingsByKeyValue{V: cc.FindingVersion, Key: unstatedKey(observer, node), Kind: KindFindingsUnstated, + Subject: seat.Subject{Scope: cc.ScopeModule, ID: observer + "." + node, Machine: node}, + Severity: cc.Warning, Summary: fmt.Sprintf("%s on %s stated no findings within %s of the ask", observer, node, AskedWithin), + Headline: observer + " on " + node + " did not state its findings", + Explanation: "The keeper asked " + observer + " on " + node + " to state its findings and heard nothing; its conditions stand as they were.", + Source: "findings-asked", Observer: Module, At: now, Evidence: []seat.Evidence{{At: now, Said: "ask " + ask + " unanswered"}}} + raw, err := json.Marshal(f) + if err != nil { + return err + } + return k.Finding(Module, f.Key, raw, false) +} + +func unstatedKey(observer, node string) string { + return cc.Key(cc.ScopeModule, observer+"."+node, KindFindingsUnstated) +} + +// clearUnstated clears the finding findings-unstated of the observer on the node: it stated. +func (k *Keeper) clearUnstated(observer, node string) { + k.mu.Lock() + k.init() + _, held := k.findings[Module][unstatedKey(observer, node)] + k.mu.Unlock() + if held { + if err := k.Finding(Module, unstatedKey(observer, node), nil, true); err != nil { + k.Logf("%v", err) + } + } +} + +// FindingsViews answers each observer and node: its last stated set with its at, its pending keys and its open asks. +func (k *Keeper) FindingsViews() []map[string]any { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + var out []map[string]any + for _, v := range k.views { + if v.observer == "" { + continue + } + pending, asks := []string{}, []string{} + for key := range v.Pending { + pending = append(pending, key) + } + for ask := range v.Asks { + asks = append(asks, ask) + } + sort.Strings(pending) + sort.Strings(asks) + out = append(out, map[string]any{"observer": v.observer, "node": v.node, "stated": map[string]any{"at": v.At, "findings": v.Findings}, + "pending": pending, "asks": asks}) + } + sort.Slice(out, func(i, j int) bool { + return fmt.Sprint(out[i]["observer"], out[i]["node"]) < fmt.Sprint(out[j]["observer"], out[j]["node"]) + }) + return out +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index bba53987e..830c8be39 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -178,16 +178,20 @@ const notJudged = "not judged within its bound" // sourceView is what the keeper heard of one observer on one node. type sourceView struct { + observer, node string // At is the last findings-stated's at; Pending are its pending keys with the end of their bound. - At time.Time - Pending map[string]time.Time + At time.Time + Pending map[string]time.Time + Findings []seat.Finding + // Asks are the asks it has not answered, with the end of their bound (asks.go). + Asks map[string]time.Time } // view is the observer's view on the node, made when there is none. Called with mu held. func (k *Keeper) view(observer, node string) *sourceView { v := k.views[observer+"."+node] if v == nil { - v = &sourceView{Pending: map[string]time.Time{}} + v = &sourceView{observer: observer, node: node, Pending: map[string]time.Time{}, Asks: map[string]time.Time{}} k.views[observer+"."+node] = v } return v @@ -252,6 +256,9 @@ func (k *Keeper) FindingsStated(observer, node string, body []byte) error { return nil } v.At = s.At + v.Findings = s.Findings + answered := v.Asks + v.Asks = map[string]time.Time{} named := map[string]bool{} for _, f := range s.Findings { named[storedFinding(f).Key] = true @@ -279,6 +286,10 @@ func (k *Keeper) FindingsStated(observer, node string, body []byte) error { } } k.mu.Unlock() + for ask := range answered { + _ = k.Store.Delete(BoundsState, askedKey(observer, node, ask)) + } + k.clearUnstated(observer, node) sort.Strings(drop) for key := range oldPending { if _, still := v.Pending[key]; !still { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go index f37b7da31..8c9247581 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/hold.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -232,6 +232,9 @@ func (k *Keeper) BoundPassed(key string) error { if strings.HasPrefix(key, pendingPrefix) { return k.PendingPassed(key) } + if strings.HasPrefix(key, askedPrefix) { + return k.AskPassed(key) + } node, module := splitNodeModule(key) k.mu.Lock() k.init() diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 2ca8086a0..ae523a835 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -152,6 +152,12 @@ type Keeper struct { // Holder says, for an observer whose catalog entry claims only seats held once for the mesh, the node the seat list // names as its holder (design 47 G §384); nil, or once false, is an observer on every node. Holder func(observer string) (node string, once bool) + // Publish says the body on the subject of the seat's event (findings-asked); Record writes a hand act in the hand-act + // log before it is done (design 47 G §390). + Publish func(event string, body any) error + Record func(what, why, verb string) error + // away are the machines the watchman says are away (design 47 G §389). + away map[string]bool } func (k *Keeper) init() { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index f0074bfc0..4105d581f 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -481,6 +481,12 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { return r.said[len(r.said)-1].Why } t0 := r.now + var asked []map[string]any + var acts []string + r.k.Publish = func(_ string, b any) error { asked = append(asked, b.(map[string]any)); return nil } + r.k.Record = func(what, why, verb string) error { acts = append(acts, verb+": "+what); return nil } + unstated := cc.Key(cc.ScopeModule, watchman+".ace", KindFindingsUnstated) + var ask string steps := []struct { name string do func() error @@ -522,6 +528,33 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { } return r.k.FindingWritten(selfCheck, "ace", mustJSON(t, func() seat.Finding { x := f(selfCheck, "h"); x.At = t0.Add(7 * time.Minute); return x }())) }, []string{"machine.d.silent", "machine.g.silent"}, "not stated at " + selfCheck + "'s start"}, + // Design 47 G §390: the ask, findings-unstated and the hand act. + {"an ask for every observer waits five minutes on each", func() (err error) { + ask, err = r.k.Ask("jochen", "") + return err + }, []string{"machine.d.silent", "machine.g.silent"}, ""}, + {"an observer silent past the ask is findings-unstated", func() error { + return r.k.AskPassed(askedKey(watchman, "ace", ask)) + }, []string{"machine.d.silent", "machine.g.silent", unstated}, ""}, + {"its next findings-stated clears it", func() error { + return stated(watchman, "ace", ask, t0.Add(10*time.Minute), nil, f(watchman, "d")) + }, []string{"machine.d.silent", "machine.g.silent"}, "its finding went"}, + {"a machine away is not named", func() error { + if _, err := r.k.Ask("jochen", ""); err != nil { + return err + } + r.k.away = map[string]bool{"g14": true} + return r.k.AskPassed(askedKey(selfCheck, "g14", asked[len(asked)-1]["ask"].(string))) + }, []string{"machine.d.silent", "machine.g.silent"}, ""}, + {"resync names one module: its hand act and its ask", func() error { + if _, err := r.k.ResyncVerb("person.jochen", selfCheck); err != nil { + return err + } + if len(acts) != 1 || asked[len(asked)-1]["module"] != selfCheck || len(r.k.FindingsViews()) == 0 { + t.Errorf("hand acts %v, last ask %v", acts, asked[len(asked)-1]) + } + return nil + }, []string{"machine.d.silent", "machine.g.silent"}, ""}, } for _, s := range steps { if err := s.do(); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index fb4a6fe85..98ef5b04b 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -63,7 +63,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { } // Clause 12: each state with the tool that reads it, each act with the tool that starts it. readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, - HeldState: HeldTool, BoundsState: HeldTool, "a resync": Seat + ".resync", "what it reads": InputsTool} + HeldState: HeldTool, BoundsState: HeldTool, "a resync": Seat + ".resync", "what it holds of observers": FindingsViewsTool, "what it reads": InputsTool} tools := map[string]bool{} for _, tool := range Tools(&Keeper{}) { tools[tool.Name] = true diff --git a/modules/mesh-conditions/cmd/mesh-conditions/tools.go b/modules/mesh-conditions/cmd/mesh-conditions/tools.go index 9e0d80044..621fe6f46 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/tools.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/tools.go @@ -72,11 +72,15 @@ func Tools(k *Keeper) []stdio.Tool { c, err := k.Silence(strArg(args, "key"), d, stdio.CallerOf(args), strArg(args, "why")) return c.Stored(), err }}, - {Name: Seat + ".resync", Description: "Read every finding again and raise and clear what differs.", - Run: func(map[string]any) (any, error) { - n, err := k.Resync() - return map[string]any{"settled": n}, err + {Name: Seat + ".resync", Description: "Ask the observers to state their findings again, every one or the one named, and raise " + + "and clear what differs from the open conditions; the call is written in the hand-act log.", + Input: obj(map[string]any{"module": str("ask this observer alone; none asks all")}), + Run: func(args map[string]any) (any, error) { + return k.ResyncVerb(stdio.CallerOf(args), strArg(args, "module")) }}, + {Name: FindingsViewsTool, Description: "What the keeper holds of each observer and node: its last stated set with its at, " + + "its pending keys and its open asks (design 47 G §390).", + Run: func(map[string]any) (any, error) { return k.FindingsViews(), nil }}, {Name: HistoryTool, Description: "The keeper's history: every transition it said, newest first, at most limit.", Input: obj(map[string]any{"limit": str("how many (default 50)"), "key": str("only this condition")}), Run: func(args map[string]any) (any, error) { -- 2.54.0 From 5c8f3b8351fe28834bb0fad3bab868cac6abe247 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:11:54 +0100 Subject: [PATCH 025/243] COND5: the keeper hears a machine away and back from the watchman MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An observer on a machine away is not named in findings-unstated; a machine back has its observers asked again (mesh-watchdogs machine-away and machine-back in consumes). Builds: design 47 G §389 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/events.go | 53 ++++++++++++++++++- .../cmd/mesh-conditions/keeper_test.go | 20 +++++-- .../cmd/mesh-conditions/manifest_test.go | 2 +- modules/mesh-conditions/module.json | 4 +- 4 files changed, 72 insertions(+), 7 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index 830c8be39..4520787dc 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -16,6 +16,7 @@ import ( cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -40,6 +41,12 @@ func (k *Keeper) FindingEvent(e stdio.Envelope) error { if observer, ok := emitterOf(e.Key, EventFindingCleared); ok { return k.FindingCleared(observer, e.Node, e.Body) } + if _, ok := emitterOf(e.Key, watchdogs.EventMachineAway); ok && strings.HasPrefix(e.Key, watchdogs.Seat+".") { + return k.MachinePresence(e.Body, true) + } + if _, ok := emitterOf(e.Key, watchdogs.EventMachineBack); ok && strings.HasPrefix(e.Key, watchdogs.Seat+".") { + return k.MachinePresence(e.Body, false) + } if observer, ok := emitterOf(e.Key, EventFindingsStated); ok { return k.FindingsStated(observer, e.Node, e.Body) } @@ -148,10 +155,52 @@ func storedFinding(f seat.Finding) seat.FindingsByKeyValue { Ref: f.Ref, Run: f.Run, WaitingOn: f.WaitingOn, At: f.At} } +// MachinePresence takes the watchman's word that a machine is away or back (design 47 G §389): an observer on a machine +// away is not named in findings-unstated, and an ask is said again to the modules of a machine back. +func (k *Keeper) MachinePresence(body []byte, away bool) error { + var m struct { + Machine string `json:"machine"` + } + if err := json.Unmarshal(body, &m); err != nil || m.Machine == "" { + k.Logf("the watchman's word on a machine's presence is not readable: %v", err) + return nil + } + k.mu.Lock() + k.init() + if k.away == nil { + k.away = map[string]bool{} + } + if away { + k.away[m.Machine] = true + k.mu.Unlock() + return nil + } + delete(k.away, m.Machine) + var modules []string + for _, v := range k.views { + if v.node == m.Machine && v.observer != "" && !slices.Contains(modules, v.observer) { + modules = append(modules, v.observer) + } + } + k.mu.Unlock() + sort.Strings(modules) + var problems []string + for _, module := range modules { + if _, err := k.Ask(Module, module); err != nil { + problems = append(problems, err.Error()) + } + } + if len(problems) > 0 { + return errors.New(strings.Join(problems, "; ")) + } + return nil +} + // hear subscribes to the observers' events once the runtime serves, asked again until it takes each. func hear(k *Keeper) { - for _, event := range []string{EventFindingWritten, EventFindingCleared, EventFindingsStated} { - pattern := "*." + event + patterns := []string{"*." + EventFindingWritten, "*." + EventFindingCleared, "*." + EventFindingsStated, + watchdogs.Seat + "." + watchdogs.EventMachineAway, watchdogs.Seat + "." + watchdogs.EventMachineBack} + for _, pattern := range patterns { go func() { <-stdio.Served() for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 4105d581f..f8206d4d4 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -539,12 +539,26 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { {"its next findings-stated clears it", func() error { return stated(watchman, "ace", ask, t0.Add(10*time.Minute), nil, f(watchman, "d")) }, []string{"machine.d.silent", "machine.g.silent"}, "its finding went"}, - {"a machine away is not named", func() error { + {"a machine away is not named, and a machine back is asked again", func() error { if _, err := r.k.Ask("jochen", ""); err != nil { return err } - r.k.away = map[string]bool{"g14": true} - return r.k.AskPassed(askedKey(selfCheck, "g14", asked[len(asked)-1]["ask"].(string))) + last := asked[len(asked)-1]["ask"].(string) + away := mustJSON(t, map[string]any{"machine": "g14", "at": r.now}) + if err := r.k.FindingEvent(stdio.Envelope{Key: "mesh-watchdogs.machine-away", Node: "ace", Body: away}); err != nil { + return err + } + if err := r.k.AskPassed(askedKey(selfCheck, "g14", last)); err != nil { + return err + } + before := len(asked) + if err := r.k.FindingEvent(stdio.Envelope{Key: "mesh-watchdogs.machine-back", Node: "ace", Body: away}); err != nil { + return err + } + if len(asked) != before+2 || asked[before]["module"] != watchman || asked[before+1]["module"] != selfCheck { + t.Errorf("a machine back was asked %v", asked[before:]) + } + return nil }, []string{"machine.d.silent", "machine.g.silent"}, ""}, {"resync names one module: its hand act and its ask", func() error { if _, err := r.k.ResyncVerb("person.jochen", selfCheck); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 98ef5b04b..066f9b6b9 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -38,7 +38,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { reads := []string{HavesRead, AttemptsRead} if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act"}) || !slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments", - "*.finding-written", "*.finding-cleared", "*.findings-stated"}) { + "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back"}) { t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes) } for _, r := range m.Reads { diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index 7f1a0c112..3414ae96b 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -27,7 +27,9 @@ "mesh-delivery-assigner.views.assignments", "*.finding-written", "*.finding-cleared", - "*.findings-stated" + "*.findings-stated", + "mesh-watchdogs.machine-away", + "mesh-watchdogs.machine-back" ], "state": [ "conditions", -- 2.54.0 From fc90e6c67b7bea17361a2afc521b98c1b9804bbc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:17:03 +0100 Subject: [PATCH 026/243] OBS1 (artifact-store-tools, mesh-healers, mesh-vault, mesh-watcher, minio, nfs-server): the finding stream in each manifest MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each of the six declares the stream -findings (finding-written.*, finding-cleared.*, findings-stated.*, 7 days, 16 MiB) and consumes mesh-conditions.findings-asked; the manifest tests that hold consumes say it. The other observers' manifests are written by their own families. Builds: design 47 G §387, §383 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/artifact-store-tools/store_test.go | 2 +- modules/artifact-store-tools/module.json | 11 ++++++++++- .../mesh-healers/cmd/mesh-healers/manifest_test.go | 2 +- modules/mesh-healers/module.json | 11 ++++++++++- modules/mesh-vault/module.json | 11 ++++++++++- .../mesh-watcher/cmd/mesh-watcher/manifest_test.go | 2 +- modules/mesh-watcher/module.json | 11 ++++++++++- modules/minio/cmd/object-store/manifest_test.go | 2 +- modules/minio/module.json | 11 ++++++++++- modules/nfs-server/cmd/nfs-server/manifest_test.go | 2 +- modules/nfs-server/module.json | 11 ++++++++++- 11 files changed, 65 insertions(+), 11 deletions(-) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 3729944df..8156df52c 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -474,7 +474,7 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { t.Errorf("invokes %v", m.Invokes) } // ADR 0615: the assigner's wants and the indexer's manifests are asked once and followed by their events, not read. - if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.manifests"}) || + if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.manifests", "mesh-conditions.findings-asked"}) || slices.Contains(m.Reads, WantsRead) || slices.Contains(m.Reads, ManifestsRead) { t.Errorf("consumes %v, reads %v", m.Consumes, m.Reads) } diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index 64f4349cf..8ae3fe620 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -39,7 +39,8 @@ ], "consumes": [ "mesh-delivery-assigner.views.wants", - "mesh-delivery-indexer.views.manifests" + "mesh-delivery-indexer.views.manifests", + "mesh-conditions.findings-asked" ], "state": [ "findings" @@ -85,6 +86,14 @@ "template": "files/store-address" } ], + "streams": [ + { + "name": "artifact-store-tools-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "build": { "artifacts": [ { diff --git a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go index 001969ccb..83f59216b 100644 --- a/modules/mesh-healers/cmd/mesh-healers/manifest_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/manifest_test.go @@ -48,7 +48,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { got, want []string }{ {"emits (healer-acted is the seat's, said on its subject)", m.Emits, nil}, - {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent}}, + {"consumes", m.Consumes, []string{RaisedEvent, ChangedEvent, RecordedEvent, "mesh-conditions.findings-asked"}}, {"invokes", m.Invokes, []string{"seat:mesh-hand-acts.hand-act", "seat:mesh-hand-acts.hand-acts"}}, {"reads", m.Reads, []string{ConditionsRead}}, } { diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index 08871f822..7fe853339 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -16,7 +16,8 @@ "consumes": [ "mesh-conditions.condition-raised", "mesh-conditions.condition-changed", - "mesh-hand-acts.act-recorded" + "mesh-hand-acts.act-recorded", + "mesh-conditions.findings-asked" ], "invokes": [ "seat:mesh-hand-acts.hand-act", @@ -61,6 +62,14 @@ "unknown": "program" } ], + "streams": [ + { + "name": "mesh-healers-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "build": { "artifacts": [ { diff --git a/modules/mesh-vault/module.json b/modules/mesh-vault/module.json index 79a24ea6e..8a33f4943 100644 --- a/modules/mesh-vault/module.json +++ b/modules/mesh-vault/module.json @@ -162,6 +162,14 @@ ] } ], + "streams": [ + { + "name": "mesh-vault-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "build": { "artifacts": [ { @@ -253,6 +261,7 @@ } }, "consumes": [ - "mesh-delivery-assigner.views.assignments" + "mesh-delivery-assigner.views.assignments", + "mesh-conditions.findings-asked" ] } diff --git a/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go b/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go index c419583fe..f67524610 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/manifest_test.go @@ -39,7 +39,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { t.Fatal(err) } // Where the bus runs is the assigner's seat-holders, asked once and followed by its events (ADR 0615), never read. - if !reflect.DeepEqual(m.Consumes, []string{"mesh-self-check.checked", assignerModule + ".views." + mesh.StateSeatHolders}) { + if !reflect.DeepEqual(m.Consumes, []string{"mesh-self-check.checked", assignerModule + ".views." + mesh.StateSeatHolders, "mesh-conditions.findings-asked"}) { t.Fatalf("consumes %v", m.Consumes) } if !reflect.DeepEqual(m.Invokes, []string{SeatHoldersTool, m.Module + ".watcher_ping", "seat:mesh-hand-acts.hand-act"}) || len(m.Reads) != 0 { diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index e4555e822..5a855c5fb 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -4,7 +4,8 @@ "slug": "watch", "consumes": [ "mesh-self-check.checked", - "mesh-delivery-assigner.views.seat-holders" + "mesh-delivery-assigner.views.seat-holders", + "mesh-conditions.findings-asked" ], "invokes": [ "mesh-delivery-assigner.assigner_read_seat_holders", @@ -53,6 +54,14 @@ "template": "files/settings.json" } ], + "streams": [ + { + "name": "mesh-watcher-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "build": { "artifacts": [ { diff --git a/modules/minio/cmd/object-store/manifest_test.go b/modules/minio/cmd/object-store/manifest_test.go index e299d71b5..ad150f08f 100644 --- a/modules/minio/cmd/object-store/manifest_test.go +++ b/modules/minio/cmd/object-store/manifest_test.go @@ -41,7 +41,7 @@ func TestManifestClaimsTheSeat(t *testing.T) { } invokes, _ := m["invokes"].([]any) consumes, _ := m["consumes"].([]any) - if _, reads := m["reads"]; reads || len(invokes) != 2 || invokes[0] != SeatHoldersTool || invokes[1] != "seat:mesh-hand-acts.hand-act" || len(consumes) != 1 || consumes[0] != assignerModule+".views."+mesh.StateSeatHolders { + if _, reads := m["reads"]; reads || len(invokes) != 2 || invokes[0] != SeatHoldersTool || invokes[1] != "seat:mesh-hand-acts.hand-act" || len(consumes) != 2 || consumes[0] != assignerModule+".views."+mesh.StateSeatHolders || consumes[1] != "mesh-conditions.findings-asked" { t.Fatalf("reads %v, invokes %v, consumes %v: it asks the assigner's seat-holders once and follows its events (ADR 0615)", m["reads"], invokes, consumes) } var goBundle map[string]any diff --git a/modules/minio/module.json b/modules/minio/module.json index 0df4e3615..47f19cdf6 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -44,7 +44,8 @@ "seat:mesh-hand-acts.hand-act" ], "consumes": [ - "mesh-delivery-assigner.views.seat-holders" + "mesh-delivery-assigner.views.seat-holders", + "mesh-conditions.findings-asked" ], "state": [ "findings" @@ -192,6 +193,14 @@ "package": "minio-client" } ], + "streams": [ + { + "name": "minio-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "build": { "artifacts": [ { diff --git a/modules/nfs-server/cmd/nfs-server/manifest_test.go b/modules/nfs-server/cmd/nfs-server/manifest_test.go index 8af51fd5a..e59ac3b94 100644 --- a/modules/nfs-server/cmd/nfs-server/manifest_test.go +++ b/modules/nfs-server/cmd/nfs-server/manifest_test.go @@ -152,7 +152,7 @@ func TestTheManifestCarriesGrantsTheRosterTheWantsAndTheRemovalOrder(t *testing. if len(full.Reads) != 2 || full.Reads[0] != diskreadings.ReadingsRead || full.Reads[1] != MountsState { t.Errorf("reads: %v", full.Reads) } - if len(full.Invokes) != 2 || full.Invokes[0] != SeatHoldersTool || full.Invokes[1] != "seat:mesh-hand-acts.hand-act" || len(full.Consumes) != 1 || full.Consumes[0] != assignerModule+".views."+mesh.StateSeatHolders { + if len(full.Invokes) != 2 || full.Invokes[0] != SeatHoldersTool || full.Invokes[1] != "seat:mesh-hand-acts.hand-act" || len(full.Consumes) != 2 || full.Consumes[0] != assignerModule+".views."+mesh.StateSeatHolders || full.Consumes[1] != "mesh-conditions.findings-asked" { t.Errorf("invokes %v, consumes %v", full.Invokes, full.Consumes) } roster := full.Facts["machines"] diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index f4567f7b3..6f77dba2c 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -172,6 +172,14 @@ "template": "files/shares.conf" } ], + "streams": [ + { + "name": "nfs-server-findings", + "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], + "max-age-days": 7, + "max-mib": 16 + } + ], "build": { "artifacts": [ { @@ -197,6 +205,7 @@ "seat:mesh-hand-acts.hand-act" ], "consumes": [ - "mesh-delivery-assigner.views.seat-holders" + "mesh-delivery-assigner.views.seat-holders", + "mesh-conditions.findings-asked" ] } -- 2.54.0 From 076f63f93298ae25cadd56d3a115db82ba9e7ac0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:18:25 +0100 Subject: [PATCH 027/243] COND5 (go.mod): the keeper's go.mod requires the watchdogs seat package it imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The repository check requiredmodules refused events.go's import of the mesh-watchdogs seat. Builds: design 47 G §389 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-conditions/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod index 4666a41cc..717626fd6 100644 --- a/modules/mesh-conditions/go.mod +++ b/modules/mesh-conditions/go.mod @@ -11,6 +11,7 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 -- 2.54.0 From 6062feaa2245bfa87cbcd39b8fcc6e7639427ede Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:23:40 +0100 Subject: [PATCH 028/243] NATS3 (part, branch-independent): which module sends, and a catalog entry read for the bus module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sends reads a claim of a seat whose definition has a verb or an event, or an event of its own; AsksOf reads an entry's bus-state asks and claims. The follower's branch and the wiring wait on the design's answer. Builds: design 47 G §344, §433 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/nats/cmd/nats-tools/catalogview.go | 57 +++++++++++++++++++ .../nats/cmd/nats-tools/catalogview_test.go | 28 +++++++++ modules/nats/internal/keeper/keeper_test.go | 29 ++++++++++ modules/nats/internal/keeper/plan.go | 17 ++++++ 4 files changed, 131 insertions(+) create mode 100644 modules/nats/cmd/nats-tools/catalogview.go create mode 100644 modules/nats/cmd/nats-tools/catalogview_test.go diff --git a/modules/nats/cmd/nats-tools/catalogview.go b/modules/nats/cmd/nats-tools/catalogview.go new file mode 100644 index 000000000..ee8fb77fe --- /dev/null +++ b/modules/nats/cmd/nats-tools/catalogview.go @@ -0,0 +1,57 @@ +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "nats-tools/internal/keeper" +) + +// The keeper reads a module's asks from the module's entry in the catalog half of its revision view, not from the +// indexer's event (design 47 G §344, §433). + +// Asks is what one catalog graph entry says of a module for the keeper: what its manifest asks of the bus as an accepted +// verdict, the seats it claims, and whether it declares an event of its own (the facts keeper.Sends reads). +type Asks struct { + Accepted keeper.Accepted + Claims []string + EmitsOwn bool +} + +// AsksOf reads one entry. An entry whose manifest is not accepted asks nothing of the bus now, as an accepted verdict of +// no entries does; the digest it carries is of the entries it asks, so an entry changed in nothing the bus module reads +// is not applied again. +func AsksOf(e catalog.CatalogEntry) (Asks, error) { + if e.Module == "" { + return Asks{}, fmt.Errorf("a catalog entry names no module") + } + raw, err := json.Marshal(e.Manifest) + if err != nil { + return Asks{}, fmt.Errorf("the manifest of %s is not readable: %w", e.Module, err) + } + var m catalogue.Manifest + if err := json.Unmarshal(raw, &m); err != nil { + return Asks{}, fmt.Errorf("the manifest of %s is not readable: %w", e.Module, err) + } + state := m.BusState + if e.Verdict != "" && e.Verdict != "accepted" { + state = nil + } + sum := sha256.Sum256(mustJSON(state)) + out := Asks{Accepted: keeper.Accepted{Module: e.Module, Digest: hex.EncodeToString(sum[:]), BusState: state}, + EmitsOwn: len(m.Emits) > 0 || len(e.Streams) > 0} + for _, c := range m.Claims { + out.Claims = append(out.Claims, c.Name) + } + return out, nil +} + +func mustJSON(v any) []byte { + b, _ := json.Marshal(v) + return b +} diff --git a/modules/nats/cmd/nats-tools/catalogview_test.go b/modules/nats/cmd/nats-tools/catalogview_test.go new file mode 100644 index 000000000..b49659993 --- /dev/null +++ b/modules/nats/cmd/nats-tools/catalogview_test.go @@ -0,0 +1,28 @@ +package main + +import ( + "testing" + + catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +func TestAsksOf(t *testing.T) { + m := map[string]any{"module": "m", "claims": []any{map[string]any{"name": "seat-a"}}, "emits": []any{"done"}} + a, err := AsksOf(catalog.CatalogEntry{Module: "m", Verdict: "accepted", Manifest: m}) + if err != nil { + t.Fatal(err) + } + if len(a.Claims) != 1 || a.Claims[0] != "seat-a" || !a.EmitsOwn || a.Accepted.Module != "m" || a.Accepted.Digest == "" { + t.Errorf("read %+v", a) + } + b, _ := AsksOf(catalog.CatalogEntry{Module: "m", Manifest: map[string]any{"module": "m"}}) + if b.EmitsOwn || len(b.Claims) != 0 { + t.Errorf("an entry with neither: %+v", b) + } + if b.Accepted.Digest != a.Accepted.Digest { + t.Errorf("the digest is of the bus-state asked alone: %s vs %s", b.Accepted.Digest, a.Accepted.Digest) + } + if _, err := AsksOf(catalog.CatalogEntry{}); err == nil { + t.Error("an entry with no module was read") + } +} diff --git a/modules/nats/internal/keeper/keeper_test.go b/modules/nats/internal/keeper/keeper_test.go index 735ed9a4d..ef9af998f 100644 --- a/modules/nats/internal/keeper/keeper_test.go +++ b/modules/nats/internal/keeper/keeper_test.go @@ -14,6 +14,7 @@ import ( occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatdef" ) // fakeBus holds objects by stream, as a bus would, and refuses what a test tells it to. @@ -794,3 +795,31 @@ func TestTheKeeperMakesWhatAVerdictAsksForAndSaysIt(t *testing.T) { }) } } + +func TestSends(t *testing.T) { + defs := map[string]seatdef.Definition{ + "verbed": {Verbs: []seatdef.Verb{{Name: "go"}}}, + "evented": {Events: []seatdef.Message{{Name: "done"}}}, + "included": {Includes: []string{"part"}}, + "bare": {}, + } + for _, c := range []struct { + name string + claims []string + own bool + want bool + }{ + {"a seat with a verb", []string{"verbed"}, false, true}, + {"a seat with an event", []string{"evented"}, false, true}, + {"a seat that includes parts", []string{"included"}, false, true}, + {"a seat with neither", []string{"bare"}, false, false}, + {"a seat the list does not hold", []string{"unknown"}, false, false}, + {"an event of its own", nil, true, true}, + {"nothing claimed", nil, false, false}, + {"one of two claims sends", []string{"bare", "verbed"}, false, true}, + } { + if got := Sends(c.claims, c.own, defs); got != c.want { + t.Errorf("%s: Sends = %v, want %v", c.name, got, c.want) + } + } +} diff --git a/modules/nats/internal/keeper/plan.go b/modules/nats/internal/keeper/plan.go index 080d61755..ad89ae4e9 100644 --- a/modules/nats/internal/keeper/plan.go +++ b/modules/nats/internal/keeper/plan.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/buscompose" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/membership" + "git.novox.be/novox/mesh-sdk/go/seatdef" ) // The kinds of bus object an entry is made of: a stream, a key-value bucket (stream KV_) and an object store @@ -48,6 +49,22 @@ const ( SentAgeDays = 7 ) +// Sends says whether a module sends (design 47 G §344): it claims a seat whose definition has a verb or an event, or it +// declares an event of its own (emitsOwn). claims are the seats its catalog graph entry claims, defs the seat list's +// definitions by seat; a claimed seat the list does not hold counts for nothing. A definition that includes parts takes +// their verbs and events whole (seatdef.Resolve), so one counts as having some. +func Sends(claims []string, emitsOwn bool, defs map[string]seatdef.Definition) bool { + if emitsOwn { + return true + } + for _, c := range claims { + if d, ok := defs[c]; ok && (len(d.Verbs) > 0 || len(d.Events) > 0 || len(d.Includes) > 0) { + return true + } + } + return false +} + // Settings are one bus object's settings, as asked and as held: what check compares. type Settings struct { // MaxBytes is its cap in bytes; MaxAgeSeconds its age, 0 for none. -- 2.54.0 From a24f7809a6ddb6238e5049afc0b7fe49cad0951b Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 12:28:45 +0100 Subject: [PATCH 029/243] COND6 (partial, not landed): the keeper's sources, holder and clears from the catalog half and the assignment half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixed Observers list and the engineModule constant are removed; sources are the modules whose catalog entry declares the finding events, a module in no branch's revision is cleared (module left the catalog), a stopped module no longer placed is cleared (module left the node), Keeper.Holder is derived from the entry's claims. The per-branch catalog followers in main.go wait on a design question (how a part adopts one catalog view per registered branch); the whole integration test is red until then. Builds: design 47 G §365, §196, §200, §364, §384 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/catalog.go | 231 ++++++++++++++++++ .../cmd/mesh-conditions/events.go | 10 +- .../cmd/mesh-conditions/keeper.go | 28 +-- .../cmd/mesh-conditions/keeper_test.go | 119 +++++---- .../cmd/mesh-conditions/main.go | 10 +- .../cmd/mesh-conditions/manifest_test.go | 4 +- modules/mesh-conditions/go.mod | 2 + modules/mesh-conditions/module.json | 7 +- 8 files changed, 329 insertions(+), 82 deletions(-) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/catalog.go diff --git a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go new file mode 100644 index 000000000..2b176aa2d --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go @@ -0,0 +1,231 @@ +package main + +// The keeper finds its sources in the catalog (design 47 G §365): the modules whose catalog entry declares the finding +// events of §383, in any watched branch's newest revision. A module in no branch's revision has its conditions cleared with +// `module left the catalog`; a module's conditions on a node are cleared with `module left the node` once the node-engine +// says the module stopped there and the newest assignment graph revision no longer places it. No module is named here (§7). + +import ( + "encoding/json" + "slices" + "sort" + "strings" + + catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + + engine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" +) + +// The reasons a catalog or an assignment takes a module's conditions away (design 47 G §365). +const ( + whyLeftCatalog = "module left the catalog" + whyLeftNode = "module left the node" +) + +// declaresFindings says whether the entry declares the finding events of §383 in its streams. +func declaresFindings(e catseat.CatalogEntry) bool { + if slices.ContainsFunc(e.Streams, func(s string) bool { return strings.Contains(s, EventFindingWritten) }) { + return true + } + streams, _ := e.Manifest["streams"].([]any) + for _, s := range streams { + m, _ := s.(map[string]any) + events, _ := m["events"].([]any) + for _, ev := range events { + if name, _ := ev.(string); strings.HasPrefix(name, EventFindingWritten) { + return true + } + } + } + return false +} + +// CatalogHeld takes the newest catalog revision of one watched branch: the sources are the modules declaring the finding +// events in any branch, and the conditions of a module in no branch are cleared. +func (k *Keeper) CatalogHeld(branch string, entries map[string]catseat.CatalogEntry) error { + k.mu.Lock() + k.init() + k.halves[branch] = entries + var gone []string + for observer := range k.findings { + if observer == Module { + continue + } + if _, ok := k.entry(observer); !ok { + gone = append(gone, observer) + } + } + k.mu.Unlock() + sort.Strings(gone) + var first error + for _, observer := range gone { + k.clearObserver(observer, "", whyLeftCatalog, &first) + } + return first +} + +// entry is the observer's entry in the first branch (in name order) that holds one. Called with mu held. +func (k *Keeper) entry(observer string) (catseat.CatalogEntry, bool) { + branches := make([]string, 0, len(k.halves)) + for b := range k.halves { + branches = append(branches, b) + } + sort.Strings(branches) + for _, b := range branches { + if e, ok := k.halves[b][observer]; ok { + return e, true + } + } + return catseat.CatalogEntry{}, false +} + +// isSource says whether the keeper takes findings from the module: its catalog entry declares the finding events. +func (k *Keeper) isSource(observer string) bool { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + branches := make([]string, 0, len(k.halves)) + for b := range k.halves { + branches = append(branches, b) + } + for _, b := range branches { + if e, ok := k.halves[b][observer]; ok && declaresFindings(e) { + return true + } + } + return false +} + +// catalogHolder is Keeper.Holder: an observer whose catalog entry claims only seats held once for the mesh is held on the +// node the assignments name as the holder of one of them (design 47 G §384); unknown, it is taken from every node. +func (k *Keeper) catalogHolder(observer string) (string, bool) { + k.mu.Lock() + defer k.mu.Unlock() + k.init() + e, ok := k.entry(observer) + if !ok { + return "", false + } + claims, _ := e.Manifest["claims"].([]any) + if len(claims) == 0 { + return "", false + } + for _, c := range claims { + m, _ := c.(map[string]any) + if scope, _ := m["scope"].(string); scope != "mesh" { + return "", false + } + } + nodes := make([]string, 0, 1) + for hk, module := range k.holders { + if module == observer { + node, _, _ := strings.Cut(hk, ".") + nodes = append(nodes, node) + } + } + if len(nodes) == 0 { + return "", false + } + sort.Strings(nodes) + return nodes[0], true +} + +// AssignmentHeld takes the newest assignment graph revision: the modules it places on each node. +func (k *Keeper) AssignmentHeld(rev wants.AssignmentRevision) error { + k.mu.Lock() + k.init() + k.placed = map[string]bool{} + for _, e := range rev.Entries { + k.placed[e.Node+"."+e.Module] = true + } + k.placedHeld = true + k.mu.Unlock() + return k.sweepStopped() +} + +// EngineRunning takes a node-engine's `running` for a module on a node: stopped, it is kept until the assignment graph +// places the module there no longer. +func (k *Keeper) EngineRunning(node string, body []byte) error { + var r engine.Running + if err := json.Unmarshal(body, &r); err != nil || r.Module == "" || r.Node == "" { + k.Logf("the node-engine's running on %s is not readable: %v", node, err) + return nil + } + if node != "" && r.Node != node { + k.Logf("the node-engine's running on %s names the node %s: refused", node, r.Node) + return nil + } + k.mu.Lock() + k.init() + if r.Stopped { + k.stopped[r.Node+"."+r.Module] = true + } else { + delete(k.stopped, r.Node+"."+r.Module) + } + k.mu.Unlock() + return k.sweepStopped() +} + +// sweepStopped clears the conditions of each module stopped on a node that the held assignment revision does not place +// there; before an assignment revision is held, nothing is cleared. +func (k *Keeper) sweepStopped() error { + k.mu.Lock() + k.init() + var left []string + if k.placedHeld { + for pair := range k.stopped { + if !k.placed[pair] { + left = append(left, pair) + } + } + } + for _, pair := range left { + delete(k.stopped, pair) + } + k.mu.Unlock() + sort.Strings(left) + var first error + for _, pair := range left { + node, module, _ := strings.Cut(pair, ".") + k.clearObserver(module, node, whyLeftNode, &first) + } + return first +} + +// clearObserver clears every condition the module has, or has from the node when node is not empty, with the reason, and +// forgets what it heard of the module there. +func (k *Keeper) clearObserver(observer, node, why string, first *error) { + k.mu.Lock() + var keys []string + for key := range k.findings[observer] { + if node == "" || k.keyNode[observer][key] == node { + keys = append(keys, key) + } + } + for name, v := range k.views { + if v.observer == observer && (node == "" || v.node == node) { + for key := range v.Pending { + _ = k.Store.Delete(BoundsState, pendingKey(observer, v.node, key)) + } + for ask := range v.Asks { + _ = k.Store.Delete(BoundsState, askedKey(observer, v.node, ask)) + } + delete(k.views, name) + } + } + k.mu.Unlock() + sort.Strings(keys) + for _, key := range keys { + k.mu.Lock() + k.whys[key] = why + k.mu.Unlock() + if err := k.Finding(observer, key, nil, true); err != nil && *first == nil { + *first = err + } + k.mu.Lock() + delete(k.whys, key) + delete(k.keyNode[observer], key) + k.mu.Unlock() + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index 4520787dc..ec7abe5d8 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -14,6 +14,7 @@ import ( "strings" "time" + engine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" watchdogs "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs" @@ -47,15 +48,15 @@ func (k *Keeper) FindingEvent(e stdio.Envelope) error { if _, ok := emitterOf(e.Key, watchdogs.EventMachineBack); ok && strings.HasPrefix(e.Key, watchdogs.Seat+".") { return k.MachinePresence(e.Body, false) } + if e.Key == engine.Seat+"."+engine.EventRunning { + return k.EngineRunning(e.Node, e.Body) + } if observer, ok := emitterOf(e.Key, EventFindingsStated); ok { return k.FindingsStated(observer, e.Node, e.Body) } return nil } -// isSource says whether the keeper takes findings from the module. -func (k *Keeper) isSource(observer string) bool { return slices.Contains(Observers, observer) } - // FindingWritten raises or changes the condition of the finding an observer says on a node. A body that is not a finding // the keeper can raise, or names another observer, is said in the journal and changes nothing. func (k *Keeper) FindingWritten(observer, node string, body []byte) error { @@ -199,7 +200,8 @@ func (k *Keeper) MachinePresence(body []byte, away bool) error { // hear subscribes to the observers' events once the runtime serves, asked again until it takes each. func hear(k *Keeper) { patterns := []string{"*." + EventFindingWritten, "*." + EventFindingCleared, "*." + EventFindingsStated, - watchdogs.Seat + "." + watchdogs.EventMachineAway, watchdogs.Seat + "." + watchdogs.EventMachineBack} + watchdogs.Seat + "." + watchdogs.EventMachineAway, watchdogs.Seat + "." + watchdogs.EventMachineBack, + engine.Seat + "." + engine.EventRunning} for _, pattern := range patterns { go func() { <-stdio.Served() diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index ae523a835..6753dffc2 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -25,6 +25,7 @@ import ( cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers" + catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" ) // The modules whose states the keeper reads by name: the holders of the health seats (their seats' names are the @@ -35,8 +36,6 @@ const ( healersModule = "mesh-healers" // The holders of the assignments and the catalog, whose views and findings the keeper follows (the manifest test holds // these equal to the manifest's `consumes` and `invokes`). - // engineModule is the node-engine, an observer of §387 (design 47 G §387 names mesh-host). - engineModule = "mesh-host" assignerModule = "mesh-delivery-assigner" indexerModule = "mesh-delivery-indexer" ) @@ -55,23 +54,6 @@ const ( // HistoryKeptFor is how long a transition is kept (clause 3). const HistoryKeptFor = 90 * 24 * time.Hour -// Observers are the modules whose `findings` the keeper reads (clause 2; ADR 0565 §8 for the assigner's; the lead's -// table of 2026-10-29 for the provisioner's and the router's; ADR 0571 clause 2 for the seat artifact-store's holder, -// which observes S25, image-missing). -var Observers = []string{watchmanModule, selfCheckModule, healersModule, assignerModule, - "mesh-provisioner", "messenger", artifactStore, configModule, vaultModule, - // The readers of the indexer's and the assigner's state say a first view unanswered past its bound in their own findings - // (ADR 0615 §6). - "mesh-watcher", "minio", "nfs-server", gitea, "mesh-build-go", - // The builder's, the indexer's (asked of its read tool and followed by its views events, ADR 0615 §1) and the other - // readers' of the indexer's and the assigner's state (the registry milestone's join). - builder, indexer, "mesh-delivery-checker", "mesh-delivery-teller", "mesh-delivery-view", enrolment, - // The tool runner's and claude-code's own first views (views.UnansweredFinding in their findings, ADR 0615 §6). - "node-tools", "claude-code", - // The node-engine's: a node that has read no seat list five minutes after its start, seat-list-unread (ADR 0636 §1; - // issue 1966: written on ace and raised by nobody). - engineModule} - // The other states it reads, as `.`. const ( AttemptsRead = healersModule + "." + meshhealers.StateAttempts @@ -158,6 +140,13 @@ type Keeper struct { Record func(what, why, verb string) error // away are the machines the watchman says are away (design 47 G §389). away map[string]bool + // halves are the newest catalog revisions of the watched branches, by branch and module (catalog.go); placed are the + // `.` the newest assignment graph revision places, placedHeld whether one is held; stopped are the + // `.` the node-engines say stopped. + halves map[string]map[string]catseat.CatalogEntry + placed map[string]bool + placedHeld bool + stopped map[string]bool } func (k *Keeper) init() { @@ -168,6 +157,7 @@ func (k *Keeper) init() { map[string]string{}, map[string]held{}, map[string]clearedMemo{} k.passed, k.outOfHold, k.holders = map[string]string{}, map[string]string{}, map[string]string{} k.assignments = assignerclient.NewKeys() + k.halves, k.placed, k.stopped = map[string]map[string]catseat.CatalogEntry{}, map[string]bool{}, map[string]bool{} k.views, k.keyNode, k.whys = map[string]*sourceView{}, map[string]map[string]string{}, map[string]string{} } if k.Now == nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index f8206d4d4..446b155fd 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -3,10 +3,9 @@ package main import ( "encoding/json" "fmt" - "io/fs" - "os" - "path/filepath" - "regexp" + nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" + catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "slices" "sort" "strings" @@ -112,9 +111,27 @@ func newRig() *rig { r.said = append(r.said, e) return nil }} + // The catalog half of the watched branch: the modules the tests hear findings from declare the finding events (§365); + // the self-check's holder claims a seat held once for the mesh. + r.half(watchmanModule, "node") + r.half(selfCheckModule, "mesh") return r } +// half puts a module in the watched branch's catalog half, declaring the finding events and claiming one seat of scope. +func (r *rig) half(module, scope string) { + r.k.mu.Lock() + r.k.init() + if r.k.halves["main"] == nil { + r.k.halves["main"] = map[string]catseat.CatalogEntry{} + } + r.k.halves["main"][module] = catseat.CatalogEntry{Module: module, Manifest: map[string]any{ + "claims": []any{map[string]any{"name": module, "scope": scope}}, + "streams": []any{map[string]any{"name": module + "-findings", "events": []any{"finding-written.*", "finding-cleared.*", "findings-stated.*"}}}, + }} + r.k.mu.Unlock() +} + func (r *rig) changes() string { var out []string for _, e := range r.said { @@ -346,10 +363,6 @@ func TestEveryControllerKindHasItsHome(t *testing.T) { if h, ok := Homes[kind]; !ok || h.Observer != observer || h.Carried { t.Errorf("%s's home is %+v", kind, h) } - // The keeper reads every observer's findings that a kind's home names (the registry milestone's join). - if !slices.Contains(Observers, observer) { - t.Errorf("%s's observer %s is not read by the keeper", kind, observer) - } } for _, kind := range []string{"resolver-wrong", "holder-silent", "archives-missing", "archives-unheld", "binding-kept", "binding-moved", "binding-moving", "grant-composes-nothing", "root-not-free", "cleanup-waiting", "data-held-twice"} { @@ -370,18 +383,6 @@ func TestEveryControllerKindHasItsHome(t *testing.T) { t.Errorf("%s's home is %+v", kind, h) } } - // Each module that writes a reader's unanswered first view (ADR 0615 §6), or that a delivery part's consumer stopped - // hearing, is read: the delivery parts and the tool runner by name (other repositories), and every catalogue module - // whose code writes a views finding, found in the catalogue's tree, so a new writer cannot go unobserved. - // And the node-engine, whose node writes seat-list-unread when it has read no seat list (ADR 0636 §1, issue 1966). - readers := []string{watchman, "mesh-delivery-teller", enrolment, builder, "mesh-delivery-checker", "mesh-delivery-view", - indexer, assignerModule, configModule, vaultModule, gitea, "node-tools", "mesh-host"} - readers = append(readers, viewFindingWriters(t, "../../..")...) - for _, reader := range readers { - if !slices.Contains(Observers, reader) { - t.Errorf("the reader %s's findings are not read by the keeper", reader) - } - } // 19 of ADR 0564, the 63 of the table, plan-failed and stalled, mesh-config's four (design 32), the readers' unanswered // first view, the owners' view too large, the builder's four, the indexer's four and the package seats' token refused. // And the builder's three publishing kinds and the delivery parts' family not heard: 103. @@ -393,40 +394,6 @@ func TestEveryControllerKindHasItsHome(t *testing.T) { } } -// viewFindingWriters are the catalogue modules (under modules) whose non-test Go code writes a views finding into its own -// findings: views.UnansweredFinding, views.TooLargeFinding, views.UnsaidFinding or views.OwnFindings. -func viewFindingWriters(t *testing.T, modules string) []string { - t.Helper() - writes := regexp.MustCompile(`(views\.(UnansweredFinding|TooLargeFinding|UnsaidFinding)|meshconditionsclient\.FindingWriter)\b`) - var out []string - entries, err := os.ReadDir(modules) - if err != nil { - t.Fatal(err) - } - for _, e := range entries { - if !e.IsDir() || e.Name() == Module { - continue - } - found := false - _ = filepath.WalkDir(filepath.Join(modules, e.Name()), func(path string, d fs.DirEntry, err error) error { - if err != nil || found || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { - return nil - } - if src, err := os.ReadFile(path); err == nil && writes.Match(src) { - found = true - } - return nil - }) - if found { - out = append(out, e.Name()) - } - } - if len(out) < 5 { - t.Fatalf("found %d catalogue modules writing views findings in %s: the tree was not read (%v)", len(out), modules, out) - } - return out -} - // ADR 0634 §1, §4 (issue 1966): the keeper reads a condition written before the seat named its layout (the controller's // body: `silenced` null, `actions` empty, the times with an offset) and writes it back in the seat's layout. func TestTheKeeperReadsTheOldLayoutAndWritesTheSeatsLayout(t *testing.T) { @@ -569,6 +536,50 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { } return nil }, []string{"machine.d.silent", "machine.g.silent"}, ""}, + // Design 47 G §365: the sources are the catalog's, a module that left the catalog or the node loses its conditions. + {"a module not declaring the finding events is no source, one the catalog half declares is heard with no code change", func() error { + if err := r.k.FindingWritten("fixture", "ace", mustJSON(t, f("fixture", "z"))); err != nil { + return err + } + if _, ok := r.open("machine.z.silent"); ok { + t.Errorf("a module in no catalog half raised a condition") + } + r.half("fixture", "node") + return r.k.FindingWritten("fixture", "ace", mustJSON(t, f("fixture", "z"))) + }, []string{"machine.d.silent", "machine.g.silent", "machine.z.silent"}, ""}, + {"an observer claiming only mesh seats is held where the assignments place it, with no fixed name", func() error { + r.k.holders["g14.mesh-self-check"] = selfCheck + if node, once := r.k.catalogHolder(selfCheck); node != "g14" || !once { + t.Errorf("the holder of %s is %q, %v", selfCheck, node, once) + } + if _, once := r.k.catalogHolder("fixture"); once { + t.Errorf("a node seat's module is held once") + } + return nil + }, []string{"machine.d.silent", "machine.g.silent", "machine.z.silent"}, ""}, + {"stopped but still placed, or placed by no revision held, clears nothing", func() error { + body := mustJSON(t, nodeengine.Running{Module: "fixture", Node: "ace", Stopped: true, At: r.now}) + if err := r.k.EngineRunning("ace", body); err != nil { + return err + } + return r.k.AssignmentHeld(meshwants.AssignmentRevision{Entries: []meshwants.NodeEntryRef{{Module: "fixture", Node: "ace"}}}) + }, []string{"machine.d.silent", "machine.g.silent", "machine.z.silent"}, ""}, + {"stopped and no longer placed is cleared as left the node", func() error { + return r.k.AssignmentHeld(meshwants.AssignmentRevision{Revision: 2}) + }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftNode}, + {"an assignment revision alone clears nothing, and a module that left the catalog is cleared", func() error { + if err := r.k.FindingWritten("fixture", "ace", mustJSON(t, f("fixture", "z"))); err != nil { + return err + } + r.said = nil + if err := r.k.AssignmentHeld(meshwants.AssignmentRevision{Revision: 3}); err != nil { + return err + } + if _, ok := r.open("machine.z.silent"); !ok { + t.Errorf("an assignment revision alone cleared a module's condition") + } + return r.k.CatalogHeld("main", map[string]catseat.CatalogEntry{watchmanModule: {}, selfCheckModule: {}}) + }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftCatalog}, } for _, s := range steps { if err := s.do(); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index cbec7f8c2..9b5a81ddc 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -18,6 +18,8 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" + meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -59,7 +61,9 @@ func sayOnSeat(e Event) error { // NewKeeper is the keeper over the runtime. func NewKeeper() *Keeper { - return &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf} + k := &Keeper{Store: runtimeStore{}, Say: sayOnSeat, Now: time.Now, Logf: logf} + k.Holder = k.catalogHolder + return k } // watch keeps a watch of one state, said blind while it cannot be read: a state not on the bus yet (an observer not @@ -146,6 +150,10 @@ func NewViews(k *Keeper) *views.Reader { v := &views.Reader{Module: Module, Logf: logf, Findings: ownFindings{k}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} v.Follow(assignerModule, "wants", WantsTool) v.Follow(assignerModule, "assignments", AssignmentsTool) + // The newest assignment graph revision, which says where a module that stopped is placed no more (design 47 G §365). + if _, err := meshwantsclient.WireAssignment(v, stdio.Ask, func(rev meshwants.AssignmentRevision) { _ = k.AssignmentHeld(rev) }); err != nil { + logf("the assignment half is not followed: %v", err) + } return v } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 066f9b6b9..6dc3c9aa5 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -36,9 +36,9 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { // The assigner's wants and assignments are asked once of their read tools and followed by their events (ADR 0615 §1); // the observers' findings are heard as events and no `findings` is read (design 47 G §383). reads := []string{HavesRead, AttemptsRead} - if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act"}) || + if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.revision"}) || !slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments", - "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back"}) { + "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-wants.assignment-revision", "node-engine.running"}) { t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes) } for _, r := range m.Reads { diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod index 717626fd6..7ce3ace27 100644 --- a/modules/mesh-conditions/go.mod +++ b/modules/mesh-conditions/go.mod @@ -12,6 +12,8 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshwatchdogs v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index 3414ae96b..b0431616d 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -20,7 +20,8 @@ "invokes": [ "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-assigner.assigner_read_assignments", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "seat:mesh-wants.revision" ], "consumes": [ "mesh-delivery-assigner.views.wants", @@ -29,7 +30,9 @@ "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", - "mesh-watchdogs.machine-back" + "mesh-watchdogs.machine-back", + "mesh-wants.assignment-revision", + "node-engine.running" ], "state": [ "conditions", -- 2.54.0 From bc2e5574ef5b1b294c732adfe110d510afbe3054 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:01:27 +0100 Subject: [PATCH 030/243] GIT7, GIT8: the forge module keeps the runner's failed required steps and refuses a head by them; git_failed_steps and git_failed_steps_drop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit merge-pull refuses a head at whose fingerprint a run failed a required step (policy-failed, naming run, module, step), from the module's own state failed-steps kept from step-ended, with the seen keys of every step end and the deletion once no watched branch's head nor open pull request's head names a commit. The two tools read it with its fill and drop a key after the hand act. Builds: design 47 G §39, §83, §84, §85, §135, §475, §487, §548 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/gitea/failedsteps.ts | 249 +++++++++++++++++++++++++ modules/gitea/index.ts | 38 +++- modules/gitea/module.json | 15 +- modules/gitea/registrations.ts | 12 +- modules/gitea/seat.ts | 21 ++- modules/gitea/test/failedsteps.test.ts | 98 ++++++++++ modules/gitea/test/guard.test.ts | 14 +- modules/gitea/tools/failedsteps.ts | 44 +++++ 8 files changed, 477 insertions(+), 14 deletions(-) create mode 100644 modules/gitea/failedsteps.ts create mode 100644 modules/gitea/test/failedsteps.test.ts create mode 100644 modules/gitea/tools/failedsteps.ts diff --git a/modules/gitea/failedsteps.ts b/modules/gitea/failedsteps.ts new file mode 100644 index 000000000..bbdb5f89c --- /dev/null +++ b/modules/gitea/failedsteps.ts @@ -0,0 +1,249 @@ +// **The forge module judges a head by the failed steps it heard, never by a call** (novox/hq design 47 §39, §475, §487, §548, +// §135): the seat git's `merge-pull` refuses a head at whose build-input fingerprint any run failed a `required` step, from +// the module's own state `failed-steps`, kept from the runner's `step-ended`, so a head merges while the runner is down. +// +// Two key forms live in the state: +// .. a failed `required` step (the contract's FailedStep): the run that failed it and every +// commit it was seen at; cleared by a later passing `rerun` of the same step. +// seen.. the build-input fingerprints the runs named for that commit and module, written from +// every `step-ended` that carries a pipeline's commit and fingerprint, whatever its outcome +// (§487, §548), so a head whose runs all ended before another commit of the same content +// failed is still refused. +// +// An entry is deleted once none of its commits is a watched branch's head or the head an open pull request names (§135), +// judged on each `branch-moved` and each pull request's close (Heads, below). + +import type { State } from "@novox/mesh-sdk/state"; +import { + failedStepsByModuleFingerprintStepKey as failedKey, type FailedStep, type PullClosedEvent, type PullMergedEvent, type PullUpdatedEvent, +} from "@novox/mesh-delivery-contracts-git"; +import type { StepEndedEvent } from "@novox/mesh-delivery-contracts-mesh-pipeline-runner"; + +/** The key prefix of the commits' fingerprints (§487). */ +export const SEEN = "seen."; +/** The template state's cap (design 47 *The runner's kept data*): the fill is shown against it. */ +export const CAP_BYTES = 64 * 1024 * 1024; + +/** What a `seen` key holds: a commit's fingerprints at one module. */ +export interface Seen { + v: 1; + commit: string; + module: string; + fingerprints: string[]; + at: string; +} + +/** The step a head is refused for: the run, the module and the step the refusal names. */ +export interface Judged { + run: string; + module: string; + step: string; + fingerprint: string; +} + +const SHA = /^[0-9a-f]{40}$/; +/** A part of a key as the bus keeps keys: letters, digits and `-_=/`. */ +const safe = (s: string): string => s.replace(/[^A-Za-z0-9_=/-]/g, "_"); +export const seenKey = (commit: string, module: string): string => `${SEEN}${commit}.${safe(module)}`; +export const failedStepKey = (module: string, fingerprint: string, step: string): string => failedKey(safe(module), safe(fingerprint), safe(step)); + +export class FailedSteps { + /** Every act runs after the one before it, so a step-ended and a head's loss never interleave their reads and writes. */ + private chain: Promise = Promise.resolve(); + + constructor(private readonly st: State, private readonly log: (line: string) => void = () => {}, + /** Whether a commit is now a watched branch's head or an open pull request's head. */ + private readonly current: (commit: string) => boolean = () => false) {} + + private turn(act: () => Promise): Promise { + const r = this.chain.then(act, act); + this.chain = r.catch(() => {}); + return r; + } + + private async all(): Promise<{ failed: Map; seen: Map }> { + const failed = new Map(), seen = new Map(); + for (const key of await this.st.keys()) { + const e = await this.st.get(key); + if (!e?.value) continue; + if (key.startsWith(SEEN)) seen.set(key, e.value as Seen); + else failed.set(key, e.value as FailedStep); + } + return { failed, seen }; + } + + /** + * One `step-ended` (§39, §475, §487, §548). A `step-ended` that carries no pipeline's commit and fingerprint (an ask of another + * kind, §602) writes nothing. Its commit's fingerprint is kept; a failed `required` step is kept with every commit seen at its + * module and fingerprint; a later step-ended at a kept failure's module and fingerprint adds its commit; a passing `rerun` of + * a kept failure's step clears it, and nothing else does. + */ + stepEnded(ev: StepEndedEvent): Promise { + const r = ev?.result; + const fingerprint = r?.fingerprint || ev?.fp || ""; + if (!r || !SHA.test(String(ev.commit ?? "")) || !fingerprint || !ev.module) return Promise.resolve(); + return this.turn(async () => { + const at = ev.at ?? new Date().toISOString(); + const sk = seenKey(ev.commit, ev.module); + const held = (await this.st.get(sk))?.value as Seen | undefined; + if (!held?.fingerprints.includes(fingerprint)) { + await this.st.put(sk, { v: 1, commit: ev.commit, module: ev.module, fingerprints: [...(held?.fingerprints ?? []), fingerprint], at } satisfies Seen); + } else if (held.at !== at) { + await this.st.put(sk, { ...held, at }); + } + const key = failedStepKey(ev.module, fingerprint, r.step); + const all = await this.all(); + // A step-ended at a kept failure's module and fingerprint, of any outcome, adds its commit to every failure there (§475). + for (const [k, f] of all.failed) { + if (f.module === ev.module && f.fingerprint === fingerprint && !f.commits.includes(ev.commit)) { + const next = { ...f, commits: [...f.commits, ev.commit].sort() }; + all.failed.set(k, next); + await this.st.put(k, next); + } + } + const kept = all.failed.get(key); + if (r.outcome === "failed" && r.status === "required") { + if (kept) return; + const commits = new Set([ev.commit]); + for (const s of all.seen.values()) if (s.module === ev.module && s.fingerprints.includes(fingerprint)) commits.add(s.commit); + await this.st.put(key, { v: 1, module: ev.module, fingerprint, step: r.step, run: ev.pipeline, ...(r.artifact ? { artifact: r.artifact } : {}), + commits: [...commits].sort(), ...(r.why ? { said: r.why } : {}), at } satisfies FailedStep); + return; + } + // The runner's rerun is granted to the operator and the healers alone (its definition's callers), so a step-ended that says + // it is a rerun was asked by one of them; a plain pass, or any other outcome, clears nothing (§39). + if (r.rerun === true && r.outcome === "passed" && kept) { + await this.st.delete(key); + this.log(`${ev.module} ${r.step} at ${fingerprint} passed again by ${r.by ?? "a rerun"}: its failure is cleared`); + } + }); + } + + /** The failed `required` step that refuses this head, or null: a head with a `seen` key is judged by it, one with none merges (§487). */ + judge(head: string): Promise { + return this.turn(async () => { + const { failed, seen } = await this.all(); + for (const [, s] of [...seen].sort(([a], [b]) => (a < b ? -1 : 1))) { + if (s.commit !== head) continue; + for (const fingerprint of s.fingerprints) { + const hit = [...failed.values()].filter((f) => f.module === s.module && f.fingerprint === fingerprint).sort((a, b) => (a.step < b.step ? -1 : 1))[0]; + if (hit) return { run: hit.run, module: hit.module, step: hit.step, fingerprint }; + } + } + return null; + }); + } + + /** + * Commits that are no longer a watched branch's head nor an open pull request's head (§135): their `seen` keys go, they leave the + * failures' commits, and a failure none of whose commits is a head any longer is deleted. + */ + lost(commits: readonly string[]): Promise { + const gone = new Set(commits.filter((c) => !this.current(c))); + if (gone.size === 0) return Promise.resolve(); + return this.turn(async () => { + const { failed, seen } = await this.all(); + for (const [k, s] of seen) if (gone.has(s.commit)) await this.st.delete(k); + for (const [k, f] of failed) { + if (!f.commits.some((c) => gone.has(c))) continue; + const rest = f.commits.filter((c) => !gone.has(c)); + if (rest.some((c) => this.current(c))) await this.st.put(k, { ...f, commits: rest }); + else await this.st.delete(k); + } + }); + } + + /** What is held, for `git_failed_steps`: the failures, the `seen` keys (of one commit when asked), and the store's fill. */ + held(commit?: string): Promise<{ failed: FailedStep[]; seen: Seen[]; fill: { keys: number; bytes: number; cap: number } }> { + return this.turn(async () => { + const { failed, seen } = await this.all(); + const bytes = [...failed.values(), ...seen.values()].reduce((n, v) => n + JSON.stringify(v).length, 0); + return { failed: [...failed.values()].filter((f) => !commit || f.commits.includes(commit)).sort((a, b) => (a.run < b.run ? -1 : 1)), + seen: [...seen.values()].filter((s) => !commit || s.commit === commit), fill: { keys: failed.size + seen.size, bytes, cap: CAP_BYTES } }; + }); + } + + /** Drops one held key, a failure by its key or a `seen` key; false when it is not there. */ + drop(key: string): Promise { + return this.turn(async () => { + if (!(await this.st.get(key))) return false; + await this.st.delete(key); + return true; + }); + } +} + +/** + * The commits that are now a watched branch's head or the head an open pull request names (§39, §135). A watched branch is a + * branch a registration names for its origin; a head is read from the forge at the start and when a registration changes, and + * kept by `branch-moved` (its head read again from the forge, never from the event) and a pull request's updates and closes. + */ +export class Heads { + private watched = new Set(); + private branches = new Map(); + private pulls = new Map(); + + has(commit: string): boolean { + for (const h of this.branches.values()) if (h === commit) return true; + for (const h of this.pulls.values()) if (h === commit) return true; + return false; + } + + /** The heads the forge holds now of the watched branches (`@`) and the open pull requests (`#`): + * replaces both pictures and answers the commits that were heads and are not any more. */ + replace(watched: Set, branches: Map, pulls: Map): string[] { + const before = new Set([...this.branches.values(), ...this.pulls.values()]); + this.watched = watched; + this.branches = branches; + this.pulls = pulls; + return [...before].filter((c) => !this.has(c)); + } + + /** A watched branch's new head; the commit it replaces when that is no head any more. A branch no registration watches moves nothing. */ + branchMoved(repository: string, branch: string, head: string): string[] { + if (!this.watched.has(`${repository}@${branch}`)) return []; + const old = this.branches.get(`${repository}@${branch}`); + this.branches.set(`${repository}@${branch}`, head); + return old && old !== head && !this.has(old) ? [old] : []; + } + + pullUpdated(ev: PullUpdatedEvent): string[] { + const k = `${ev.repository}#${ev.number}`, old = this.pulls.get(k); + this.pulls.set(k, ev.head); + return old && old !== ev.head && !this.has(old) ? [old] : []; + } + + pullEnded(ev: PullClosedEvent | PullMergedEvent): string[] { + const k = `${ev.repository}#${ev.number}`, old = this.pulls.get(k); + this.pulls.delete(k); + return old && !this.has(old) ? [old] : []; + } + + watches(repository: string, branch: string): boolean { + return this.watched.has(`${repository}@${branch}`); + } +} + +/** What the forge holds now: the heads of the watched branches (`@`) and of the open pull requests (`#`). */ +export interface HeadsSource { + branchHead(owner: string, repo: string, branch: string): Promise; + openPulls(): Promise<{ owner: string; repo: string; number: number; head_sha: string }[]>; +} + +/** Reads the heads the picture is made of: each watched branch's head (one a forge cannot name is said and left out) and every open pull request's. */ +export async function readHeads(forge: HeadsSource, watched: ReadonlyMap>, log: (line: string) => void = () => {}) { + const names = new Set(), branches = new Map(), pulls = new Map(); + for (const [repository, list] of watched) { + const [owner, repo] = repository.split("/"); + for (const branch of list) { + names.add(`${repository}@${branch}`); + try { + branches.set(`${repository}@${branch}`, await forge.branchHead(owner, repo, branch)); + } catch (err) { + log(`the head of ${repository}'s watched branch ${branch} could not be read: ${err instanceof Error ? err.message : String(err)}`); + } + } + } + for (const p of await forge.openPulls()) pulls.set(`${p.owner}/${p.repo}#${p.number}`, p.head_sha); + return { names, branches, pulls }; +} diff --git a/modules/gitea/index.ts b/modules/gitea/index.ts index d5ff40722..0f61601bd 100644 --- a/modules/gitea/index.ts +++ b/modules/gitea/index.ts @@ -48,6 +48,10 @@ import { announce, keepWatermark, readWatermark } from "./announce.js"; import { SEAT_ONLY, sayOnSeat } from "./seatevents.js"; import { FORGE_CONTAINER, seatTools } from "./seat.js"; import { REGISTRATION_CHANGED_EVENT, Registrations } from "./registrations.js"; +import { FailedSteps, Heads, readHeads } from "./failedsteps.js"; +import { failedStepsTools } from "./tools/failedsteps.js"; +import { seat as GIT, events as GIT_EVENTS, type BranchMovedEvent, type PullClosedEvent, type PullMergedEvent, type PullUpdatedEvent } from "@novox/mesh-delivery-contracts-git"; +import { seat as RUNNER, events as RUNNER_EVENTS, type StepEndedEvent } from "@novox/mesh-delivery-contracts-mesh-pipeline-runner"; import { run } from "./delivery.js"; import { GiteaAdmin, GiteaClient, movedSince } from "./client.js"; import { keepMainRules, mainRulesStep } from "./mainrules.js"; @@ -393,13 +397,18 @@ const firstViews = state(STATUS_STATE); const followers: Follower[] = []; // The first registrations switch merge-pull on (registrations.ts; design 47 §95). const registrations = new Registrations((verb, input) => broker().request(verb, input), (line) => console.log(`[gitea] ${line}`)); +// The failed steps the runner's step-ended said, and the heads that keep them (failedsteps.ts; design 47 §39, §475, §487, §135). +const heldHeads = new Heads(); +let reloadHeads: () => Promise = async () => {}; +const failedSteps = new FailedSteps(state("failed-steps"), (line) => console.log(`[gitea] ${line}`), (commit) => heldHeads.has(commit)); // gitea_events is served whether or not the forge is watched: it says so (novox/hq issue 729). The seat git's verbs are // served here, beside the one writer of a head's statuses: the seat's status takes its turn on the head's chain, after the // announcement's pending and a trunk's move, so a recheck's pending never lands over a later verdict (issue 456). registerModuleTools("gitea", () => [eventsTool(pace), ...readAllTools(readAll), ...writeTools(writes), ...statusTools("gitea", "gitea_first_views", "gitea_first_view_again", followers, firstViews, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); }), - ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, statuses: gates, registered: () => registrations.written }) : [])]); + ...(gitea ? seatTools(gitea, { run, container: FORGE_CONTAINER, statuses: gates, registered: () => registrations.written, judge: (head) => failedSteps.judge(head) }) : []), + ...failedStepsTools(failedSteps, async (act) => { await broker().request("seat:mesh-hand-acts.hand-act", act); })]); /** What the forge's event asks of the announcer at once (issue 729, 1422): see watch.ts. */ const actOnEvent = (client: GiteaClient, event: ForgeEvent): Promise => actOn(event, { @@ -538,11 +547,36 @@ if (gitea) { void view.follower.start().then(() => w.viewed()); void bindUntilHeld(REGISTRATION_CHANGED_EVENT, () => on(REGISTRATION_CHANGED_EVENT, async (event) => { try { registrations.hear(event.body); } catch (err) { console.error(`[gitea] ${err instanceof Error ? err.message : err}`); } + // The registrations' watched branches may have changed: asked again, and the heads read again. + await registrations.start(); + await reloadHeads().catch((err) => console.error(`[gitea] the watched heads could not be read — ${err instanceof Error ? err.message : err}`)); })); - void registrations.start(); } catch (err) { console.error(`[gitea] the merging account is not given write on the delivered repositories: no site administrator's login — ${err instanceof Error ? err.message : err}`); } + // The runner's step-ended and the heads' moves, heard through durable consumers of the module's own (design 47 §39, §475, §135): a failed + // `required` step is kept, and what no watched branch's head and no open pull request's head names any more is let go. + if (gitea) { + const forge = gitea; + const say = (err: unknown) => console.error(`[gitea] ${err instanceof Error ? err.message : String(err)}`); + reloadHeads = async () => { + const read = await readHeads(forge, registrations.watched, (line) => console.log(`[gitea] ${line}`)); + await failedSteps.lost(heldHeads.replace(read.names, read.branches, read.pulls)); + }; + void bindUntilHeld(`${RUNNER}.${RUNNER_EVENTS.stepEnded}`, () => on(`${RUNNER}.${RUNNER_EVENTS.stepEnded}`, (event) => failedSteps.stepEnded(event.body))); + // A branch's head is read from the forge, never taken from the event (the seat's definition of branch-moved). + void bindUntilHeld(`${GIT}.${GIT_EVENTS.branchMoved}`, () => on(`${GIT}.${GIT_EVENTS.branchMoved}`, async (event) => { + const { repository, branch } = event.body; + if (!heldHeads.watches(repository, branch)) return; + const [owner, repo] = repository.split("/"); + await failedSteps.lost(heldHeads.branchMoved(repository, branch, await forge.branchHead(owner, repo, branch))); + })); + void bindUntilHeld(`${GIT}.${GIT_EVENTS.pullUpdated}`, () => on(`${GIT}.${GIT_EVENTS.pullUpdated}`, (event) => failedSteps.lost(heldHeads.pullUpdated(event.body)))); + for (const name of [GIT_EVENTS.pullClosed, GIT_EVENTS.pullMerged]) { + void bindUntilHeld(`${GIT}.${name}`, () => on(`${GIT}.${name}`, (event) => failedSteps.lost(heldHeads.pullEnded(event.body)))); + } + void registrations.start().then(() => reloadHeads()).catch(say); + } // Main takes no push and only the tools' account merges, in every repository, kept and not only when a token is // minted: at the start and every ten minutes after, so a repository made later gets it too (issue 948, ADR 0398 §2). // Only while the setting keep-main-rules is true (mainrules.ts); false, the default, leaves every repository's rule as set. diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 02c8fcd63..7cabb3024 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -29,6 +29,14 @@ "options": { "key-ttl": true } + }, + { + "name": "failed-steps", + "storage": "key-value", + "template": "state", + "options": { + "history": 1 + } } ], "contributes": { @@ -647,6 +655,11 @@ ], "consumes": [ "mesh-catalog.catalog-revision", - "mesh-catalog.registration-changed" + "mesh-catalog.registration-changed", + "mesh-pipeline-runner.step-ended", + "git.branch-moved", + "git.pull-updated", + "git.pull-closed", + "git.pull-merged" ] } diff --git a/modules/gitea/registrations.ts b/modules/gitea/registrations.ts index 82a46d634..3d8f5828d 100644 --- a/modules/gitea/registrations.ts +++ b/modules/gitea/registrations.ts @@ -17,6 +17,7 @@ export const REGISTRATION_CHANGED_EVENT = `${CATALOG}.${CATALOG_EVENTS.registrat export class Registrations { private first = false; + private byRepository = new Map>(); constructor(private readonly ask: (verb: string, input: RegistrationsInput) => Promise, private readonly log: (line: string) => void = () => {}) {} @@ -25,6 +26,11 @@ export class Registrations { return this.first; } + /** The watched branches the registrations name, by their origin repository (`/`) as of the last read. */ + get watched(): ReadonlyMap> { + return this.byRepository; + } + /** One `registration-changed`: the first registration written, however many were before this module started. */ hear(ev: RegistrationChangedEvent | undefined): void { if (!ev || typeof ev.module !== "string" || ev.module === "") { @@ -34,13 +40,17 @@ export class Registrations { this.first = true; } - /** The start's first view: any registration held is the first written. A failed ask is said and left to the event. */ + /** The start's first view, asked again after each `registration-changed`: any registration held is the first written, and its + * watched branches are read. A failed ask is said and left to the event. */ async start(): Promise { try { let a: unknown = await this.ask(REGISTRATIONS_VERB, {}); if (typeof a === "string") a = JSON.parse(a); const r = (a as RegistrationsAnswer | undefined)?.registrations; if (!Array.isArray(r)) throw new Error(`${REGISTRATIONS_VERB} answered no registrations: ${JSON.stringify(a ?? null).slice(0, 200)}`); + const by = new Map>(); + for (const x of r) for (const b of x.branches ?? []) (by.get(x.origin) ?? by.set(x.origin, new Set()).get(x.origin)!).add(b); + this.byRepository = by; if (r.length > 0) { this.first = true; this.log(`${r.length} registrations are held; merge-pull merges`); diff --git a/modules/gitea/seat.ts b/modules/gitea/seat.ts index 8ba0d9725..8cd08bd95 100644 --- a/modules/gitea/seat.ts +++ b/modules/gitea/seat.ts @@ -10,8 +10,8 @@ // only when the forge said so. // // **The acts carry the caller's id** (status, comment, view, note, merge-pull): an id the pull request or the note already -// holds is answered `held` and nothing is done again; `merge-pull` merges the head the caller names and judges no check -// (design 47 §314): only into a base whose protection refuses a push, and only that head. +// holds is answered `held` and nothing is done again; `merge-pull` merges the head the caller names, only into a base +// whose protection refuses a push, and refuses a head at whose fingerprint a run failed a `required` step (design 47 §39, §314). import type { ToolDefinition } from "@novox/mesh-sdk/tools"; import { @@ -25,6 +25,7 @@ import { MAX_FILE_BYTES } from "./client.js"; import { containedFrom, newestByContext } from "./checks.js"; import { Refused, checkName } from "./guard.js"; import { appendNote, viewBody, viewComment, type Runner } from "./delivery.js"; +import type { Judged } from "./failedsteps.js"; /** The verbs of the seat, in its definition's order. */ export const VERBS: readonly string[] = Object.values(GIT_VERBS); @@ -50,6 +51,8 @@ export interface SeatDeps { statuses: HeadStatuses; /** Whether the first registrations were written (registrations.ts): until then merge-pull refuses every merge (design 47 §95). */ registered: () => boolean; + /** The failed `required` step that refuses a head, judged from the module's own state `failed-steps` (failedsteps.ts), or null. */ + judge: (head: string) => Promise; } /** A failure as the contract says it: the forge's or this holder's refusal, or a forge that failed. */ @@ -252,8 +255,8 @@ export function seatTools(gitea: GiteaClient, deps: SeatDeps): ToolDefinition[] const said = await appendNote(deps.run, deps.container, owner, repo, commit, "mesh-plan", `${String(args.line ?? "")} [${id}]`, id); return said.added ? { id, performed: true } : { id, held: true, performed: true }; }), - verb(GIT_VERBS.mergePull, "The seat git's merge-pull: an act of the forge bridge, a pull request merged at the head the caller names; it judges no check.", - async (args): Promise => mergePull(gitea, args, deps.registered)), + verb(GIT_VERBS.mergePull, "The seat git's merge-pull: an act of the forge bridge, a pull request merged at the head the caller names; it refuses a head at whose fingerprint a required step failed.", + async (args): Promise => mergePull(gitea, args, deps.registered, deps.judge)), verb(GIT_VERBS.tag, "The seat git's tag: the git tag go/vMAJOR.MINOR.PATCH of a released version, written at the commit its build was made from; a tag that is at that commit is performed.", async (args): Promise => { const { owner, repo } = repositoryOf(args.repository); @@ -279,7 +282,8 @@ function holds(comments: GiteaComment[], id: string): boolean { } /** merge-pull (design 47 §314): only the head the caller names, only into a base that takes no push, no check judged. */ -export async function mergePull(gitea: GiteaClient, args: Record, registered: () => boolean): Promise { +export async function mergePull(gitea: GiteaClient, args: Record, registered: () => boolean, + judge: (head: string) => Promise): Promise { const id = idOf(args.id); const { owner, repo, full } = repositoryOf(args.repository); const number = numberOf(args.number); @@ -303,6 +307,13 @@ export async function mergePull(gitea: GiteaClient, args: Record(); + return { kv, st: { + get: async (k: string) => (kv.has(k) ? { key: k, value: kv.get(k), revision: 1 } : null), + put: async (k: string, v: unknown) => { kv.set(k, structuredClone(v)); return 1; }, + delete: async (k: string) => { kv.delete(k); }, + keys: async () => [...kv.keys()].sort(), + watch: async () => {}, + } }; +} +const ended = (commit: string, outcome: string, extra: Record = {}, over: Record = {}) => ({ + v: 1, pipeline: `main.7.nats-${commit.slice(0, 2)}`, module: "nats", branch: "main", commit, at: "2026-11-11T12:00:00Z", + result: { step: "unit", status: "required", outcome, fingerprint: "fp-1", ...extra }, ...over }) as any; + +test("a head is refused by a required step that failed at its fingerprint in any run, until a rerun passes it", async () => { + const { kv, st } = memory(); + const heads = new Heads(); + const f = new FailedSteps(st as any, () => {}, (c) => heads.has(c)); + const notes: [string, string | null][] = []; + const judged = async (name: string, commit: string, want: string | null) => { + const got = await f.judge(commit); + assert.equal(got ? `${got.run} ${got.module} ${got.step}` : null, want, name); + }; + await f.stepEnded(ended(C1, "passed")); + await judged("a head that passed alone merges", C1, null); + await judged("a head no run named merges", C3, null); + await f.stepEnded(ended(C2, "failed", {}, { pipeline: "run-on-request" })); + await judged("the head that failed", C2, "run-on-request nats unit"); + await judged("an earlier commit of the same content (passed before, failed after)", C1, "run-on-request nats unit"); + assert.deepEqual((kv.get(failedStepKey("nats", "fp-1", "unit")) as any).commits, [C1, C2], "the commits it was seen at"); + // A step-ended of an ask that is no pipeline's step (no commit, no fingerprint) writes nothing (§548). + const before = kv.size; + await f.stepEnded({ v: 1, asker: "mesh-build-go", ask: "a-1", node: "ace", result: { step: "unit", status: "required", outcome: "failed" } } as any); + assert.equal(kv.size, before, "a builder's failed build is no run of a commit"); + // A later commit at the fingerprint is refused too, and a failed optional step refuses nothing. + await f.stepEnded(ended(C3, "passed")); + await judged("a later commit of the same content", C3, "run-on-request nats unit"); + await f.stepEnded(ended(C3, "failed", { step: "lint", status: "optional", fingerprint: "fp-2" })); + await judged("a failed optional step", C3, "run-on-request nats unit"); + assert.equal(kv.has(failedStepKey("nats", "fp-2", "lint")), false); + // A plain pass clears nothing; a rerun that passes clears it. + await f.stepEnded(ended(C2, "passed")); + await judged("a plain pass", C2, "run-on-request nats unit"); + await f.stepEnded(ended(C2, "passed", { rerun: true, by: "operator" })); + await judged("a passing rerun", C2, null); + await judged("a passing rerun, for the earlier commit too", C1, null); + void notes; +}); + +test("an entry goes once none of its commits is a watched branch's head or an open pull request's head", async () => { + const { kv, st } = memory(); + const heads = new Heads(); + const f = new FailedSteps(st as any, () => {}, (c) => heads.has(c)); + heads.replace(new Set(["novox/app@main"]), new Map([["novox/app@main", C1]]), new Map([["novox/app#4", C2]])); + await f.stepEnded(ended(C2, "failed")); + await f.stepEnded(ended(C1, "passed")); + assert.deepEqual([...kv.keys()].sort(), [failedStepKey("nats", "fp-1", "unit"), seenKey(C1, "nats"), seenKey(C2, "nats")].sort()); + // The pull request is closed: its commit is no head, its seen key goes; the entry stays for the trunk's head that shares it. + await f.lost(heads.pullEnded({ repository: "novox/app", number: 4 } as any)); + assert.equal(kv.has(seenKey(C2, "nats")), false); + assert.deepEqual((kv.get(failedStepKey("nats", "fp-1", "unit")) as any).commits, [C1]); + // The branch moves on: nothing names the entry's commits, so it goes. A branch no registration watches moves nothing. + assert.deepEqual(heads.branchMoved("novox/app", "feature", C3), []); + await f.lost(heads.branchMoved("novox/app", "main", C3)); + assert.deepEqual([...kv.keys()], [], "the failure and the seen keys are gone"); +}); + +test("git_failed_steps shows what is held and its fill, and git_failed_steps_drop writes the hand act before it drops", async () => { + const { failedStepsTools } = await import("../tools/failedsteps.js"); + const { runAs } = await import("@novox/mesh-sdk/callers"); + const { st } = memory(); + const f = new FailedSteps(st as any); + await f.stepEnded(ended(C2, "failed")); + const acts: any[] = []; + let recorded = true; + const [read, drop] = failedStepsTools(f, async (a) => { if (!recorded) throw new Error("no hand-act log"); acts.push(a); }); + const shown: any = await read.run({}); + assert.deepEqual([shown.failed.map((x: any) => [x.key, x.run]), shown.seen.length, shown.fill.keys, shown.fill.cap], + [[["nats.fp-1.unit", "main.7.nats-22"]], 1, 2, 64 * 1024 * 1024]); + assert.equal(((await read.run({ commit: C1 })) as any).failed.length, 0, "a commit asked for filters"); + recorded = false; + await assert.rejects(runAs({ "mesh/caller": "operator" }, {}, () => drop.run({ key: "nats.fp-1.unit", why: "stale" })), /hand act was not recorded, so nothing was dropped/); + assert.equal((await f.judge(C2))?.step, "unit", "nothing dropped without the hand act"); + recorded = true; + const done: any = await runAs({ "mesh/caller": "operator" }, {}, () => drop.run({ key: "nats.fp-1.unit", why: "stale" })); + assert.deepEqual([done.dropped, acts.length, acts[0].verb], [true, 1, "git_failed_steps_drop"]); + assert.equal(await f.judge(C2), null); +}); diff --git a/modules/gitea/test/guard.test.ts b/modules/gitea/test/guard.test.ts index d939bcfb9..de835a654 100644 --- a/modules/gitea/test/guard.test.ts +++ b/modules/gitea/test/guard.test.ts @@ -434,11 +434,12 @@ test("the two new tools are named for what mesh_search is asked, and gitea_api s // **merge-pull judges the head it is asked at and no check** (design 47 *The seat git*, §314): merge-pull merges the // head it is named into a base that takes no push, whatever its statuses say, and an id said again after the merge is // answered performed. -test("merge-pull merges the head the caller names and judges no check", async () => { +test("merge-pull merges the head the caller names, and refuses one at whose fingerprint a required step failed", async () => { const { seatTools } = await import("../dist/seat.js"); const { runAs } = await import("@novox/mesh-sdk/callers"); const bridge = "ace.mesh-delivery-forge-bridge"; let registered = true; + let failedStep: { run: string; module: string; step: string; fingerprint: string } | null = null; // The head's one writer of its statuses (gate.ts), over a forge whose first write is still in flight when the next is asked. const { GateWriter } = await import("../dist/gate.js"); const landed: string[] = []; @@ -452,7 +453,7 @@ test("merge-pull merges the head the caller names and judges no check", async () }; const gates = new GateWriter(slow); const seat = seatTools(Object.assign(Object.create(client), { setCommitStatus: slow.setCommitStatus }), - { run: async () => ({ stdout: "", code: 0 }), container: "gitea", statuses: gates, registered: () => registered }); + { run: async () => ({ stdout: "", code: 0 }), container: "gitea", statuses: gates, registered: () => registered, judge: async () => failedStep }); const call = (verb: string, args: Record, caller = bridge) => runAs({ "mesh/caller": caller }, args, () => seat.find((t: any) => t.name === `git.${verb}`)!.run(args)) as Promise; const merge = (extra: Record = {}, caller?: string) => @@ -463,6 +464,8 @@ test("merge-pull merges the head the caller names and judges no check", async () { name: "a base no rule guards", set: () => { state.rules = []; }, want: /guarded by no protection rule/, kind: "unprotected" }, { name: "a base whose protection was removed after the last branch-moved", set: () => { state.rules = [protectedMain()]; state.rules = []; }, want: /guarded by no protection rule/, kind: "unprotected" }, { name: "a base that takes a push", set: () => { state.rules = [{ rule_name: "main", enable_push: true }]; }, want: /guarded by no protection rule/, kind: "unprotected" }, + { name: "a required step failed at the head's fingerprint", set: () => { failedStep = { run: "main.7.nats", module: "nats", step: "unit", fingerprint: "fp-1" }; }, + want: /the unit step of nats failed in run main\.7\.nats/, kind: "policy-failed" }, { name: "a failed check", set: () => { state.statuses = [{ context: "mesh/check", state: "failure", description: "fail" }]; }, want: "merged" }, { name: "no status at all", set: () => { state.statuses = []; }, want: "merged" }, { name: "a check of another trunk", set: () => { state.statuses = judgedAt(MOVED); }, want: "merged" }, @@ -474,7 +477,7 @@ test("merge-pull merges the head the caller names and judges no check", async () { name: "the forge merged another head", set: () => { state.landed = MOVED; }, want: /WAS merged .*the forge merged 3c4d5e6f7a8b, not the abc1234def56/, loud: true }, ]; for (const r of rows) { - registered = true; state.merged = false; state.head = REVIEWED; state.statuses = judgedAt(TRUNK); state.sent = []; + registered = true; failedStep = null; state.merged = false; state.head = REVIEWED; state.statuses = judgedAt(TRUNK); state.sent = []; state.rules = [protectedMain()]; state.prState = "open"; state.parents = null; state.commitsFail = 0; state.landed = ""; r.set(); const got = await merge(r.extra, r.caller); @@ -488,6 +491,7 @@ test("merge-pull merges the head the caller names and judges no check", async () } else { assert.match(String(got.refused ?? got.error), r.want, r.name); if (r.kind) assert.equal(got["refused-kind"], r.kind, r.name); + if (r.kind === "policy-failed") assert.deepEqual([got.run, got.module, got.step], ["main.7.nats", "nats", "unit"], r.name); assert.deepEqual(writes(), r.loud ? ["POST /repos/novox/mesh-catalog/pulls/7/merge"] : [], `${r.name}: what was merged`); } } @@ -548,10 +552,10 @@ test("every read and act of the seat git answers its contract, and an error is a }; const run = async (_f: string, args: string[]) => args.includes("show") || args.includes("--ref") && !args.includes("append") ? { stdout: noteShown, code: noteShown ? 0 : 1 } : (noteWritten++, { stdout: "", code: 0 }); - const seat = seatTools(forge as any, { run: run as any, container: "gitea", statuses: { status: async () => {} }, registered: () => true }); + const seat = seatTools(forge as any, { run: run as any, container: "gitea", statuses: { status: async () => {} }, registered: () => true, judge: async () => null }); const call = (verb: string, args: Record, over: Record = {}) => { const t = seat.find((x: any) => x.name === `git.${verb}`)!; - const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", statuses: { status: async () => {} }, registered: () => true }); + const wrapped = seatTools(Object.assign(Object.create(forge), over) as any, { run: run as any, container: "gitea", statuses: { status: async () => {} }, registered: () => true, judge: async () => null }); return runAs({ "mesh/caller": "ace.x" }, args, () => wrapped.find((x: any) => x.name === t.name)!.run(args)) as Promise; }; const rows: [string, string, Record, Record, (a: any) => unknown, unknown][] = [ diff --git a/modules/gitea/tools/failedsteps.ts b/modules/gitea/tools/failedsteps.ts new file mode 100644 index 000000000..430aaaa61 --- /dev/null +++ b/modules/gitea/tools/failedsteps.ts @@ -0,0 +1,44 @@ +// The tools of the forge module's state `failed-steps` (novox/hq design 47 §83, §84): the read of everything it holds, with the +// store's fill against its cap, and the one recovery, dropping a held key, written in the hand-act log before it is done. + +import type { ToolDefinition } from "@novox/mesh-sdk/tools"; +import { callerOf } from "@novox/mesh-sdk/callers"; +import type { HandAct } from "@novox/mesh-sdk/views"; +import type { FailedSteps } from "../failedsteps.js"; + +export function failedStepsTools(kept: FailedSteps, acts: (act: HandAct) => Promise): ToolDefinition[] { + return [ + { + name: "git_failed_steps", + description: "The failed required steps the forge module holds, by module and fingerprint, each with the run that failed it, the commits it was seen at and why; " + + "the `seen` keys (each commit's fingerprints, a commit's with `commit`); and the store's fill against its cap (design 47 §83, §487).", + input: { type: "object", properties: { commit: { type: "string", description: "only what names this commit" } } }, + run: async (args: Record) => { + const commit = typeof args.commit === "string" && args.commit !== "" ? args.commit : undefined; + const held = await kept.held(commit); + return { failed: held.failed.map((f) => ({ ...f, key: `${f.module}.${f.fingerprint}.${f.step}` })), seen: held.seen, fill: held.fill }; + }, + }, + { + name: "git_failed_steps_drop", + description: "Drops one held key of the state failed-steps, a failed step by its key (..) or a seen key (seen..), with why; " + + "the head it refused merges. Written in the hand-act log first (design 47 §84).", + input: { + type: "object", required: ["key", "why"], + properties: { key: { type: "string", description: "the key, as git_failed_steps shows it" }, why: { type: "string", description: "why it is dropped, in plain words" } }, + }, + run: async (args: Record) => { + const key = typeof args.key === "string" ? args.key.trim() : ""; + const why = typeof args.why === "string" ? args.why.trim() : ""; + if (!key || !why) throw new Error("git_failed_steps_drop names the key and why"); + const by = callerOf(args); + try { + await acts({ what: `dropped the held ${key} of the forge module's failed-steps`, why: `${why} (${by})`, cause: "failed-step-dropped", verb: "git_failed_steps_drop" }); + } catch (err) { + throw new Error(`the hand act was not recorded, so nothing was dropped: ${err instanceof Error ? err.message : String(err)}`); + } + return { dropped: await kept.drop(key), key, why, by }; + }, + }, + ]; +} -- 2.54.0 From 86342c4399a4a1235653ce67f5f07b7f680d8f96 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:04:56 +0100 Subject: [PATCH 031/243] GIT12: the forge module's package registry checks a publish against the builder's reservation, asked by the package's name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The holder asks the seat mesh-builds' verb reserved with the publish's name and version in place of builder_versions, and stores only a version reserved for an artifact of the publish's digest; the manifest invokes the verb and takes no builder package. Builds: design 47 G §381, §478, §504 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/package-registry/retention_test.go | 10 ++-- modules/gitea/cmd/package-registry/seats.go | 5 +- .../whole_integration_test.go | 17 +++--- modules/gitea/go.mod | 2 +- modules/gitea/internal/packageseats/holder.go | 8 +-- .../internal/packageseats/holder_test.go | 2 +- .../gitea/internal/packageseats/reserved.go | 42 +++++++++++++++ .../internal/packageseats/reserved_test.go | 54 +++++++++++++++++++ modules/gitea/module.json | 5 +- 9 files changed, 120 insertions(+), 25 deletions(-) create mode 100644 modules/gitea/internal/packageseats/reserved.go create mode 100644 modules/gitea/internal/packageseats/reserved_test.go diff --git a/modules/gitea/cmd/package-registry/retention_test.go b/modules/gitea/cmd/package-registry/retention_test.go index 10116919d..6fd6081aa 100644 --- a/modules/gitea/cmd/package-registry/retention_test.go +++ b/modules/gitea/cmd/package-registry/retention_test.go @@ -450,12 +450,12 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - // The builder's question is its published package, taken by its series (ADR 0634 §1). - if m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"] != "0.1" { - t.Errorf("packages %v: want builds of series 0.1", m.Packages) + // The reservation is asked through the seat mesh-builds, so the manifest takes no package of the builder (design 47 §381). + if len(m.Packages) != 0 { + t.Errorf("packages %v: want none", m.Packages) } // Both package seats are claimed serving the four verbs this bundle answers as ., over its own incoming - // and transfers, asking the builder's numbering (design 51 *Publishing*). + // and transfers, asking the builder's reservation (design 51 *Publishing*, design 47 §381). claimed := map[string]string{} for _, c := range m.Claims { claimed[c.Name] = strings.Join(c.Serves, ",") @@ -488,7 +488,7 @@ func TestToolsAreTheOnesTheReadmeNamesAndSayWhatTheyDo(t *testing.T) { t.Errorf("transfers has options (a lifetime): %s", req) } // The retention asks the builder's versions; the forge's other bundles ask the catalog half and record a re-read. - if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-catalog.registrations,seat:mesh-hand-acts.hand-act,mesh-delivery-builder.builder_versions" { + if strings.Join(m.Invokes, ",") != "seat:mesh-catalog.revision,seat:mesh-catalog.registrations,seat:mesh-hand-acts.hand-act,seat:mesh-builds.reserved" { t.Errorf("the manifest invokes %v", m.Invokes) } if len(m.Tools) != 0 { diff --git a/modules/gitea/cmd/package-registry/seats.go b/modules/gitea/cmd/package-registry/seats.go index 5b409e603..4ec5770e8 100644 --- a/modules/gitea/cmd/package-registry/seats.go +++ b/modules/gitea/cmd/package-registry/seats.go @@ -19,7 +19,6 @@ import ( "encoding/json" "errors" "fmt" - builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "io" "net/http" "net/url" @@ -274,11 +273,11 @@ func recordAct(q meshhandactsclient.Request) error { // boundCap is incoming's cap from its binding, read once per bundle. var boundCap = parts.BusCap(packageseats.IncomingObjects) -// busHolder is the holder over the runtime's bus state, the builder's numbering and the forge. +// busHolder is the holder over the runtime's bus state, the builder's reservation and the forge. func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) { return &packageseats.Holder{Objects: parts.BusObjects(packageseats.IncomingObjects), Transfers: parts.BusTransfers[packageseats.Opening](packageseats.TransfersState), - Builder: builds.BuilderVersions{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding, + Builder: packageseats.Reserved{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding, Cap: boundCap, Record: recordAct, Idle: parts.BusArm(packageseats.IdleState)}, nil } diff --git a/modules/gitea/cmd/package-registry/whole_integration_test.go b/modules/gitea/cmd/package-registry/whole_integration_test.go index 6c766e8af..5a52ce7a4 100644 --- a/modules/gitea/cmd/package-registry/whole_integration_test.go +++ b/modules/gitea/cmd/package-registry/whole_integration_test.go @@ -6,7 +6,7 @@ package main // the built package-registry bundle, played over its real stdin and stdout as the node's tool runtime plays it, its // `incoming` an object store and its `transfers` and `findings` key-value buckets of a nats-server of this test alone // (so the publish in parts runs through mesh-sdk's parts.BusObjects and BusTransfers, ruling 4 of the lead's 14:0x), the -// builder's builder_versions answered on mesh/ask, and the forge an +// builder's reserved answered on mesh/ask, and the forge an // httptest server that takes reads only with the tools' token and writes only with the packages account's. // // A Go package the builder numbered goes up in two chunks, is uploaded whole as the packages account and says @@ -37,7 +37,6 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "git.novox.be/novox/mesh-sdk/go/testbus" "gitea/internal/packageseats" @@ -117,11 +116,15 @@ func (p *player) handle(m wire) { } _ = json.Unmarshal(m.Params, &a) switch { - case m.Method == "mesh/ask" && a.Key == builds.BuilderVersionsAddress: - var q struct{ Name, Version, Digest string } + case m.Method == "mesh/ask" && a.Key == packageseats.Address: + var q struct{ Name, Version string } _ = json.Unmarshal(a.Body, &q) d, ok := p.numbered[q.Name+"@"+q.Version] - p.answer(m.ID, map[string]any{"recorded": ok && d == q.Digest, "why": "not numbered with that digest"}, nil) + if !ok { + p.answer(m.ID, nil, errors.New("no version of "+q.Name+" is reserved")) + break + } + p.answer(m.ID, map[string]any{"module": "m", "version": q.Version, "artifacts": []map[string]any{{"artifact": "m:a", "digest": d}}}, nil) case m.Method == "mesh/seat.publish": p.answer(m.ID, map[string]any{}, p.nc.Publish(a.Subject, a.Body)) case m.Method == "mesh/publish": @@ -395,12 +398,12 @@ func TestThePackageSeatsWhole(t *testing.T) { t.Errorf("an ended transfer is still kept: %v", keys) } - // One the builder did not number: refused before the forge sees it, said, and the seat's finding written. + // One the builder did not reserve: refused before the forge sees it, said, and the seat's finding written. other := bigZip(t, "git.example/acme/sdk@v0.1.16") od := "sha256:" + sha(other) transfer, _ = publish("0.1.16", od, len(other)) _ = send(transfer, od, other) - if o := event("refused"); !strings.Contains(fmt.Sprint(o["reason"]), "did not number") { + if o := event("refused"); !strings.Contains(fmt.Sprint(o["reason"]), "no version of git.example/acme/sdk is reserved") { t.Errorf("refused says %v", o) } findings, _ := js.KeyValue(ctx, "gitea_findings") diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index 62607444b..d84ba67cf 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -8,8 +8,8 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats-server/v2 v2.11.17 diff --git a/modules/gitea/internal/packageseats/holder.go b/modules/gitea/internal/packageseats/holder.go index 7b03101fb..10f05af16 100644 --- a/modules/gitea/internal/packageseats/holder.go +++ b/modules/gitea/internal/packageseats/holder.go @@ -108,7 +108,7 @@ type Version struct { Published time.Time } -// Builder answers whether the builder numbered a version with a digest (the builder package's builds.BuilderVersions). +// Builder answers whether the builder reserved a version with a digest (Reserved). type Builder interface { Recorded(name, version, digest string) (ok bool, why string, err error) } @@ -165,14 +165,14 @@ func key(seat, transfer string) string { return seat + "_" + transfer } func (h *Holder) receiver() *parts.Receiver[Opening] { return &parts.Receiver[Opening]{Objects: h.Objects, Transfers: h.Transfers, Cap: IncomingCap, Bound: h.Cap, MaxChunk: MaxChunk, Idle: TransferIdle, Arm: h.Idle, Full: IncomingFullPct, Now: h.now, - // The builder's numbering is asked before anything is uploaded (ADR 0605 §4). + // The builder's reservation is asked before anything is uploaded (ADR 0605 §4, design 47 §381). Check: func(t Transfer) error { ok, why, err := h.Builder.Recorded(t.Shape.Name, t.Shape.Version, t.Shape.Digest) if err != nil { - return errors.New("builder_versions did not answer, so nothing is stored: " + err.Error()) + return errors.New("the reservation could not be read, so nothing is stored: " + err.Error()) } if !ok { - return errors.New("the builder did not number it: " + why) + return errors.New("the builder did not reserve it: " + why) } return nil }, diff --git a/modules/gitea/internal/packageseats/holder_test.go b/modules/gitea/internal/packageseats/holder_test.go index 63ebf4de8..0495e041a 100644 --- a/modules/gitea/internal/packageseats/holder_test.go +++ b/modules/gitea/internal/packageseats/holder_test.go @@ -252,7 +252,7 @@ func TestTheForgeHoldsBothPackageSeats(t *testing.T) { numbered bool data []byte want string - }{{"the builder did not number it", false, file, "did not number"}} { + }{{"the builder did not reserve it", false, file, "did not reserve"}} { b := newBed(c.numbered) if _, err := b.h.Publish(NpmSeat, nodeBuild, pub("x", "0.1.0", file)); err != nil { t.Fatal(err) diff --git a/modules/gitea/internal/packageseats/reserved.go b/modules/gitea/internal/packageseats/reserved.go new file mode 100644 index 000000000..ef5228a84 --- /dev/null +++ b/modules/gitea/internal/packageseats/reserved.go @@ -0,0 +1,42 @@ +package packageseats + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" +) + +// Reserved is the holder's check of a publish against the builder's reservation (novox/hq design 47 §381, §478, §504): it asks the +// seat mesh-builds' verb reserved with the publish's name and version, a package's name in its ecosystem, and accepts the publish +// only when that version is the one reserved and an artifact of it has the publish's digest. It reads no catalog and a publish gains +// no field. It is the Builder the holder's Check asks. +type Reserved struct { + // Ask is the runtime's request and reply over the bus (mesh-sdk stdio.Ask). + Ask func(address string, body any) (json.RawMessage, error) +} + +// Address is the seat verb asked, as the manifest's `invokes` names it. +const Address = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbReserved + +// Recorded answers whether name@version with digest is reserved, and why not when it is not. An answer that cannot be read is an +// error, never a yes; a refusal by the builder (a name no entry carries, a version it did not reserve) is a no that names it. +func (r Reserved) Recorded(name, version, digest string) (bool, string, error) { + raw, err := r.Ask(Address, meshbuilds.ReservedInput{Name: name, Version: version}) + if err != nil { + return false, "", fmt.Errorf("%s did not answer for %s@%s: %w", Address, name, version, err) + } + var a meshbuilds.ReservedAnswer + if err := json.Unmarshal(raw, &a); err != nil || a.Version == "" { + return false, "", fmt.Errorf("%s answered no reservation for %s@%s: %s", Address, name, version, raw) + } + if a.Version != version { + return false, fmt.Sprintf("%s reserved %s for %s, not %s", a.Module, a.Version, name, version), nil + } + for _, art := range a.Artifacts { + if art.Digest == digest { + return true, "", nil + } + } + return false, fmt.Sprintf("%s reserved %s@%s for another digest than %s", a.Module, name, version, digest), nil +} diff --git a/modules/gitea/internal/packageseats/reserved_test.go b/modules/gitea/internal/packageseats/reserved_test.go new file mode 100644 index 000000000..b5ea18483 --- /dev/null +++ b/modules/gitea/internal/packageseats/reserved_test.go @@ -0,0 +1,54 @@ +package packageseats + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +// A publish is checked against the builder's reservation by the seat mesh-builds' verb reserved, asked with the package's name +// (design 47 §381, §478). +func TestReservedChecksAPublishAgainstTheReservation(t *testing.T) { + for _, c := range []struct { + name string + answer string + err error + ok bool + why string // part of the reason, or of the error + }{ + {name: "the reserved version and digest", answer: `{"module":"mesh-sdk","version":"0.1.19","artifacts":[{"artifact":"mesh-sdk:ts","digest":"sha256:aa"}]}`, ok: true}, + {name: "another artifact of it carries the digest", answer: `{"module":"mesh-sdk","version":"0.1.19","artifacts":[{"artifact":"mesh-sdk:go"},{"artifact":"mesh-sdk:ts","digest":"sha256:aa"}]}`, ok: true}, + {name: "another digest", answer: `{"module":"mesh-sdk","version":"0.1.19","artifacts":[{"artifact":"mesh-sdk:ts","digest":"sha256:bb"}]}`, why: "another digest than sha256:aa"}, + {name: "another version reserved", answer: `{"module":"mesh-sdk","version":"0.1.20","artifacts":[]}`, why: "reserved 0.1.20 for @novox/mesh-sdk, not 0.1.19"}, + {name: "the builder refused the name", err: errors.New("no entry carries @novox/mesh-sdk"), why: "no entry carries @novox/mesh-sdk"}, + {name: "an answer that is none", answer: `{"error":"x"}`, why: "answered no reservation"}, + } { + asked := "" + r := Reserved{Ask: func(address string, body any) (json.RawMessage, error) { + b, _ := json.Marshal(body) + asked = address + " " + string(b) + return json.RawMessage(c.answer), c.err + }} + ok, why, err := r.Recorded("@novox/mesh-sdk", "0.1.19", "sha256:aa") + if asked != `seat:mesh-builds.reserved {"name":"@novox/mesh-sdk","version":"0.1.19"}` { + t.Fatalf("%s: asked %s", c.name, asked) + } + if c.ok { + if !ok || err != nil { + t.Errorf("%s: not accepted: %v %v", c.name, why, err) + } + continue + } + if ok || !strings.Contains(why+fmtErr(err), c.why) { + t.Errorf("%s: ok=%v why=%q err=%v, want %q", c.name, ok, why, err, c.why) + } + } +} + +func fmtErr(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 7cabb3024..5255a55f8 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -1,9 +1,6 @@ { "module": "gitea", "version": "1", - "packages": { - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds": "0.1" - }, "moved": true, "requires": [ "postgres-database", @@ -651,7 +648,7 @@ "seat:mesh-catalog.revision", "seat:mesh-catalog.registrations", "seat:mesh-hand-acts.hand-act", - "mesh-delivery-builder.builder_versions" + "seat:mesh-builds.reserved" ], "consumes": [ "mesh-catalog.catalog-revision", -- 2.54.0 From 713b59633e1045b1379bbb3ae1270c3a0647236f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:05:41 +0100 Subject: [PATCH 032/243] REG1: distribution checks a publish against the builder's reservation, asked by the artifact's name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The container image and OCI artifact seats' holder asks the seat mesh-builds' verb reserved in place of builder_versions; the manifest invokes it and takes no builder package. Builds: design 47 G §381, §478, §504 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/distribution-tools/main.go | 3 +- modules/distribution/go.mod | 2 +- .../distribution/internal/holder/callers.go | 2 +- .../internal/holder/holder_test.go | 10 ++-- .../distribution/internal/holder/reserved.go | 42 +++++++++++++++ .../internal/holder/reserved_test.go | 54 +++++++++++++++++++ .../distribution/internal/holder/transfer.go | 8 +-- modules/distribution/module.json | 5 +- 8 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 modules/distribution/internal/holder/reserved.go create mode 100644 modules/distribution/internal/holder/reserved_test.go diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index 4270a2adb..41c949a4a 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -12,7 +12,6 @@ import ( "encoding/json" "errors" "fmt" - builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "net/url" "os" "strings" @@ -520,7 +519,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder. func main() { base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: builds.BuilderVersions{Ask: stdio.Ask}, Store: store, Say: say, Finding: finding, + h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: finding, Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Record: recordAct, Idle: parts.BusArm(holder.IdleState)} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, diff --git a/modules/distribution/go.mod b/modules/distribution/go.mod index 4a037fc1a..4fbf8a54a 100644 --- a/modules/distribution/go.mod +++ b/modules/distribution/go.mod @@ -8,7 +8,7 @@ require ( git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 github.com/nats-io/nats.go v1.54.0 diff --git a/modules/distribution/internal/holder/callers.go b/modules/distribution/internal/holder/callers.go index b3021139e..897c12152 100644 --- a/modules/distribution/internal/holder/callers.go +++ b/modules/distribution/internal/holder/callers.go @@ -2,7 +2,7 @@ // `oci-artifact-registry`, and of the old seat `mesh-artifact-store` until its window closes (novox/hq design 51 *The // four registry seats*, ADR 0605, ADR 0611, ADR 0617 §2). A build module publishes into a seat through `publish` and // `publish-part`: the parts land in distribution's own object store `incoming`, are hashed, checked against the -// builder's `builder_versions` and stored in the artifact store, which listens on its node's loopback; the store's port +// builder's reservation (`reserved`) and stored in the artifact store, which listens on its node's loopback; the store's port // is the program's, reads open, writes from its own process alone (front.go). package holder diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index d3cbe2bac..e0820d49f 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -286,12 +286,12 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { _, err := b.h.Publish(ArtifactSeat, builder, req("t")) return err }, "at their cap"}, - {"a version the builder did not number", func(b *bed) error { + {"a version the builder did not reserve", func(b *bed) error { b.h.Builder = fakeBuilder{ok: false} b.store.has[part.Digest] = true // held already: the publish ends at once _, err := b.h.Publish(ArtifactSeat, builder, req("t")) return err - }, "the builder did not number it"}, + }, "the builder did not reserve it"}, {"an image on the artifact seat", func(b *bed) error { r := req("t") r.Kind = KindImage @@ -843,9 +843,9 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) } - // The builder's question is its published package, taken by its series (ADR 0634 §1). - if m.Packages["git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds"] != "0.1" { - t.Errorf("packages %v: want builds of series 0.1", m.Packages) + // The reservation is asked through the seat mesh-builds, so the manifest takes no package of the builder (design 47 §381). + if len(m.Packages) != 0 { + t.Errorf("packages %v: want none", m.Packages) } // E15: the provision artifact-store retires (ADR 0632 §3); the bootstrap keeps it as an alias on port 5100 while // the running node-engines still find the store by it (issue 1966), and nothing else is provided or served. diff --git a/modules/distribution/internal/holder/reserved.go b/modules/distribution/internal/holder/reserved.go new file mode 100644 index 000000000..89ebd9ae2 --- /dev/null +++ b/modules/distribution/internal/holder/reserved.go @@ -0,0 +1,42 @@ +package holder + +import ( + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshbuilds" +) + +// Reserved is the holder's check of a publish against the builder's reservation (novox/hq design 47 §381, §478, §504): it asks the +// seat mesh-builds' verb reserved with the publish's name and version, a package's name in its ecosystem, and accepts the publish +// only when that version is the one reserved and an artifact of it has the publish's digest. It reads no catalog and a publish gains +// no field. It is the Builder the holder's Check asks. +type Reserved struct { + // Ask is the runtime's request and reply over the bus (mesh-sdk stdio.Ask). + Ask func(address string, body any) (json.RawMessage, error) +} + +// Address is the seat verb asked, as the manifest's `invokes` names it. +const Address = "seat:" + meshbuilds.Seat + "." + meshbuilds.VerbReserved + +// Recorded answers whether name@version with digest is reserved, and why not when it is not. An answer that cannot be read is an +// error, never a yes; a refusal by the builder (a name no entry carries, a version it did not reserve) is a no that names it. +func (r Reserved) Recorded(name, version, digest string) (bool, string, error) { + raw, err := r.Ask(Address, meshbuilds.ReservedInput{Name: name, Version: version}) + if err != nil { + return false, "", fmt.Errorf("%s did not answer for %s@%s: %w", Address, name, version, err) + } + var a meshbuilds.ReservedAnswer + if err := json.Unmarshal(raw, &a); err != nil || a.Version == "" { + return false, "", fmt.Errorf("%s answered no reservation for %s@%s: %s", Address, name, version, raw) + } + if a.Version != version { + return false, fmt.Sprintf("%s reserved %s for %s, not %s", a.Module, a.Version, name, version), nil + } + for _, art := range a.Artifacts { + if art.Digest == digest { + return true, "", nil + } + } + return false, fmt.Sprintf("%s reserved %s@%s for another digest than %s", a.Module, name, version, digest), nil +} diff --git a/modules/distribution/internal/holder/reserved_test.go b/modules/distribution/internal/holder/reserved_test.go new file mode 100644 index 000000000..312bf4afc --- /dev/null +++ b/modules/distribution/internal/holder/reserved_test.go @@ -0,0 +1,54 @@ +package holder + +import ( + "encoding/json" + "errors" + "strings" + "testing" +) + +// A publish is checked against the builder's reservation by the seat mesh-builds' verb reserved, asked with the package's name +// (design 47 §381, §478). +func TestReservedChecksAPublishAgainstTheReservation(t *testing.T) { + for _, c := range []struct { + name string + answer string + err error + ok bool + why string // part of the reason, or of the error + }{ + {name: "the reserved version and digest", answer: `{"module":"mesh-image","version":"0.1.7","artifacts":[{"artifact":"mesh-image:image","digest":"sha256:aa"}]}`, ok: true}, + {name: "another artifact of it carries the digest", answer: `{"module":"mesh-image","version":"0.1.7","artifacts":[{"artifact":"mesh-image:aux"},{"artifact":"mesh-image:image","digest":"sha256:aa"}]}`, ok: true}, + {name: "another digest", answer: `{"module":"mesh-image","version":"0.1.7","artifacts":[{"artifact":"mesh-image:image","digest":"sha256:bb"}]}`, why: "another digest than sha256:aa"}, + {name: "another version reserved", answer: `{"module":"mesh-image","version":"0.1.8","artifacts":[]}`, why: "reserved 0.1.8 for registry/mesh-image, not 0.1.7"}, + {name: "the builder refused the name", err: errors.New("no entry carries registry/mesh-image"), why: "no entry carries registry/mesh-image"}, + {name: "an answer that is none", answer: `{"error":"x"}`, why: "answered no reservation"}, + } { + asked := "" + r := Reserved{Ask: func(address string, body any) (json.RawMessage, error) { + b, _ := json.Marshal(body) + asked = address + " " + string(b) + return json.RawMessage(c.answer), c.err + }} + ok, why, err := r.Recorded("registry/mesh-image", "0.1.7", "sha256:aa") + if asked != `seat:mesh-builds.reserved {"name":"registry/mesh-image","version":"0.1.7"}` { + t.Fatalf("%s: asked %s", c.name, asked) + } + if c.ok { + if !ok || err != nil { + t.Errorf("%s: not accepted: %v %v", c.name, why, err) + } + continue + } + if ok || !strings.Contains(why+fmtErr(err), c.why) { + t.Errorf("%s: ok=%v why=%q err=%v, want %q", c.name, ok, why, err, c.why) + } + } +} + +func fmtErr(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/modules/distribution/internal/holder/transfer.go b/modules/distribution/internal/holder/transfer.go index 6f2a4e5c3..e07812af9 100644 --- a/modules/distribution/internal/holder/transfer.go +++ b/modules/distribution/internal/holder/transfer.go @@ -129,7 +129,7 @@ type Objects = parts.Objects // TransferState is the state transfers. type TransferState = parts.Transfers[Opening] -// Builder is the builder's tool builder_versions, asked before anything is stored (ADR 0605 §4). +// Builder is the builder's reservation (Reserved), asked before anything is stored (ADR 0605 §4, design 47 §381). type Builder interface { Recorded(name, version, digest string) (ok bool, why string, err error) } @@ -196,17 +196,17 @@ func (h *Holder) receiver() *parts.Receiver[Opening] { ok, err := h.Store.Has(t.Shape.Name, digest) return err == nil && ok }, - // builder_versions is asked before anything is stored (ADR 0605 §4). + // The reservation is asked before anything is stored (ADR 0605 §4, design 47 §381). Check: func(t Transfer) error { if t.Shape.Version == "" { return nil } ok, why, err := h.Builder.Recorded(t.Shape.Name, t.Shape.Version, t.Shape.Digest) if err != nil { - return errors.New("builder_versions did not answer, so nothing is stored: " + err.Error()) + return errors.New("the reservation could not be read, so nothing is stored: " + err.Error()) } if !ok { - return errors.New("the builder did not number it: " + why) + return errors.New("the builder did not reserve it: " + why) } return nil }, diff --git a/modules/distribution/module.json b/modules/distribution/module.json index e834f13fc..04b974bbb 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -1,9 +1,6 @@ { "module": "distribution", "version": "2", - "packages": { - "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds": "0.1" - }, "provides": [ { "name": "artifact-store", @@ -63,7 +60,7 @@ ], "invokes": [ "distribution.registry_callers", - "mesh-delivery-builder.builder_versions", + "seat:mesh-builds.reserved", "mesh-delivery-view.node_haves", "seat:node-engine.manifest-reads", "seat:mesh-hand-acts.hand-act" -- 2.54.0 From 299f43167c7c54bfdde001842e4dd5d1d86e28dd Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:15:01 +0100 Subject: [PATCH 033/243] GIT12, REG1: the reservation check's test is a case of the holder's own test, so no module ends over its start count MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §381; test budget Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/distribution/internal/holder/holder_test.go | 1 + modules/distribution/internal/holder/reserved_test.go | 2 +- modules/gitea/internal/packageseats/holder_test.go | 1 + modules/gitea/internal/packageseats/reserved_test.go | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index e0820d49f..c428e942b 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -223,6 +223,7 @@ const builder = "ace.mesh-build-go" // TestTheHolderOfTheTwoImageSeats holds design 51's holder cases at the unit level (CAT-D1, D2, D3, D3b, D4, D4b, D5): // one table or case per rule, the caller list and the store front's. func TestTheHolderOfTheTwoImageSeats(t *testing.T) { + t.Run("a publish is checked against the builder's reservation", reservedChecksAPublishAgainstTheReservation) blob := []byte(strings.Repeat("a", 3000)) part := Part{Digest: digestOf(blob), Size: int64(len(blob))} req := func(id string) PublishRequest { diff --git a/modules/distribution/internal/holder/reserved_test.go b/modules/distribution/internal/holder/reserved_test.go index 312bf4afc..faa32b074 100644 --- a/modules/distribution/internal/holder/reserved_test.go +++ b/modules/distribution/internal/holder/reserved_test.go @@ -9,7 +9,7 @@ import ( // A publish is checked against the builder's reservation by the seat mesh-builds' verb reserved, asked with the package's name // (design 47 §381, §478). -func TestReservedChecksAPublishAgainstTheReservation(t *testing.T) { +func reservedChecksAPublishAgainstTheReservation(t *testing.T) { for _, c := range []struct { name string answer string diff --git a/modules/gitea/internal/packageseats/holder_test.go b/modules/gitea/internal/packageseats/holder_test.go index 0495e041a..029810f47 100644 --- a/modules/gitea/internal/packageseats/holder_test.go +++ b/modules/gitea/internal/packageseats/holder_test.go @@ -119,6 +119,7 @@ const goBuild, nodeBuild = "ace.mesh-build-go", "ace.mesh-build-node" // file comes in parts, is checked against the builder's numbering and its digest, and is uploaded with the forge's own // write side; the outcome is said on the seat's events. func TestTheForgeHoldsBothPackageSeats(t *testing.T) { + t.Run("a publish is checked against the builder's reservation", reservedChecksAPublishAgainstTheReservation) file := []byte("a package's tarball") t.Run("a publish in parts is uploaded and said published", func(t *testing.T) { for _, seat := range []string{NpmSeat, GoSeat} { diff --git a/modules/gitea/internal/packageseats/reserved_test.go b/modules/gitea/internal/packageseats/reserved_test.go index b5ea18483..dff3f891f 100644 --- a/modules/gitea/internal/packageseats/reserved_test.go +++ b/modules/gitea/internal/packageseats/reserved_test.go @@ -9,7 +9,7 @@ import ( // A publish is checked against the builder's reservation by the seat mesh-builds' verb reserved, asked with the package's name // (design 47 §381, §478). -func TestReservedChecksAPublishAgainstTheReservation(t *testing.T) { +func reservedChecksAPublishAgainstTheReservation(t *testing.T) { for _, c := range []struct { name string answer string -- 2.54.0 From 119e0a6f13beb1fb3f11352ab8c19ca5a2bfbc4f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:32:36 +0100 Subject: [PATCH 034/243] A-C2: a build module runs the pipeline step kind step and says its end on its node (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner asks a lint or test step of an artifact as the build seat's accepted kind step, and hears its end as the seat's step-ended on the node that ran it (ADR 0666 sections 7, 8, 21; design 47 *The steps and who serves them* line 571 and *Asked on a work queue, ended by an event* lines 583-590 at hq b229f5c4). The build modules' shared package runs a test step as a check of its one level, carries the packages whose digest the ask names kept, keeps what it passed and says the counts; a level with no tests ends nothing to run, a source fault could not run, and a transport fault or an end that could not be said is handed back unsaid, so the work queue gives the step again and it ends once under its ask's id. The neutral forms are the domain SDK's (sdk/buildstep pipelinestep.go, mesh-delivery dc0df1b5); the Go module says the end through the runtime's seat publish. Milestone 47/G, research 086 area A. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 72fee3c6.) Builds: design 47 G §22, §24, §86, §116, §140 (BLD3, BLD4) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-build-go/buildstep/askrun.go | 10 + .../mesh-build-go/buildstep/askrun_test.go | 100 +++++++ modules/mesh-build-go/buildstep/steprun.go | 252 ++++++++++++++++++ .../mesh-build-go/cmd/mesh-build-go/asks.go | 11 + .../mesh-build-go/cmd/mesh-build-go/module.go | 2 + 5 files changed, 375 insertions(+) create mode 100644 modules/mesh-build-go/buildstep/steprun.go diff --git a/modules/mesh-build-go/buildstep/askrun.go b/modules/mesh-build-go/buildstep/askrun.go index 6400ec15d..050a0b046 100644 --- a/modules/mesh-build-go/buildstep/askrun.go +++ b/modules/mesh-build-go/buildstep/askrun.go @@ -47,6 +47,13 @@ type AskSteps struct { // Has says whether this node has a need a package's tests declare (sdkstep.AskNeeds, issue 890); a package needing // what it lacks is not tested here but named in the level's Unmet, for the asker to place. Nil has nothing. Has func(need string) bool + // Lint is the module's lint body for a pipeline step lint (novox/hq ADR 0666 §7): it takes the workspace and the + // module's directory and answers a LintAnswer. Nil ends every lint step as nothing to run. + Lint StepFunc + // Node is the node this holder runs on, which a step's end names; SayStep says a pipeline step's end as the seat's + // event step-ended on that node's subject (sdkstep.StepEndedSubject), under the ask's id. Nil runs no step ask. + Node string + SayStep func(end sdkstep.StepEnded) error } // RunAsk runs ask's job with steps and answers a sdkstep.BuildAnswer or a sdkstep.CheckAnswer. @@ -56,6 +63,9 @@ func RunAsk(ctx context.Context, steps AskSteps, ask sdkstep.StepAsk) (any, erro job.Fetch = withPackages(job.Fetch, ask.Packages) return steps.build(ctx, job) } + if ask.Step != nil { + return steps.runStep(ctx, ask) + } if ask.Check != nil { job := *ask.Check job.Fetch = *withPackages(&job.Fetch, ask.Packages) diff --git a/modules/mesh-build-go/buildstep/askrun_test.go b/modules/mesh-build-go/buildstep/askrun_test.go index d9a93e195..693afbb83 100644 --- a/modules/mesh-build-go/buildstep/askrun_test.go +++ b/modules/mesh-build-go/buildstep/askrun_test.go @@ -369,3 +369,103 @@ func TestASeatsAskTypeRunsAsTheNeutralAsk(t *testing.T) { t.Errorf("the ask as another seat's type: %+v, %v", back, err) } } + +// 086 A-C2 (ADR 0666 §7, §8, §21; design 47 *Asked on a work queue, ended by an event*, test 65): a pipeline step ask +// runs its one level as a check of it, carries the packages the runner kept at their digest, keeps what it passed, says +// its counts, and says its end once on its own node; a level with no tests ends nothing to run, a fault of the source +// could not run, and a transport fault or an end that could not be said is handed back unsaid, to be given again. +func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { + type said struct{ ends []sdkstep.StepEnded } + stepOf := func(step string, kept ...sdkstep.KeptResult) sdkstep.StepAsk { + return sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(), + Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: step, Module: "m", Artifact: "m:program", Commit: runCommit, Fingerprint: "fp-1", + Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: runCommit, Dir: "modules/m"}, Kept: kept, + Names: sdkstep.StepNames{Modules: []string{"m"}, Repositories: []string{"novox/x"}, Branches: []string{"main"}}}} + } + for _, row := range []struct { + name string + ask sdkstep.StepAsk + digests map[string]string + tests map[string]bool + lint StepFunc + fetchErr error + sayErr error + outcome string // "" when nothing is said + counts int + carried []string + kept []string + passing bool // the error is a transport fault + noTests bool // the level holds no test, as the test body says it + }{ + {name: "unit: the kept package carried, the other tested and kept", ask: stepOf(sdkstep.StepUnit, sdkstep.KeptResult{Package: "x/a", Digest: "d-a", Level: "unit"}), + digests: map[string]string{"x/a": "d-a", "x/b": "d-b"}, tests: map[string]bool{"x/b": true}, outcome: sdkstep.OutcomePassed, counts: 1, carried: []string{"x/a"}, kept: []string{"x/b"}}, + {name: "unit failed: said with why, nothing kept", ask: stepOf(sdkstep.StepUnit), digests: map[string]string{"x/b": "d-b"}, tests: map[string]bool{"x/b": false}, + outcome: sdkstep.OutcomeFailed, counts: 1}, + {name: "integration with no tests: nothing to run", ask: stepOf(sdkstep.StepIntegration), digests: map[string]string{}, noTests: true, outcome: sdkstep.OutcomeNothingToRun}, + {name: "lint failed", ask: stepOf(sdkstep.StepLint), lint: func(context.Context, map[string]any) (any, error) { + return LintAnswer{Failed: []sdkstep.Failure{{Package: "modules/m", Test: "gofmt", Cause: "not as gofmt writes them"}}}, nil + }, outcome: sdkstep.OutcomeFailed}, + {name: "lint passed", ask: stepOf(sdkstep.StepLint), lint: func(context.Context, map[string]any) (any, error) { return LintAnswer{}, nil }, outcome: sdkstep.OutcomePassed}, + {name: "no lint body: nothing to run", ask: stepOf(sdkstep.StepLint), outcome: sdkstep.OutcomeNothingToRun}, + {name: "the source cannot be fetched: could not run", ask: stepOf(sdkstep.StepUnit), fetchErr: Refuse("no commit %s", runCommit), outcome: sdkstep.OutcomeCouldNotRun}, + {name: "a transport fault: unsaid, given again", ask: stepOf(sdkstep.StepUnit), fetchErr: &sdkstep.Passing{Err: errors.New("no responders")}, passing: true}, + {name: "the end cannot be said: given again", ask: stepOf(sdkstep.StepLint), sayErr: errors.New("bus down"), passing: true}, + } { + r := &recorder{fetch: map[string]any{"workspace": "/w"}, tests: row.tests, digests: row.digests} + s := r.steps() + if row.fetchErr != nil { + s.Fetch = func(context.Context, map[string]any) (any, error) { return nil, row.fetchErr } + } + s.Lint, s.Node = row.lint, "ace" + if row.noTests { + s.Test = func(context.Context, map[string]any) (any, error) { + return map[string]any{"passed": true, "tests": 0, "no_tests": true}, nil + } + } + var got said + s.SayStep = func(e sdkstep.StepEnded) error { + if row.sayErr != nil { + return row.sayErr + } + got.ends = append(got.ends, e) + return nil + } + out, err := RunAsk(context.Background(), s, row.ask) + if row.passing { + if !sdkstep.IsPassing(err) || len(got.ends) != 0 { + t.Errorf("%s: %v, said %+v; want a transport fault, nothing said", row.name, err, got.ends) + } + continue + } + if err != nil || len(got.ends) != 1 { + t.Fatalf("%s: %v, said %+v; want one end", row.name, err, got.ends) + } + e := got.ends[0] + if e.Outcome != row.outcome || e.Node != "ace" || e.ID != row.ask.ID || e.Fingerprint != "fp-1" || out.(StepAnswer).Ended.Outcome != row.outcome { + t.Errorf("%s: said %+v, want %s on ace for the ask", row.name, e, row.outcome) + } + if (row.outcome == sdkstep.OutcomeFailed || row.outcome == sdkstep.OutcomeCouldNotRun) && e.Said == "" { + t.Errorf("%s: %s and not why", row.name, row.outcome) + } + if n := 0; e.Tests != nil { + n = e.Tests.Unit + e.Tests.Integration + if n != row.counts { + t.Errorf("%s: counts %+v, want %d", row.name, e.Tests, row.counts) + } + } else if row.counts != 0 { + t.Errorf("%s: no counts, want %d", row.name, row.counts) + } + pkgs := func(ks []sdkstep.KeptResult) (out []string) { + for _, k := range ks { + out = append(out, k.Package) + } + return out + } + if !slices.Equal(pkgs(e.Carried), row.carried) || !slices.Equal(pkgs(e.Kept), row.kept) { + t.Errorf("%s: carried %v kept %v, want %v and %v", row.name, pkgs(e.Carried), pkgs(e.Kept), row.carried, row.kept) + } + if _, err := sdkstep.EncodeStepEnded(e); err != nil { + t.Errorf("%s: the end does not encode: %v", row.name, err) + } + } +} diff --git a/modules/mesh-build-go/buildstep/steprun.go b/modules/mesh-build-go/buildstep/steprun.go new file mode 100644 index 000000000..3549c170b --- /dev/null +++ b/modules/mesh-build-go/buildstep/steprun.go @@ -0,0 +1,252 @@ +package buildstep + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" +) + +// **A pipeline step, run and said** (novox/hq ADR 0666 §7, §8, §17, §21; design 47 *Pipeline steps and the policy*, +// *Asked on a work queue, ended by an event*, *Carried, not run*). The runner asks one lint or test step of an artifact +// as the accepted kind step; the holder fetches the change's heads, runs the step and says its end as the seat's event +// step-ended on its own node's subject before the ask is acknowledged, so a holder restarted mid-step gets the ask again +// from its work queue and finishes it (ADR 0664 §1), its end said under the ask's id. A test step runs only the packages +// with no kept result for their digest at its level, says the others carried, and keeps what it passed; a step with +// nothing to do ends `nothing to run`, never passed; one that cannot run ends `could not run`, saying why. + +// StepAnswer is what a step ask answers on its step key: the end it said. +type StepAnswer struct { + Ended sdkstep.StepEnded `json:"ended"` +} + +// LintAnswer is what a module's lint body answers: the failures of its language's lint, none when it passed, and +// Nothing when the tree holds nothing its lint reads. +type LintAnswer struct { + Failed []sdkstep.Failure `json:"failed,omitempty"` + Nothing bool `json:"nothing,omitempty"` +} + +// runStep runs ask's pipeline step and says its end. A transport fault, a decline or the process stopping is handed back +// unsaid, for the bus to give the ask again, here or on another node. +func (s AskSteps) runStep(ctx context.Context, ask sdkstep.StepAsk) (any, error) { + if s.SayStep == nil { + return nil, Refuse("this build module says no step's end") + } + end, err := sdkstep.EndOf(ask, s.Node, sdkstep.OutcomePassed) + if err != nil { + return nil, Refuse("%v", err) + } + if err := s.stepOutcome(ctx, ask, &end); err != nil { + return nil, err + } + if err := s.SayStep(end); err != nil { + return nil, &sdkstep.Passing{Err: fmt.Errorf("the end of the step %s could not be said: %w", ask.ID, err)} + } + return StepAnswer{Ended: end}, nil +} + +// stepOutcome runs the step into end; the error it answers is one that keeps the end unsaid. +func (s AskSteps) stepOutcome(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { + var err error + if ask.Step.Step == sdkstep.StepLint { + err = s.lintStep(ctx, *ask.Step, end) + } else { + err = s.testStep(ctx, ask, end) + } + var dec *Declining + switch { + case err == nil: + return nil + case sdkstep.IsPassing(err), errors.As(err, &dec), ctx.Err() != nil: + return err + default: + end.Outcome, end.Said = sdkstep.OutcomeCouldNotRun, err.Error() + end.Tests, end.Carried, end.Kept = nil, nil, nil + return nil + } +} + +// lintStep fetches the heads and runs the module's lint body over the artifact's tree. +func (s AskSteps) lintStep(ctx context.Context, run sdkstep.StepRun, end *sdkstep.StepEnded) error { + if s.Lint == nil { + end.Outcome = sdkstep.OutcomeNothingToRun + return nil + } + if _, err := s.onToolchain(ctx, run.Toolchain); err != nil { + return err + } + fetch := run.Fetch + _, f, err := s.fetch(ctx, &fetch) + if err != nil { + return err + } + defer s.release(f.Workspace) + if len(f.Conflicts) > 0 { + return fmt.Errorf("the heads do not merge: %s", strings.Join(f.Conflicts, ", ")) + } + Phase(ctx, "lint") + out, err := s.Lint(ctx, map[string]any{"workspace": f.Workspace, "dir": fetch.Dir}) + if err != nil { + return err + } + r, err := raw(out) + if err != nil { + return err + } + var la LintAnswer + if err := json.Unmarshal(r, &la); err != nil { + return fmt.Errorf("the lint answered what is not a lint answer: %s: %w", r, err) + } + switch { + case len(la.Failed) > 0: + end.Outcome, end.Said = sdkstep.OutcomeFailed, failedSaid(la.Failed) + case la.Nothing: + end.Outcome = sdkstep.OutcomeNothingToRun + default: + end.Outcome = sdkstep.OutcomePassed + } + return nil +} + +// testStep runs one test level as a check of that level alone, taking the step's kept results in the checker's place. +func (s AskSteps) testStep(ctx context.Context, ask sdkstep.StepAsk, end *sdkstep.StepEnded) error { + run := *ask.Step + level := run.Step + s.Kept = func(_ context.Context, lvl, digest string) (int, bool, error) { + for _, k := range run.Kept { + if k.Level == lvl && k.Digest == digest { + return 0, true, nil + } + } + return 0, false, nil + } + job := sdkstep.CheckJob{Fetch: *withPackages(&run.Fetch, ask.Packages), Levels: []string{level}, Kept: true, Toolchain: run.Toolchain} + out, err := s.check(ctx, job) + if err != nil { + return err + } + answer, _ := out.(sdkstep.CheckAnswer) + if len(answer.Levels) == 0 { + var conflicts struct { + Conflicts []string `json:"conflicts"` + } + _ = json.Unmarshal(mustRaw(answer.Fetch), &conflicts) + return fmt.Errorf("the heads do not merge: %s", strings.Join(conflicts.Conflicts, ", ")) + } + la := answer.Levels[0] + if len(la.Unmet) > 0 { + var what []string + for _, u := range la.Unmet { + what = append(what, u.Package+" ("+strings.Join(u.Needs, ", ")+")") + } + return fmt.Errorf("the %s tests of %s need what this node lacks", level, strings.Join(what, ", ")) + } + tests, passed, ran := 0, true, false + var why []string + for _, t := range la.Tests { + var tr struct { + Passed bool `json:"passed"` + Tests int `json:"tests"` + NoTests bool `json:"no_tests"` + Why string `json:"why"` + } + if err := json.Unmarshal(t, &tr); err != nil { + return fmt.Errorf("the %s level answered what is no test result: %w", level, err) + } + tests += tr.Tests + ran = ran || (!tr.NoTests && tr.Tests > 0) || !tr.Passed + if !tr.Passed { + passed = false + if tr.Why != "" { + why = append(why, tr.Why) + } + } + } + for _, k := range la.Kept { + end.Carried = append(end.Carried, keptOf(run, k.Package, k.Digest)) + } + switch { + case !passed: + end.Outcome, end.Said = sdkstep.OutcomeFailed, strings.Join(why, "; ") + if end.Said == "" { + end.Said = "the " + level + " tests failed" + } + case !ran && len(end.Carried) == 0: + end.Outcome = sdkstep.OutcomeNothingToRun + default: + end.Outcome = sdkstep.OutcomePassed + if la.Digest != nil { + for _, p := range la.Digest.Packages { + if !carried(end.Carried, p.Package) { + end.Kept = append(end.Kept, keptOf(run, p.Package, p.Digest)) + } + } + } + } + if end.Outcome != sdkstep.OutcomeNothingToRun { + end.Tests = &sdkstep.TestCounts{} + if level == sdkstep.StepUnit { + end.Tests.Unit = tests + } else { + end.Tests.Integration = tests + } + } + return nil +} + +func keptOf(run sdkstep.StepRun, pkg, digest string) sdkstep.KeptResult { + return sdkstep.KeptResult{Package: pkg, Digest: digest, Level: run.Step, Toolchain: run.Toolchain, Pipeline: run.Pipeline} +} + +func carried(kept []sdkstep.KeptResult, pkg string) bool { + for _, k := range kept { + if k.Package == pkg { + return true + } + } + return false +} + +func mustRaw(v any) json.RawMessage { + r, _ := raw(v) + return r +} + +// failedSaid names each failure in one line, the first ten. +func failedSaid(failed []sdkstep.Failure) string { + var names []string + for i, f := range failed { + if i == 10 { + names = append(names, fmt.Sprintf("and %d more", len(failed)-10)) + break + } + names = append(names, strings.TrimSpace(f.Package+" "+f.Test+": "+firstOf(f.Cause))) + } + return fmt.Sprintf("%d failed: %s", len(failed), strings.Join(names, "; ")) +} + +func firstOf(s string) string { + l, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return l +} + +// SayStepOn is a holder's SayStep: the end, encoded and checked by the domain SDK (sdkstep.EncodeStepEnded), published +// on the language's build seat's step-ended subject of node with publish (stdio.SeatPublish), de-duplicated by the +// ask's id, so a step run again after a restart is said once. +func SayStepOn(language, node string, publish func(subject string, body any, id string) error) func(sdkstep.StepEnded) error { + return func(end sdkstep.StepEnded) error { + subject, err := sdkstep.StepEndedSubject(language, node) + if err != nil { + return err + } + body, err := sdkstep.EncodeStepEnded(end) + if err != nil { + return err + } + return publish(subject, json.RawMessage(body), "step-ended."+end.ID) + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index a1fcf324a..fa751589a 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -56,6 +56,8 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { Has: m.Has, Release: func(ws string) error { return m.spaces().Release(ws) }, Logf: m.Say, + Node: m.Node, + SayStep: m.sayStep(), } } @@ -151,3 +153,12 @@ func (m *Module) endLeftRunning(ctx context.Context, scan buildrun.Scan, booted func (m *Module) stepScan() buildrun.Scan { return buildrun.ScanOf(stdio.State(seat.StateSteps)) } + +// sayStep says a pipeline step's end on this node's subject of the seat (design 47 *Asked on a work queue, ended by an +// event*, lines 584-586 at b229f5c4): SayStep when a test sets it, else the runtime's seat publish. +func (m *Module) sayStep() func(sdkstep.StepEnded) error { + if m.SayStep != nil { + return m.SayStep + } + return buildstep.SayStepOn(sdkstep.AskGo, m.Node, stdio.SeatPublish) +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 1c57a3d28..b5c99cecf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -102,6 +102,8 @@ type Module struct { Settings string Capacity func() int ReadKept func(key string) (json.RawMessage, error) + // SayStep says a pipeline step's end (buildstep.AskSteps.SayStep); the seat's publish on this node when nil. + SayStep func(sdkstep.StepEnded) error // ReadManifest asks the delivery indexer's read tool (ManifestsTool) for one manifest by its key, which a compose ask's // request names its manifests by (compose.go); it reads none of the indexer's state (ADR 0596); GoModCache, when set, is the Go module cache in place of Cache's (a test). ReadManifest func(key string) (json.RawMessage, error) -- 2.54.0 From 663d65640dbe689d6e70d46ec7b8eebfdebd5667 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:32:41 +0100 Subject: [PATCH 035/243] A-C3: gofmt and vet are the lint step's, not the unit level's (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lint is a pipeline step of its own, served by the build seat (ADR 0666 section 7; design 47 *The steps and who serves them* line 571 at hq b229f5c4), so the unit level runs tests alone and the lint step runs gofmt and go vet, with every tag the files name, over each go.mod directory of the artifact's module directory; a module directory holding no go.mod has nothing to lint (prepare finding F12). Milestone 47/G, research 086 area A. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from a46e01a2.) Builds: design 47 G §328, §417 (BLD6) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/cmd/mesh-build-go/asks.go | 1 + .../mesh-build-go/cmd/mesh-build-go/levels.go | 2 - .../mesh-build-go/cmd/mesh-build-go/lint.go | 49 ++++++++++++++++--- .../cmd/mesh-build-go/lint_test.go | 36 +++++++++----- .../mesh-build-go/cmd/mesh-build-go/test.go | 15 ------ .../test_real_integration_test.go | 16 ++++-- 6 files changed, 77 insertions(+), 42 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index fa751589a..6d4430e1f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -56,6 +56,7 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { Has: m.Has, Release: func(ws string) error { return m.spaces().Release(ws) }, Logf: m.Say, + Lint: m.lintStep, Node: m.Node, SayStep: m.sayStep(), } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/levels.go b/modules/mesh-build-go/cmd/mesh-build-go/levels.go index bfc2a1d7a..903636356 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/levels.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/levels.go @@ -54,8 +54,6 @@ type TestResult struct { Elapsed float64 `json:"elapsed"` // Setup is the level's setup time in seconds, when it has a setup. Setup float64 `json:"setup,omitempty"` - // Lint is the unit level's gofmt and go vet time in seconds, before its tests (ADR 0529 §1). - Lint float64 `json:"lint,omitempty"` // Bound and SetupBound are the bounds the level was judged by, in seconds. Bound float64 `json:"bound"` SetupBound float64 `json:"setup_bound,omitempty"` diff --git a/modules/mesh-build-go/cmd/mesh-build-go/lint.go b/modules/mesh-build-go/cmd/mesh-build-go/lint.go index 8a9f4fc1c..90138afad 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/lint.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/lint.go @@ -19,11 +19,12 @@ import ( "git.novox.be/novox/mesh-sdk/go/forgekey" ) -// **gofmt and go vet are the unit level's** (novox/hq ADR 0529 §1; issue 1887 D7-go): what every repository's script ran -// beside its tests, run for every repository by the build module that runs its unit level. In each go.mod directory the -// level runs, every Go file outside testdata and vendor must be as gofmt writes it, and `go vet ./...` must pass with no -// tag and with every build tag the directory's files name together (integration, isolation, ...), so code behind a tag -// nothing else compiles is vetted too. A failure is the level's, naming the files or vet's lines. +// **gofmt and go vet are the pipeline step lint's** (novox/hq ADR 0666 §7; design 47 *The steps and who serves them*, +// line 571 at b229f5c4: lint is a step of its own, served by the build seat; ADR 0529 §1 had them in the unit level). In +// each go.mod directory of the artifact's module directory, every Go file outside testdata and vendor must be as gofmt +// writes it, and `go vet ./...` must pass with no tag and with every build tag the directory's files name together +// (integration, isolation, ...), so code behind a tag nothing else compiles is vetted too. A failure is the step's, +// naming the files or vet's lines; the unit level runs tests alone. // buildTag is one name in a //go:build line. var buildTag = regexp.MustCompile(`[A-Za-z_][A-Za-z0-9_.]*`) @@ -134,8 +135,7 @@ func (m *Module) lint(ctx context.Context, run forgekey.Runner, tree, d string, // and the error is the first directory's, so the answer does not depend on which ended first (issue 1930). func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string, dirs []string, env func(d string) []string) ([]Failure, error) { // The ask's phases say it (issue 1890): lint of the directories, then their tests. - buildstep.Phase(ctx, fmt.Sprintf("unit: lint of %d directories", len(dirs))) - defer buildstep.Phase(ctx, fmt.Sprintf("unit: tests of %d directories", len(dirs))) + buildstep.Phase(ctx, fmt.Sprintf("lint of %d directories", len(dirs))) type answer struct { failed []Failure err error @@ -174,3 +174,38 @@ func (m *Module) lintAll(ctx context.Context, run forgekey.Runner, tree string, } return failed, nil } + +// lintStep is the body of the pipeline step lint (buildstep.AskSteps.Lint): gofmt and go vet of every go.mod directory of +// the fetched workspace under the module's directory dir (the whole tree when dir is empty), answering a +// buildstep.LintAnswer; a tree with no go.mod there has nothing to lint. +func (m *Module) lintStep(ctx context.Context, args map[string]any) (any, error) { + workspace, _ := args["workspace"].(string) + dir, _ := args["dir"].(string) + if !workspaceOK.MatchString(workspace) { + return nil, fmt.Errorf("%q is not a workspace id", workspace) + } + tree := filepath.Join(m.Work, workspace, "tree") + all, err := m.goModDirs(ctx, tree) + if err != nil { + return nil, err + } + dir = strings.Trim(filepath.ToSlash(filepath.Clean(dir)), "/") + var dirs []string + for _, d := range all { + if dir == "" || dir == "." || d == dir || strings.HasPrefix(d, dir+"/") { + dirs = append(dirs, d) + } + } + if len(dirs) == 0 { + return buildstep.LintAnswer{Nothing: true}, nil + } + run, err := m.stepRunner(workspace, false) + if err != nil { + return nil, err + } + failed, err := m.lintAll(ctx, run, tree, dirs, func(d string) []string { return m.goEnvFor(filepath.Join(tree, d)) }) + if err != nil { + return nil, err + } + return buildstep.LintAnswer{Failed: failed}, nil +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/lint_test.go b/modules/mesh-build-go/cmd/mesh-build-go/lint_test.go index 3836304af..efd981bff 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/lint_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/lint_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -72,10 +73,11 @@ func TestTheUnitLevelRunsGofmtAndVetWithEveryTag(t *testing.T) { } } -// 1887/D7-go: through the step, the unit level of a tree holding an unformatted file fails, naming the file; its tests -// still run, so one check names the lint's failure and every failing test (the operator, 2026-11-02). -func TestAnUnformattedFileFailsTheUnitLevel(t *testing.T) { - m, states, ws, calls := workspaceFor(t, "empty") +// 086 A-C3 (design 47 *The steps and who serves them*, line 571 at b229f5c4; ADR 0666 §7): gofmt and vet are the lint +// step's, not the unit level's: a tree holding an unformatted file passes its unit level, and the lint step's body fails +// naming the file; a module directory holding no go.mod has nothing to lint. +func TestAnUnformattedFileFailsTheLintStepAndNotTheUnitLevel(t *testing.T) { + m, states, ws, _ := workspaceFor(t, "empty") tree := filepath.Join(m.Work, ws, "tree") for name, body := range map[string]string{"go.mod": "module x\n", "x.go": "package x\nfunc X() {}\n"} { if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o644); err != nil { @@ -83,16 +85,24 @@ func TestAnUnformattedFileFailsTheUnitLevel(t *testing.T) { } } res, _ := runUnit(t, m, states, ws) - if res.Passed || len(res.Failed) != 1 || res.Failed[0].Test != "gofmt" || !strings.Contains(res.Failed[0].Output, "x.go") { - t.Fatalf("result = %+v, want the level failed naming x.go", res) + for _, f := range res.Failed { + if f.Test == "gofmt" { + t.Errorf("the unit level still lints: %+v", res.Failed) + } } - // Review round 2 item 5: before the level's tests, which still run once gofmt failed. - tested := false - for _, c := range *calls { - tested = tested || strings.Contains(c.line, " test -json") - } - if !tested { - t.Errorf("the tests did not run after gofmt failed: %v", *calls) + for _, row := range []struct { + dir string + failed bool + nothing bool + }{{"", true, false}, {"modules/none", false, true}} { + out, err := m.lintStep(context.Background(), map[string]any{"workspace": ws, "dir": row.dir}) + if err != nil { + t.Fatal(err) + } + la := out.(buildstep.LintAnswer) + if got := len(la.Failed) == 1 && la.Failed[0].Test == "gofmt" && strings.Contains(la.Failed[0].Output, "x.go"); got != row.failed || la.Nothing != row.nothing { + t.Errorf("lint of %q = %+v, want failed %v nothing %v", row.dir, la, row.failed, row.nothing) + } } } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test.go b/modules/mesh-build-go/cmd/mesh-build-go/test.go index f9dcccb74..3ca5d79be 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test.go @@ -280,21 +280,6 @@ func (m *Module) runLevel(ctx context.Context, tree string, o levelOpts) (*TestR todo = append(todo, d) } } - if o.level == LevelUnit { - // gofmt and go vet of each directory the level runs come before its tests (ADR 0529 §1; issue 1887 D7-go): a - // directory that fails them fails the level, named first, and the tests still run, so one check names the lint's - // failures and every failing test (the operator, 2026-11-02); their time is the result's Lint. - linted := m.clock() - failed, err := m.lintAll(ctx, run, tree, todo, func(d string) []string { - return append(m.goEnvFor(filepath.Join(tree, d)), o.env...) - }) - if err != nil { - return nil, err - } - res.Lint = m.clock().Sub(linted).Seconds() - res.Failed = failed - began = m.clock() // the level's running time is its tests' - } runs := make([]dirRun, len(todo)) slots := make(chan struct{}, m.parallel()) var wg sync.WaitGroup diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go index 2b04bc9a6..369b38491 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go @@ -14,6 +14,8 @@ import ( "strings" "testing" "time" + + "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" ) // TestMain makes this test binary the bus when the module starts it as one (bus.go). @@ -99,9 +101,9 @@ func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) { } } -// 1887/D7-go against the real go: a tree holding an unformatted file fails the unit level naming the file first, its -// tests still run after it (ADR 0591: one check names every failure), and the level's result carries the lint's time. -func TestAnUnformattedFileFailsTheRealUnitLevel(t *testing.T) { +// 086 A-C3 against the real go (design 47 line 571 at b229f5c4): a tree holding an unformatted file passes its unit +// level, its two tests run, and the lint step's body fails naming the file. +func TestAnUnformattedFileFailsTheRealLintStepAndNotTheUnitLevel(t *testing.T) { needTools(t, "the unit level's gofmt and vet", "go", "git", "bwrap") home := t.TempDir() m := &Module{Home: home, Work: t.TempDir(), Cache: t.TempDir()} @@ -120,8 +122,12 @@ func TestAnUnformattedFileFailsTheRealUnitLevel(t *testing.T) { if err != nil { t.Fatal(err) } - if res.Passed || len(res.Failed) == 0 || res.Failed[0].Test != "gofmt" || !strings.Contains(res.Failed[0].Output, "ugly.go") || res.Tests != 2 || res.Lint <= 0 { - t.Fatalf("result = %+v, want the level failed by gofmt naming ugly.go, then the fixture's two tests run", res) + if !res.Passed || len(res.Failed) != 0 || res.Tests != 2 { + t.Fatalf("result = %+v, want the level passed with the fixture's two tests", res) + } + out, err := m.lintStep(context.Background(), map[string]any{"workspace": "ws1"}) + if la, _ := out.(buildstep.LintAnswer); err != nil || len(la.Failed) == 0 || la.Failed[0].Test != "gofmt" || !strings.Contains(la.Failed[0].Output, "ugly.go") { + t.Fatalf("lint = %+v, %v; want gofmt naming ugly.go", out, err) } } -- 2.54.0 From bb1e18f490fdfc4d067f1f8760d394c35e361517 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:33:05 +0100 Subject: [PATCH 036/243] K1: mesh-build-go reads no kept state of the checker; the step ask carries the kept results (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A test step runs only the packages with no kept result for their digest, and the runner makes that carry in the checker's place (ADR 0666 section 8; design 47 *Carried, not run* line 593 at hq b229f5c4): the kept results reach the step in its ask (StepRun.kept, mesh-delivery dc0df1b5), so the module's reads lose mesh-delivery-checker.kept and the lookup of the checker's kept bucket goes. The carried and kept packages are in the step's end, on its step key. Milestone 47/G, research 086 area C (K1). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 1c991294.) Builds: design 47 G §25, §27, §66, §416 (BLD5) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-build-go/cmd/mesh-build-go/asks.go | 36 ++----------------- .../cmd/mesh-build-go/asks_test.go | 7 ++-- .../mesh-build-go/cmd/mesh-build-go/main.go | 1 - .../mesh-build-go/cmd/mesh-build-go/module.go | 3 +- modules/mesh-build-go/module.json | 3 -- 5 files changed, 7 insertions(+), 43 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 6d4430e1f..463712f46 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -2,13 +2,10 @@ package main import ( "context" - "encoding/json" - "fmt" "path/filepath" "time" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" - checkerseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdeliverychecker" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" buildrun "git.novox.be/novox/mesh-delivery/sdk/buildrun" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" @@ -24,13 +21,6 @@ import ( // Seat is the node seat this module holds. const Seat = "node-build-go" -// KeptState is the checker's kept verdicts, which a check ask takes for the packages whose digest has one (issue -// 1887's design limit 1). -const KeptState = checkerModule + "." + checkerseat.StateKept - -// checkerModule is the module that holds the seat mesh-delivery-checker, whose kept state a check ask reads. -const checkerModule = "mesh-delivery-checker" - // askSteps are the module's step bodies for an ask, each holding its workspace while it runs, as a tool's does. func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { byName := map[string]buildstep.StepFunc{} @@ -52,7 +42,8 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { } return sdkstep.ToolchainUsed{Version: installed, Recipe: m.Recipe.Version()}, nil }, - Kept: m.KeptLookup, + // The kept results reach a test step in its ask (design 47 *Carried, not run*, line 593 at b229f5c4; issue 1966 + // K1): this module reads no other part's kept state. Has: m.Has, Release: func(ws string) error { return m.spaces().Release(ws) }, Logf: m.Say, @@ -62,29 +53,6 @@ func (m *Module) askSteps(steps []StepTool) buildstep.AskSteps { } } -// keptValue reads one key of the checker's kept state; nil when it is absent. Main sets it to the state's read; a test -// its own. -func (m *Module) keptValue(key string) (json.RawMessage, error) { - if m.ReadKept == nil { - return nil, nil - } - return m.ReadKept(key) -} - -// KeptLookup is whether the checker kept a verdict for the level and digest, and how many tests it counted. -func (m *Module) KeptLookup(ctx context.Context, level, digest string) (int, bool, error) { - key := checkerseat.KeptByLevelDigestKey(level, digest) - raw, err := m.keptValue(key) - if err != nil || raw == nil { - return 0, false, err - } - v, err := checkerseat.DecodeKeptByLevelDigestValue(raw) - if err != nil { - return 0, false, fmt.Errorf("%s holds what is not a kept verdict: %w", key, err) - } - return int(v.Tests), true, nil -} - // Puller is the module's pull of its seat's asks, on source (the runtime's seat pull when nil). func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep.Puller { if source == nil { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index 810569bb7..dbc73c839 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -114,15 +114,16 @@ func TestAtCapacityZeroNothingIsPulledAndTheToolsAnswer(t *testing.T) { func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } -// 1887/A3go, A7: the manifest claims node-build-go serving its three verbs, reads the checker's kept verdicts and takes +// 1887/A3go, A7: the manifest claims node-build-go serving its three verbs, reads no other part's state (issue 1966 K1: the kept +// results come in the step ask) and takes // the node's build capacity as a setting, with the builder's default; the seat's verbs are served as its tools. func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { m := readManifest(t) if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { t.Fatalf("claims = %+v", m.Claims) } - if !slices.Contains(m.Reads, KeptState) { - t.Errorf("reads = %v lack %s", m.Reads, KeptState) + if len(m.Reads) != 0 { + t.Errorf("reads = %v, want none: the kept results come in the step ask", m.Reads) } raw, _ := os.ReadFile(moduleDir + "/module.json") var doc struct { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index bda976699..872aabacd 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -61,7 +61,6 @@ func main() { m.Run = StepRunner(m.Home) m.GoSeatFile = m.Home + "/go-package-registry.json" m.Settings = m.Home + "/settings.json" - m.ReadKept = readOf(stdio.State(KeptState)).Get m.ReadManifest = askManifest m.AskInput = stdio.Ask m.Probe = ProbeGo(m.Home, m.Cache) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index b5c99cecf..39d06d3d8 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -98,10 +98,9 @@ type Module struct { Floor buildstep.Floor // Settings is the module's settings file, which holds the node's build capacity (sdkstep.CapacitySetting); Capacity, - // when set, answers it instead (a test). ReadKept reads a key of the checker's kept state (KeptState). + // when set, answers it instead (a test). Settings string Capacity func() int - ReadKept func(key string) (json.RawMessage, error) // SayStep says a pipeline step's end (buildstep.AskSteps.SayStep); the seat's publish on this node when nil. SayStep func(sdkstep.StepEnded) error // ReadManifest asks the delivery indexer's read tool (ManifestsTool) for one manifest by its key, which a compose ask's diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index c513d1438..5e3382040 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -27,9 +27,6 @@ "consumes": [ "mesh-conditions.findings-asked" ], - "reads": [ - "mesh-delivery-checker.kept" - ], "invokes": [ "mesh-delivery-indexer.indexer_read_manifests", "seat:oci-artifact-registry.publish", -- 2.54.0 From cf383af4417911283e81b581fdb31f4aa6ae2f9c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:34:53 +0100 Subject: [PATCH 037/243] W12.5 (Go): mesh-build-go keeps the store holder's refused build and serves kept-artifact (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds design 47 *The artifact store's holder takes its own new build even when its seat is the one broken* (lines ~1811-1830, test 60): a trunk build of a module claiming oci-artifact-registry that the seat refuses or does not answer is kept as an object by digest and answered kept (BuildAnswer.Kept); kept-artifact answers its store and object, refusing a module that claims no oci-artifact-registry, one not kept here and one under another digest. The store is the template objects (ADR 0666 §4: a part outside the foundation keeps its data in ADR 0595's templates; design 47's "old objects form" contradicts it for a non-foundation part, reported). Whether a module claims the seat is asked of the seat mesh-catalog's verb module (ADR 0596). Callers are left to the seat definition and the grants part (ADR 0666 §20). build_go_kept reads the kept builds, build_go_kept_drop drops one (ADR 0666 §25). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 5e1cdade.) Builds: design 47 G §271, test 60 (BLD9) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-build-go/buildstep/askrun.go | 8 + .../cmd/mesh-build-go/archive.go | 10 +- .../mesh-build-go/cmd/mesh-build-go/asks.go | 2 +- .../cmd/mesh-build-go/asks_test.go | 2 +- .../mesh-build-go/cmd/mesh-build-go/build.go | 24 +- .../mesh-build-go/cmd/mesh-build-go/fetch.go | 1 + .../cmd/mesh-build-go/forge_test.go | 4 +- .../mesh-build-go/cmd/mesh-build-go/kept.go | 24 +- .../cmd/mesh-build-go/keptbuilds.go | 214 ++++++++++++++++++ .../cmd/mesh-build-go/keptbuilds_test.go | 120 ++++++++++ .../mesh-build-go/cmd/mesh-build-go/main.go | 3 + .../mesh-build-go/cmd/mesh-build-go/module.go | 4 + modules/mesh-build-go/module.json | 6 +- 13 files changed, 400 insertions(+), 22 deletions(-) create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go create mode 100644 modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go diff --git a/modules/mesh-build-go/buildstep/askrun.go b/modules/mesh-build-go/buildstep/askrun.go index 050a0b046..49859b27a 100644 --- a/modules/mesh-build-go/buildstep/askrun.go +++ b/modules/mesh-build-go/buildstep/askrun.go @@ -230,6 +230,10 @@ func (s AskSteps) build(ctx context.Context, job sdkstep.BuildJob) (any, error) if job.Publish { args["publish"] = true } + // The fingerprint a build kept for the artifact store's holder is asked by (kept-artifact, milestone 47/G W12.5). + if job.Fp != "" { + args["fp"] = job.Fp + } // The seat it is published through and the version the builder gave it (design 51 *Publishing*, CAT-BG2). if a.Seat != "" { args["seat"] = a.Seat @@ -254,11 +258,15 @@ func (s AskSteps) build(ctx context.Context, job sdkstep.BuildJob) (any, error) var built struct { Digest string `json:"digest"` Reference string `json:"reference"` + Kept string `json:"kept"` } if err := json.Unmarshal(r, &built); err != nil || built.Digest == "" { return nil, fmt.Errorf("the build of %s answered no digest: %s", a.Artifact, r) } answer.Artifacts = append(answer.Artifacts, sdkstep.AskBuilt{Artifact: a.Artifact, Digest: built.Digest, Reference: built.Reference}) + if built.Kept != "" { + answer.Kept = strings.TrimPrefix(built.Kept, "sha256:") + } } if s.Imports != nil && f.Workspace != "" { out, err := s.Imports(ctx, map[string]any{"workspace": f.Workspace, "module": job.Path}) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/archive.go b/modules/mesh-build-go/cmd/mesh-build-go/archive.go index b6b20191a..f7073418e 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/archive.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/archive.go @@ -45,9 +45,9 @@ func (m *Module) httpClient() *http.Client { // buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory // `from` of the module's tree. -func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact) (any, error) { +func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact, keep *keptFor) (any, error) { if art.URL != "" { - return m.pinnedArchive(ctx, workspace, module, name, art) + return m.pinnedArchive(ctx, workspace, module, name, art, keep) } from := strings.Trim(path.Clean("/"+filepath.ToSlash(art.From)), "/") if strings.TrimSpace(art.From) == "" { @@ -79,12 +79,12 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir if !within(realDir, realRoot) { return nil, fmt.Errorf("the archive %s is taken from %q, which lies outside the module: it resolves to %s, and the module is %s", art.Name, art.From, realRoot, realDir) } - return m.packInto(ctx, workspace, module, name, art.Name, "archive", root) + return m.packInto(ctx, workspace, module, name, art.Name, "archive", root, keep) } // pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says // (kept whole as `file`, or unpacked from `within`) and packs it. -func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact) (any, error) { +func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact, keep *keptFor) (any, error) { file, err := m.pinnedFile(ctx, workspace, art) if err != nil { return nil, fmt.Errorf("%s: %w", art.Name, err) @@ -113,7 +113,7 @@ func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name stri } else if err := unpackUpstream(file, laid, art.Within); err != nil { return nil, fmt.Errorf("%s: %s (%s): %w", art.Name, art.URL, art.Version, err) } - return m.packInto(ctx, workspace, module, name, art.Name, "archive", laid) + return m.packInto(ctx, workspace, module, name, art.Name, "archive", laid, keep) } // pinnedFile is the path of the pinned file in the workspace: kept from an earlier build when its bytes are still the diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks.go b/modules/mesh-build-go/cmd/mesh-build-go/asks.go index 463712f46..b49bd7d1b 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks.go @@ -105,7 +105,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // SeatTools are the seat's verbs on this node (ADR 0525 §10): what it runs, and its pause and resume. func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool { - return append(buildstep.SeatTools(Seat, "Go", p), m.stepsTool()) + return append(buildstep.SeatTools(Seat, "Go", p), m.stepsTool(), m.keptArtifactVerb()) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go index dbc73c839..d72afeaab 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/asks_test.go @@ -119,7 +119,7 @@ func mustJSON(v any) []byte { b, _ := json.Marshal(v); return b } // the node's build capacity as a setting, with the builder's default; the seat's verbs are served as its tools. func TestManifestClaimsTheGoBuildSeatAndReadsItsCapacity(t *testing.T) { m := readManifest(t) - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact"}) { t.Fatalf("claims = %+v", m.Claims) } if len(m.Reads) != 0 { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index 6a202317f..7ad77cc7c 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -39,6 +39,9 @@ type BuildResult struct { // §9): oci-artifact-registry//@. The builder writes the older form beside it while a // counted node-engine reads only that (ADR 0624 §1, §3). Reference string `json:"reference"` + // Kept is the archive's digest when the seat did not store it and this node kept it for the artifact store's holder + // (keptbuilds.go); Reference is then empty. + Kept string `json:"kept,omitempty"` } func (m *Module) buildTool() StepTool { @@ -116,12 +119,14 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { } // The version the build ask carries is the build's argument MESH_VERSION for every artifact (design 47 G §370). version, _ := args["version"].(string) + fp, _ := args["fp"].(string) + keep := keptOf(publish, manifest, fp) if art.Kind == catalogue.ArtifactPackage && art.Language == "go" { seat, _ := args["seat"].(string) return m.buildGoPackage(ctx, workspace, dir, *art, version, seat) } if art.Kind == "archive" { - return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art) + return m.buildArchive(ctx, workspace, module, manifest.Module, dir, *art, keep) } if art.Kind != "bundle" || art.Language != "go" { return nil, fmt.Errorf("the artifact %s of %s is a %s of language %q: this module builds Go bundles and archives", name, manifest.Module, art.Kind, art.Language) @@ -164,7 +169,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { if err := m.insideModule(workspace, dir, out); err != nil { return nil, fmt.Errorf("the output directory of %s: %w", art.Name, err) } - res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, "bundle", out) + res, err := m.packInto(ctx, workspace, module, manifest.Module, art.Name, "bundle", out, keep) if err != nil { return nil, err } @@ -175,15 +180,22 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { // packInto packs a directory with mesh-sdk's Pack, as the build agent packs every artifact, and publishes the archive // on the seat oci-artifact-registry from this process, after the build step ended (design 51 *Publishing*; publish.go): // a candidate's as a trunk build's, under / by its digest, of kind (bundle or archive). The step -// answers the reference; a publish the seat refuses fails the step `not stored:`. The packed bytes never stay on the -// node: they are held in this process until the publish answered (the lead's decision L9). -func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string) (BuildResult, error) { +// answers the reference; a publish the seat refuses fails the step `not stored:`, unless keep names the build of the +// artifact store's holder, which this node then keeps and answers as kept (keptbuilds.go). The packed bytes never stay on +// the node: they are held in this process until the publish answered (the lead's decision L9). +func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string, keep *keptFor) (BuildResult, error) { archive, digest, err := buildstep.Pack(root) if err != nil { return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err) } if err := m.publishArchive(ctx, name, artifact, kind, digest, archive); err != nil { - return BuildResult{}, fmt.Errorf("%s%s: %w", builds.NotStored, artifact, err) + if keep == nil { + return BuildResult{}, fmt.Errorf("%s%s: %w", builds.NotStored, artifact, err) + } + if kerr := m.keepBuild(*keep, digest, archive); kerr != nil { + return BuildResult{}, fmt.Errorf("%s%s: %w; and it could not be kept: %v", builds.NotStored, artifact, err, kerr) + } + return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Kept: digest}, nil } return BuildResult{Artifact: artifact, Digest: digest, Size: len(archive), Reference: artifactref.SeatReference(ArtifactSeat, name, artifact, digest)}, nil } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go index 81e92c75b..8e76eb0f6 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/fetch.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/fetch.go @@ -433,6 +433,7 @@ func (m *Module) goEnvFor(dir string) []string { func (m *Module) CleanWorkspaces(now time.Time) { m.sweepSaid(m.spaces().Sweep(now, buildstep.UnheldBound)) m.SweepKept(now, KeptBound) + m.sweepKeptBuilds(now, KeptBound) } // clock is the time now: Now when a test set it, else the wall clock. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go index bd3fa0e36..d9dab2fd4 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/forge_test.go @@ -148,8 +148,10 @@ func TestManifestRequiresForgeAccountKeepsKeysAndTakesFilesFromTemplates(t *test // node-tools: the module pulls its asks through the tool runner's seat pull (1911). // go-package-registry: the Go seat's read side, the fetch's first source (design 51, CAT-BG3, E2). // detected-steps: the state of the verb steps (design 47 G §326, §414), asked by the template state. + // kept-builds: the builds kept for the artifact store's holder (keptbuilds.go), by the template objects (W12.5). + kept := map[string]any{"name": "kept-builds", "storage": "object-store", "template": "objects", "options": map[string]any{"max-mib": float64(1024)}} detected := map[string]any{"name": "detected-steps", "storage": "key-value", "template": "state", "options": map[string]any{"history": float64(1)}} - if want := []any{detected, "forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { + if want := []any{detected, kept, "forge-account", "go-toolchain", "go-package-registry", "node-tools"}; !reflect.DeepEqual(m.Requires, want) { t.Errorf("requires = %v, want %v", m.Requires, want) } keeps := false diff --git a/modules/mesh-build-go/cmd/mesh-build-go/kept.go b/modules/mesh-build-go/cmd/mesh-build-go/kept.go index 7e6bc2344..f2162408d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/kept.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/kept.go @@ -109,19 +109,31 @@ func (m *Module) keptTools() []stdio.Tool { {Name: KeptTool, Description: "The programs this node keeps, built from a commit so each commit is built once: the composers of compose asks and " + "the checking programs of run asks, each with its kind, key, size in bytes and when an ask last used it. One no ask used " + - "for 30 days is removed. Read-only.", - Run: func(map[string]any) (any, error) { return map[string]any{"kept": m.Kept()}, nil }}, + "for 30 days is removed. Under builds, the builds it keeps for the artifact store's holder that the seat did not store " + + "(served by kept-artifact): each one's module, fingerprint, digest, size and when it was kept; one is removed after 30 days. Read-only.", + Run: func(map[string]any) (any, error) { + builds, err := m.keptBuilds() + if err != nil { + return nil, err + } + return map[string]any{"kept": m.Kept(), "builds": builds}, nil + }}, {Name: KeptDropTool, Description: "Remove one kept program by its kind (composers or programs) and key, as build_go_kept names them: the next ask " + - "for that commit builds it again. One already gone is no error.", + "for that commit builds it again. With the kind builds and a build's digest as its key, remove a build kept for the " + + "artifact store's holder: its want then waits for the next build. One already gone is no error.", Input: map[string]any{"type": "object", "properties": map[string]any{ - "kind": map[string]any{"type": "string", "enum": []string{"composers", "programs"}}, - "key": map[string]any{"type": "string", "description": "the key build_go_kept names"}, + "kind": map[string]any{"type": "string", "enum": []string{"composers", "programs", keptBuildsKind}}, + "key": map[string]any{"type": "string", "description": "the key build_go_kept names (a kept build's digest)"}, }, "required": []string{"kind", "key"}}, Run: func(args map[string]any) (any, error) { kind, _ := args["kind"].(string) key, _ := args["key"].(string) - if err := m.DropKept(kind, key); err != nil { + drop := m.DropKept + if kind == keptBuildsKind { + drop = func(_, key string) error { return m.dropKeptBuild(key) } + } + if err := drop(kind, key); err != nil { return nil, err } return map[string]any{"dropped": key, "kind": kind}, nil diff --git a/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go new file mode 100644 index 000000000..581bb18be --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go @@ -0,0 +1,214 @@ +package main + +// **The artifact store's holder takes its own new build even when its seat is the one broken** (novox/hq design 47; test +// 60; milestone 47/G W12.5, issue 1966): when the seat oci-artifact-registry refuses or does not answer the store of a +// trunk build of a module that claims that seat, this module keeps that build on the bus instead, in its object store +// `kept-builds` (asked for in `requires` by the template objects: ADR 0666 §4 keeps a part outside the foundation in +// ADR 0595's templates), one object per build named by its digest; the build's answer says it kept it (BuildAnswer.Kept), +// so the step key and the builder's done key name it. The node-engine composing the holder asks the seat's verb +// kept-artifact of the keeping node for where the bytes are, and reads them from the store itself. The kept builds are +// read by build_go_kept, dropped one by one by build_go_kept_drop, and swept with the kept programs after KeptBound. +// Who may call the verb is the seat definition's (the node-engines alone), granted by the grants part from it (ADR 0666 +// §20): this module checks no caller of its own. + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// KeptBuildsStore is the local name of the object store of kept builds, as the manifest's `requires` asks for it. +const KeptBuildsStore = "kept-builds" + +// keptBuildsKind is the kind build_go_kept and build_go_kept_drop name a kept build by, beside composers and programs. +const keptBuildsKind = "builds" + +// KeptObjects is the object store of kept builds as the module uses it (busstate.ObjectsOf(KeptBuildsStore)). +type KeptObjects interface { + Put(name string, data []byte, description string) (buscalls.ObjectInfo, error) + List() ([]buscalls.ObjectInfo, error) + Delete(name string) error +} + +// keptFor is what a build keeps a refused artifact for: the module, which claims the store's seat, and its build's +// fingerprint, which kept-artifact is asked by. +type keptFor struct{ Module, Fp string } + +// keptOf is the keep of a build, or nil when nothing may be kept: only a trunk build (publish) of a module whose +// manifest at the built commit claims oci-artifact-registry, named by its fingerprint. +func keptOf(publish bool, manifest catalogue.Manifest, fp string) *keptFor { + if !publish || fp == "" || !manifest.ClaimsSeat(ociseat.Seat) { + return nil + } + return &keptFor{Module: manifest.Module, Fp: fp} +} + +// KeptBuild is one kept build: the object's description names its module, fingerprint and the node that kept it. +type KeptBuild struct { + Module string `json:"module"` + Fp string `json:"fp"` + Node string `json:"node"` + Digest string `json:"digest"` + Bytes uint64 `json:"bytes"` + LastUsed time.Time `json:"last-used"` +} + +// keepBuild keeps the archive of digest ("sha256:") for k on this node; the object is named by the hex digest. +func (m *Module) keepBuild(k keptFor, digest string, archive []byte) error { + if m.KeptBuilds == nil { + return fmt.Errorf("this module keeps no builds (no %s store)", KeptBuildsStore) + } + hex := strings.TrimPrefix(digest, "sha256:") + desc, _ := json.Marshal(KeptBuild{Module: k.Module, Fp: k.Fp, Node: m.Node}) + if _, err := m.KeptBuilds.Put(hex, archive, string(desc)); err != nil { + return fmt.Errorf("keeping the build of %s on %s: %w", k.Module, KeptBuildsStore, err) + } + if m.Say != nil { + m.Say("kept the build %s of %s (%s) on this node: the seat %s did not store it", hex, k.Module, k.Fp, ociseat.Seat) + } + return nil +} + +// keptBuilds are the builds this node keeps, by the store's objects that name it. +func (m *Module) keptBuilds() ([]KeptBuild, error) { + if m.KeptBuilds == nil { + return []KeptBuild{}, nil + } + infos, err := m.KeptBuilds.List() + if err != nil { + return nil, err + } + out := []KeptBuild{} + for _, i := range infos { + var k KeptBuild + if i.Deleted || json.Unmarshal([]byte(i.Description), &k) != nil || k.Node != m.Node { + continue + } + k.Digest, k.Bytes, k.LastUsed = i.Name, i.Size, i.Modified.UTC() + out = append(out, k) + } + slices.SortFunc(out, func(a, b KeptBuild) int { return strings.Compare(a.Digest, b.Digest) }) + return out, nil +} + +// dropKeptBuild removes the kept build of the hex digest this node keeps; one it does not keep is no error. +func (m *Module) dropKeptBuild(hex string) error { + kept, err := m.keptBuilds() + if err != nil { + return err + } + if !slices.ContainsFunc(kept, func(k KeptBuild) bool { return k.Digest == hex }) { + return nil + } + return m.KeptBuilds.Delete(hex) +} + +// sweepKeptBuilds drops the kept builds older than bound before now: the want of one waits, and the node-engine says it. +func (m *Module) sweepKeptBuilds(now time.Time, bound time.Duration) { + kept, err := m.keptBuilds() + if err != nil { + return + } + for _, k := range kept { + if now.Sub(k.LastUsed) > bound && m.dropKeptBuild(k.Digest) == nil && m.Say != nil { + m.Say("removed the kept build %s of %s, kept for %s", k.Digest, k.Module, bound) + } + } +} + +// claimsStore asks the seat mesh-catalog whether module's manifest claims oci-artifact-registry. +func (m *Module) claimsStore(module string) (bool, error) { + ask := m.AskInput + if ask == nil { + ask = stdio.Ask + } + raw, err := ask(CatalogModuleVerb, map[string]any{"module": module}) + if err != nil { + return false, err + } + var a struct { + Module struct { + Manifest json.RawMessage `json:"manifest"` + } `json:"module"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return false, fmt.Errorf("%s answered what is no module: %w", CatalogModuleVerb, err) + } + if len(a.Module.Manifest) == 0 { + return false, nil + } + man, err := catalogue.Decode(a.Module.Manifest) + if err != nil { + return false, fmt.Errorf("the manifest of %s: %w", module, err) + } + return man.ClaimsSeat(ociseat.Seat), nil +} + +// CatalogModuleVerb is the seat mesh-catalog's verb that answers one module's manifest, as the manifest invokes it. +const CatalogModuleVerb = "seat:mesh-catalog.module" + +// keptArtifact is the verb kept-artifact's answer for in. +func (m *Module) keptArtifact(in seat.KeptArtifactInput) seat.KeptArtifactAnswer { + a := seat.KeptArtifactAnswer{Module: in.Module, Fp: in.Fp} + kept, err := m.keptBuilds() + if err != nil { + a.Error = "reading the kept builds: " + err.Error() + return a + } + var other *KeptBuild + for i, k := range kept { + if k.Module != in.Module || k.Fp != in.Fp { + continue + } + if k.Digest == in.Digest { + a.Digest, a.Size, a.Object = k.Digest, int64(k.Bytes), k.Digest + if m.Self == "" { + a.Error = "this module runs with no name of its own (MESH_MODULE), so its store's name on the bus is unknown" + return a + } + a.Store = catalogue.ObjectsBucket(m.Self, KeptBuildsStore) + return a + } + other = &kept[i] + } + claims, err := m.claimsStore(in.Module) + switch { + case err != nil: + a.Error = "asking whether " + in.Module + " claims " + ociseat.Seat + ": " + err.Error() + case !claims: + a.Refused = seat.KeptArtifactAnswerRefusedClaimsNoOciArtifactRegistry + a.Words = fmt.Sprintf("%s claims no %s: a build node keeps builds for that seat's holder alone", in.Module, ociseat.Seat) + case other != nil: + a.Refused, a.Digest = seat.KeptArtifactAnswerRefusedDigestDiffers, other.Digest + a.Words = fmt.Sprintf("this node keeps the build of %s at %s under the digest %s, not %s", in.Module, in.Fp, other.Digest, in.Digest) + default: + a.Refused = seat.KeptArtifactAnswerRefusedNotKept + a.Words = fmt.Sprintf("this node keeps no build of %s at %s (never kept, or swept)", in.Module, in.Fp) + } + return a +} + +// keptArtifactVerb serves `.kept-artifact`. +func (m *Module) keptArtifactVerb() stdio.Tool { + return stdio.Tool{Name: Seat + "." + seat.VerbKeptArtifact, + Description: "Where the build this node kept for the artifact store's holder is: the new build of the holder of " + + ociseat.Seat + " that the seat refused or did not store, as an object of this module's store of kept builds, by its " + + "digest. Refuses a module that claims no " + ociseat.Seat + ", a build not kept here (never kept, or swept) and one " + + "kept under another digest, naming the module.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "digest"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module, the holder of " + ociseat.Seat}, + "fp": map[string]any{"type": "string", "description": "its build's fingerprint"}, + "digest": map[string]any{"type": "string", "description": "the artifact's sha256, 64 lowercase hex digits"}}}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + return m.keptArtifact(seat.KeptArtifactInput{Module: str("module"), Fp: str("fp"), Digest: str("digest")}), nil + }} +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go new file mode 100644 index 000000000..e6de5202b --- /dev/null +++ b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go @@ -0,0 +1,120 @@ +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// memObjects is an object store in memory, as the tool runner keeps one for the module. +type memObjects struct { + objects map[string]buscalls.ObjectInfo +} + +func (o *memObjects) Put(name string, data []byte, description string) (buscalls.ObjectInfo, error) { + i := buscalls.ObjectInfo{Name: name, Size: uint64(len(data)), Description: description, Modified: time.Now()} + o.objects[name] = i + return i, nil +} +func (o *memObjects) List() ([]buscalls.ObjectInfo, error) { + out := []buscalls.ObjectInfo{} + for _, i := range o.objects { + out = append(out, i) + } + return out, nil +} +func (o *memObjects) Delete(name string) error { delete(o.objects, name); return nil } + +// Design 47 *The artifact store's holder takes its own new build even when its seat is the one broken* (test 60, W12.5): +// a trunk build of a module claiming oci-artifact-registry that the seat refuses is kept on this node and answered kept; +// any other build the seat refuses fails not stored; kept-artifact answers the store and object of a kept build, refuses +// a module that claims no oci-artifact-registry, one not kept here and one kept under another digest, and a drop or the +// sweep removes it. +func TestTheStoreHoldersRefusedBuildIsKeptAndServedByKeptArtifact(t *testing.T) { + holder := `{"module":"demo","version":"1","claims":[{"name":"oci-artifact-registry","scope":"mesh"}],"build":{"artifacts":[` + goBundle + `]}}` + refuse := func(key string, _ any) (json.RawMessage, error) { + return nil, errors.New("refused: the store did not take it") + } + for _, c := range []struct { + name, manifest string + publish bool + kept bool + }{ + {"the store holder's trunk build", holder, true, true}, + {"another module's trunk build", manifestWith(goBundle), true, false}, + {"the store holder's candidate build", holder, false, false}, + } { + m, states, _ := buildModule(t, c.manifest, "the program") + objects := &memObjects{objects: map[string]buscalls.ObjectInfo{}} + m.KeptBuilds, m.Self, m.Node = objects, "mesh-build-go", "ace" + m.Call = refuse + args := map[string]any{"workspace": "ws1", "module": "modules/demo", "artifact": "tools", "fp": "src2:" + strings.Repeat("a", 64)} + if c.publish { + args["publish"] = true + } + for _, tool := range m.Tools(m.StepTools()) { + if tool.Name == "build_go_build" { + if _, err := tool.Run(args); err != nil { + t.Fatal(err) + } + } + } + st := states.waitEnd(t) + var res BuildResult + _ = json.Unmarshal(st.Result, &res) + if got := st.State == sdkstep.StepPassed && res.Kept != "" && len(objects.objects) == 1; got != c.kept { + t.Fatalf("%s: kept %v (step %s %q, kept %q, %d objects), want %v", c.name, got, st.State, st.Reason, res.Kept, len(objects.objects), c.kept) + } + if !c.kept { + continue + } + hex := strings.TrimPrefix(res.Kept, "sha256:") + m.AskInput = func(key string, body any) (json.RawMessage, error) { + if key != CatalogModuleVerb { + return nil, errors.New("asked " + key) + } + mod, _ := body.(map[string]any)["module"].(string) + man := `{"module":"` + mod + `"}` + if mod == "demo" || mod == "lost" { + man = `{"module":"` + mod + `","claims":[{"name":"oci-artifact-registry","scope":"mesh"}]}` + } + return json.RawMessage(`{"module":{"module":"` + mod + `","manifest":` + man + `}}`), nil + } + fp := args["fp"].(string) + for _, a := range []struct { + name string + in seat.KeptArtifactInput + refused, store string + }{ + {"the kept build", seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: hex}, "", "mesh-build-go_kept-builds"}, + {"another digest", seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: strings.Repeat("b", 64)}, seat.KeptArtifactAnswerRefusedDigestDiffers, ""}, + {"a module claiming no store", seat.KeptArtifactInput{Module: "other", Fp: fp, Digest: hex}, seat.KeptArtifactAnswerRefusedClaimsNoOciArtifactRegistry, ""}, + {"a holder not kept here", seat.KeptArtifactInput{Module: "lost", Fp: fp, Digest: hex}, seat.KeptArtifactAnswerRefusedNotKept, ""}, + } { + got := m.keptArtifact(a.in) + if got.Refused != a.refused || got.Store != a.store || (a.refused == "" && (got.Object != hex || got.Size == 0)) || + (a.refused != "" && !strings.Contains(got.Words, a.in.Module)) { + t.Errorf("%s: %+v", a.name, got) + } + } + // Another node's build in the shared store is not this node's to answer. + m.Node = "g14" + if got := m.keptArtifact(seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: hex}); got.Refused != seat.KeptArtifactAnswerRefusedNotKept { + t.Errorf("another node's kept build answered %+v", got) + } + m.Node = "ace" + m.sweepKeptBuilds(time.Now(), time.Hour) + if len(objects.objects) != 1 { + t.Errorf("a build kept within the bound was swept") + } + if err := m.dropKeptBuild(hex); err != nil || len(objects.objects) != 0 { + t.Errorf("drop: %v, %d objects left", err, len(objects.objects)) + } + } +} diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 872aabacd..317911181 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -16,6 +16,7 @@ import ( seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/busstate" ) // The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module @@ -63,6 +64,8 @@ func main() { m.Settings = m.Home + "/settings.json" m.ReadManifest = askManifest m.AskInput = stdio.Ask + m.KeptBuilds = busstate.ObjectsOf(KeptBuildsStore) + m.Self = strings.TrimSpace(os.Getenv("MESH_MODULE")) m.Probe = ProbeGo(m.Home, m.Cache) m.Bubblewrap = ProbeBubblewrap // Outside a launch by the runtime — a build, a check — there is no node and no state to write. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/module.go b/modules/mesh-build-go/cmd/mesh-build-go/module.go index 39d06d3d8..26632aed3 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/module.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/module.go @@ -112,6 +112,10 @@ type Module struct { // PlaceHead, when set, lays a run's head at a directory instead of fetching it (a test). PlaceHead func(ctx context.Context, h sdkstep.RunHead, dir string) error GoModCache string + // KeptBuilds is the object store of the builds kept for the artifact store's holder (keptbuilds.go); nil keeps none. + // Self is this module's name as the runtime launched it (MESH_MODULE), the first part of the store's name on the bus. + KeptBuilds KeptObjects + Self string spacesOnce sync.Once workspaces *buildstep.Workspaces diff --git a/modules/mesh-build-go/module.json b/modules/mesh-build-go/module.json index 5e3382040..e1cbbdd5e 100644 --- a/modules/mesh-build-go/module.json +++ b/modules/mesh-build-go/module.json @@ -21,7 +21,7 @@ { "name": "node-build-go", "scope": "node", - "serves": ["current", "drop", "pause", "resume", "steps"] + "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact"] } ], "consumes": [ @@ -36,7 +36,8 @@ "seat:mesh-delivery-checker.input", "seat:mesh-catalog.input", "seat:mesh-wants.input", - "seat:mesh-module-assigner.assignments" + "seat:mesh-module-assigner.assignments", + "seat:mesh-catalog.module" ], "config": { "capacity": { @@ -88,6 +89,7 @@ ], "requires": [ {"name": "detected-steps", "storage": "key-value", "template": "state", "options": {"history": 1}}, + {"name": "kept-builds", "storage": "object-store", "template": "objects", "options": {"max-mib": 1024}}, "forge-account", "go-toolchain", "go-package-registry", -- 2.54.0 From d3781fd840d6a720e9e04085790b0530f353ccd2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:35:53 +0100 Subject: [PATCH 038/243] W12.5: the three build modules answer kept-artifact with the domain SDK's judgement (issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 47 lines 1888-1906 at b229f5c4 (test 60): the verb sits in every build seat's definition. Its judgement (what may be kept, the kept builds a node holds, the three refusals) is shared code of one domain's modules, so it is the domain SDK's (sdk/buildstep KeptArtifact, ADR 0634 §2): mesh-build-go keeps its store, sweep and tools and drops its own copy of the judgement and its table; mesh-build-node and mesh-build-image keep no build and answer over none, so a module claiming no oci-artifact-registry is refused and the holder is answered not kept. Their manifests serve the verb and invoke seat:mesh-catalog.module, which the judgement asks. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 42d364fe.) Builds: design 47 G §271, §464 (BLD9) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-build-go/archive.go | 5 +- .../mesh-build-go/cmd/mesh-build-go/build.go | 5 +- .../cmd/mesh-build-go/keptbuilds.go | 110 +++--------------- .../cmd/mesh-build-go/keptbuilds_test.go | 41 +++---- .../cmd/mesh-build-image/asks.go | 2 +- .../cmd/mesh-build-image/asks_test.go | 28 ++++- .../cmd/mesh-build-image/compose_test.go | 2 +- .../cmd/mesh-build-image/keptartifact.go | 42 +++++++ .../cmd/mesh-build-image/manifest_test.go | 2 +- modules/mesh-build-image/module.json | 5 +- .../cmd/mesh-build-node/asks.go | 2 +- .../cmd/mesh-build-node/asks_test.go | 27 ++++- .../cmd/mesh-build-node/keptartifact.go | 45 +++++++ .../cmd/mesh-build-node/manifest_test.go | 2 +- modules/mesh-build-node/module.json | 5 +- 15 files changed, 183 insertions(+), 140 deletions(-) create mode 100644 modules/mesh-build-image/cmd/mesh-build-image/keptartifact.go create mode 100644 modules/mesh-build-node/cmd/mesh-build-node/keptartifact.go diff --git a/modules/mesh-build-go/cmd/mesh-build-go/archive.go b/modules/mesh-build-go/cmd/mesh-build-go/archive.go index f7073418e..819b6b839 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/archive.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/archive.go @@ -24,6 +24,7 @@ import ( "time" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/therootcompany/xz" @@ -45,7 +46,7 @@ func (m *Module) httpClient() *http.Client { // buildArchive packs one archive artifact of the module in dir: a pinned file when it names a url, else the directory // `from` of the module's tree. -func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact, keep *keptFor) (any, error) { +func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir string, art catalogue.Artifact, keep *sdkstep.KeptFor) (any, error) { if art.URL != "" { return m.pinnedArchive(ctx, workspace, module, name, art, keep) } @@ -84,7 +85,7 @@ func (m *Module) buildArchive(ctx context.Context, workspace, module, name, dir // pinnedArchive fetches the file an artifact is pinned to, checks it against the pin, lays it out as the artifact says // (kept whole as `file`, or unpacked from `within`) and packs it. -func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact, keep *keptFor) (any, error) { +func (m *Module) pinnedArchive(ctx context.Context, workspace, module, name string, art catalogue.Artifact, keep *sdkstep.KeptFor) (any, error) { file, err := m.pinnedFile(ctx, workspace, art) if err != nil { return nil, fmt.Errorf("%s: %w", art.Name, err) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/build.go b/modules/mesh-build-go/cmd/mesh-build-go/build.go index 7ad77cc7c..d42fd7dbf 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/build.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/build.go @@ -16,6 +16,7 @@ import ( "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -120,7 +121,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { // The version the build ask carries is the build's argument MESH_VERSION for every artifact (design 47 G §370). version, _ := args["version"].(string) fp, _ := args["fp"].(string) - keep := keptOf(publish, manifest, fp) + keep := sdkstep.KeepOf(publish, manifest, fp) if art.Kind == catalogue.ArtifactPackage && art.Language == "go" { seat, _ := args["seat"].(string) return m.buildGoPackage(ctx, workspace, dir, *art, version, seat) @@ -183,7 +184,7 @@ func (m *Module) build(ctx context.Context, args map[string]any) (any, error) { // answers the reference; a publish the seat refuses fails the step `not stored:`, unless keep names the build of the // artifact store's holder, which this node then keeps and answers as kept (keptbuilds.go). The packed bytes never stay on // the node: they are held in this process until the publish answered (the lead's decision L9). -func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string, keep *keptFor) (BuildResult, error) { +func (m *Module) packInto(ctx context.Context, workspace, module, name, artifact, kind, root string, keep *sdkstep.KeptFor) (BuildResult, error) { archive, digest, err := buildstep.Pack(root) if err != nil { return BuildResult{}, fmt.Errorf("packing %s: %w", artifact, err) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go index 581bb18be..bdb9800d9 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds.go @@ -12,14 +12,13 @@ package main // §20): this module checks no caller of its own. import ( - "encoding/json" "fmt" "slices" - "strings" "time" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildgo" ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -38,37 +37,16 @@ type KeptObjects interface { Delete(name string) error } -// keptFor is what a build keeps a refused artifact for: the module, which claims the store's seat, and its build's -// fingerprint, which kept-artifact is asked by. -type keptFor struct{ Module, Fp string } - -// keptOf is the keep of a build, or nil when nothing may be kept: only a trunk build (publish) of a module whose -// manifest at the built commit claims oci-artifact-registry, named by its fingerprint. -func keptOf(publish bool, manifest catalogue.Manifest, fp string) *keptFor { - if !publish || fp == "" || !manifest.ClaimsSeat(ociseat.Seat) { - return nil - } - return &keptFor{Module: manifest.Module, Fp: fp} -} - -// KeptBuild is one kept build: the object's description names its module, fingerprint and the node that kept it. -type KeptBuild struct { - Module string `json:"module"` - Fp string `json:"fp"` - Node string `json:"node"` - Digest string `json:"digest"` - Bytes uint64 `json:"bytes"` - LastUsed time.Time `json:"last-used"` -} +// KeptBuild is one kept build, the domain SDK's form (sdk/buildstep KeptBuild). +type KeptBuild = sdkstep.KeptBuild // keepBuild keeps the archive of digest ("sha256:") for k on this node; the object is named by the hex digest. -func (m *Module) keepBuild(k keptFor, digest string, archive []byte) error { +func (m *Module) keepBuild(k sdkstep.KeptFor, digest string, archive []byte) error { if m.KeptBuilds == nil { return fmt.Errorf("this module keeps no builds (no %s store)", KeptBuildsStore) } - hex := strings.TrimPrefix(digest, "sha256:") - desc, _ := json.Marshal(KeptBuild{Module: k.Module, Fp: k.Fp, Node: m.Node}) - if _, err := m.KeptBuilds.Put(hex, archive, string(desc)); err != nil { + hex, desc := sdkstep.KeptObject(k, m.Node, digest) + if _, err := m.KeptBuilds.Put(hex, archive, desc); err != nil { return fmt.Errorf("keeping the build of %s on %s: %w", k.Module, KeptBuildsStore, err) } if m.Say != nil { @@ -86,17 +64,7 @@ func (m *Module) keptBuilds() ([]KeptBuild, error) { if err != nil { return nil, err } - out := []KeptBuild{} - for _, i := range infos { - var k KeptBuild - if i.Deleted || json.Unmarshal([]byte(i.Description), &k) != nil || k.Node != m.Node { - continue - } - k.Digest, k.Bytes, k.LastUsed = i.Name, i.Size, i.Modified.UTC() - out = append(out, k) - } - slices.SortFunc(out, func(a, b KeptBuild) int { return strings.Compare(a.Digest, b.Digest) }) - return out, nil + return sdkstep.KeptBuildsOf(infos, m.Node), nil } // dropKeptBuild removes the kept build of the hex digest this node keeps; one it does not keep is no error. @@ -130,70 +98,24 @@ func (m *Module) claimsStore(module string) (bool, error) { if ask == nil { ask = stdio.Ask } - raw, err := ask(CatalogModuleVerb, map[string]any{"module": module}) + raw, err := ask(sdkstep.CatalogModuleVerb, map[string]any{"module": module}) if err != nil { return false, err } - var a struct { - Module struct { - Manifest json.RawMessage `json:"manifest"` - } `json:"module"` - } - if err := json.Unmarshal(raw, &a); err != nil { - return false, fmt.Errorf("%s answered what is no module: %w", CatalogModuleVerb, err) - } - if len(a.Module.Manifest) == 0 { - return false, nil - } - man, err := catalogue.Decode(a.Module.Manifest) - if err != nil { - return false, fmt.Errorf("the manifest of %s: %w", module, err) - } - return man.ClaimsSeat(ociseat.Seat), nil + return sdkstep.ClaimsStoreOf(module, raw) } -// CatalogModuleVerb is the seat mesh-catalog's verb that answers one module's manifest, as the manifest invokes it. -const CatalogModuleVerb = "seat:mesh-catalog.module" - -// keptArtifact is the verb kept-artifact's answer for in. -func (m *Module) keptArtifact(in seat.KeptArtifactInput) seat.KeptArtifactAnswer { - a := seat.KeptArtifactAnswer{Module: in.Module, Fp: in.Fp} +// keptArtifact is the verb kept-artifact's answer for in, the domain SDK's judgement over the builds this node keeps. +func (m *Module) keptArtifact(in seat.KeptArtifactInput) sdkstep.KeptArtifactAnswer { kept, err := m.keptBuilds() if err != nil { - a.Error = "reading the kept builds: " + err.Error() - return a + return sdkstep.KeptArtifactAnswer{Module: in.Module, Fp: in.Fp, Error: "reading the kept builds: " + err.Error()} } - var other *KeptBuild - for i, k := range kept { - if k.Module != in.Module || k.Fp != in.Fp { - continue - } - if k.Digest == in.Digest { - a.Digest, a.Size, a.Object = k.Digest, int64(k.Bytes), k.Digest - if m.Self == "" { - a.Error = "this module runs with no name of its own (MESH_MODULE), so its store's name on the bus is unknown" - return a - } - a.Store = catalogue.ObjectsBucket(m.Self, KeptBuildsStore) - return a - } - other = &kept[i] + store := "" + if m.Self != "" { + store = catalogue.ObjectsBucket(m.Self, KeptBuildsStore) } - claims, err := m.claimsStore(in.Module) - switch { - case err != nil: - a.Error = "asking whether " + in.Module + " claims " + ociseat.Seat + ": " + err.Error() - case !claims: - a.Refused = seat.KeptArtifactAnswerRefusedClaimsNoOciArtifactRegistry - a.Words = fmt.Sprintf("%s claims no %s: a build node keeps builds for that seat's holder alone", in.Module, ociseat.Seat) - case other != nil: - a.Refused, a.Digest = seat.KeptArtifactAnswerRefusedDigestDiffers, other.Digest - a.Words = fmt.Sprintf("this node keeps the build of %s at %s under the digest %s, not %s", in.Module, in.Fp, other.Digest, in.Digest) - default: - a.Refused = seat.KeptArtifactAnswerRefusedNotKept - a.Words = fmt.Sprintf("this node keeps no build of %s at %s (never kept, or swept)", in.Module, in.Fp) - } - return a + return sdkstep.KeptArtifact(in.Module, in.Fp, in.Digest, kept, store, m.claimsStore) } // keptArtifactVerb serves `.kept-artifact`. diff --git a/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go index e6de5202b..aff556c19 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/keptbuilds_test.go @@ -75,35 +75,26 @@ func TestTheStoreHoldersRefusedBuildIsKeptAndServedByKeptArtifact(t *testing.T) continue } hex := strings.TrimPrefix(res.Kept, "sha256:") - m.AskInput = func(key string, body any) (json.RawMessage, error) { - if key != CatalogModuleVerb { - return nil, errors.New("asked " + key) - } - mod, _ := body.(map[string]any)["module"].(string) - man := `{"module":"` + mod + `"}` - if mod == "demo" || mod == "lost" { - man = `{"module":"` + mod + `","claims":[{"name":"oci-artifact-registry","scope":"mesh"}]}` - } - return json.RawMessage(`{"module":{"module":"` + mod + `","manifest":` + man + `}}`), nil - } + // The answer's judgement is the domain SDK's (sdk/buildstep KeptArtifact, its own table); here only that the verb + // answers this node's kept build from this module's store, asking the seat mesh-catalog nothing for it. + m.AskInput = func(key string, _ any) (json.RawMessage, error) { return nil, errors.New("asked " + key) } fp := args["fp"].(string) - for _, a := range []struct { - name string - in seat.KeptArtifactInput - refused, store string - }{ - {"the kept build", seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: hex}, "", "mesh-build-go_kept-builds"}, - {"another digest", seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: strings.Repeat("b", 64)}, seat.KeptArtifactAnswerRefusedDigestDiffers, ""}, - {"a module claiming no store", seat.KeptArtifactInput{Module: "other", Fp: fp, Digest: hex}, seat.KeptArtifactAnswerRefusedClaimsNoOciArtifactRegistry, ""}, - {"a holder not kept here", seat.KeptArtifactInput{Module: "lost", Fp: fp, Digest: hex}, seat.KeptArtifactAnswerRefusedNotKept, ""}, - } { - got := m.keptArtifact(a.in) - if got.Refused != a.refused || got.Store != a.store || (a.refused == "" && (got.Object != hex || got.Size == 0)) || - (a.refused != "" && !strings.Contains(got.Words, a.in.Module)) { - t.Errorf("%s: %+v", a.name, got) + var verb func(map[string]any) (any, error) + for _, tool := range m.SeatTools(nil) { + if tool.Name == Seat+"."+seat.VerbKeptArtifact { + verb = tool.Run } } + out, _ := verb(map[string]any{"module": "demo", "fp": fp, "digest": hex}) + raw, _ := json.Marshal(out) + var got seat.KeptArtifactAnswer + if err := json.Unmarshal(raw, &got); err != nil || got.Refused != "" || got.Error != "" || got.Store != "mesh-build-go_kept-builds" || got.Object != hex || got.Size == 0 { + t.Errorf("kept-artifact of the kept build: %s", raw) + } // Another node's build in the shared store is not this node's to answer. + m.AskInput = func(string, any) (json.RawMessage, error) { + return json.RawMessage(`{"module":{"module":"demo","manifest":` + holder + `}}`), nil + } m.Node = "g14" if got := m.keptArtifact(seat.KeptArtifactInput{Module: "demo", Fp: fp, Digest: hex}); got.Refused != seat.KeptArtifactAnswerRefusedNotKept { t.Errorf("another node's kept build answered %+v", got) diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks.go b/modules/mesh-build-image/cmd/mesh-build-image/asks.go index ca33045ad..38a1bda81 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks.go @@ -77,7 +77,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // SeatTools are the seat's verbs on this node (ADR 0525 §10). func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool { - return append(buildstep.SeatTools(Seat, "image", p), stepsTool()) + return append(buildstep.SeatTools(Seat, "image", p), stepsTool(), m.keptArtifactVerb()) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go b/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go index 4ff1a2d4a..ebdece6a1 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/asks_test.go @@ -2,12 +2,14 @@ package main import ( "context" + "encoding/json" "strings" "testing" "time" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" ) const askCommit = "0123456789abcdef0123456789abcdef01234567" @@ -62,15 +64,33 @@ func TestAPulledImageAskCopiesThenBuilds(t *testing.T) { if got := strings.Join(calls, ","); got != "build_image_copy,build_image_fetch,build_image_build" { t.Errorf("calls = %s", got) } - served := map[string]bool{} + served := map[string]stdio.Tool{} for _, tool := range m.SeatTools(p) { - served[tool.Name] = true + served[tool.Name] = tool } - for _, verb := range []string{"current", "pause", "resume"} { - if !served[Seat+"."+verb] { + for _, verb := range []string{"current", "pause", "resume", "kept-artifact"} { + if _, ok := served[Seat+"."+verb]; !ok { t.Errorf("%s.%s is not served", Seat, verb) } } + // Design 47 lines 1888-1906 (test 60): kept-artifact is the domain SDK's judgement over no kept build here: the + // store's holder is not kept, any other module claims no oci-artifact-registry. + m.Call = func(key string, body any) (json.RawMessage, error) { + mod, _ := body.(map[string]any)["module"].(string) + man := `{"module":"` + mod + `"}` + if mod == "holder" { + man = `{"module":"holder","claims":[{"name":"oci-artifact-registry","scope":"mesh"}]}` + } + return json.RawMessage(`{"module":{"module":"` + mod + `","manifest":` + man + `}}`), nil + } + for mod, refused := range map[string]string{"holder": "not kept", "web": "claims no oci-artifact-registry"} { + if v, ok := served[Seat+".kept-artifact"]; ok { + out, _ := v.Run(map[string]any{"module": mod, "fp": "fp1", "digest": strings.Repeat("a", 64)}) + if a, _ := out.(sdkstep.KeptArtifactAnswer); a.Refused != refused || !strings.Contains(a.Words, mod) { + t.Errorf("kept-artifact of %s answered %+v, want refused %q", mod, out, refused) + } + } + } // A node without /dev/net/tun (a build node, 2026-10-28) takes no ask: its capacity is 0 and says why, so the ask goes to a node // that can build it instead of failing in buildah's network setup; once it is there again, the capacity is back. tun := m.Tun diff --git a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go index 8a09744da..e952b09b5 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/compose_test.go @@ -24,7 +24,7 @@ func TestTheModuleInvokesTheImageSeatAndHoldsNoRegistry(t *testing.T) { t.Fatal(err) } if want := []string{"seat:container-image-registry.publish", "seat:container-image-registry.publish-part", - "seat:container-image-registry.mirror", "seat:mesh-module-assigner.assignments"}; !slices.Equal(m.Invokes, want) { + "seat:container-image-registry.mirror", "seat:mesh-module-assigner.assignments", "seat:mesh-catalog.module"}; !slices.Equal(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // The seat's outcomes a copy waits on; the version comes in the ask, so version.numbered is not heard (design 47 G §413). diff --git a/modules/mesh-build-image/cmd/mesh-build-image/keptartifact.go b/modules/mesh-build-image/cmd/mesh-build-image/keptartifact.go new file mode 100644 index 000000000..27dd10a30 --- /dev/null +++ b/modules/mesh-build-image/cmd/mesh-build-image/keptartifact.go @@ -0,0 +1,42 @@ +package main + +// **The artifact store's holder takes its own new build even when its seat is the one broken** (novox/hq design 47, +// lines 1888-1906 at b229f5c4; test 60; milestone 47/G W12.5, issue 1966): the verb kept-artifact sits in every build +// seat's definition, so this module serves it too. It keeps no build of its own: its answer is the domain SDK's +// judgement (sdk/buildstep KeptArtifact, ADR 0634 §2) over no kept build, so a module that claims no +// oci-artifact-registry is refused as such and the store's holder is answered not kept here. Who may call it is the +// seat definition's (the node-engines alone), granted by the grants part (ADR 0666 §20): this module checks no caller. + +import ( + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildimage" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// keptArtifact is the verb kept-artifact's answer for in: this node keeps no build. +func (m *Module) keptArtifact(in seat.KeptArtifactInput) sdkstep.KeptArtifactAnswer { + return sdkstep.KeptArtifact(in.Module, in.Fp, in.Digest, nil, "", func(module string) (bool, error) { + call := m.call() + raw, err := call(sdkstep.CatalogModuleVerb, map[string]any{"module": module}) + if err != nil { + return false, err + } + return sdkstep.ClaimsStoreOf(module, raw) + }) +} + +// keptArtifactVerb serves `.kept-artifact`. +func (m *Module) keptArtifactVerb() stdio.Tool { + return stdio.Tool{Name: Seat + "." + seat.VerbKeptArtifact, + Description: "Where the build this node kept for the artifact store's holder (the holder of " + sdkstep.StoreSeat + + ") is. This build module keeps none: it refuses a module that claims no " + sdkstep.StoreSeat + + " and answers the holder's build not kept here, naming the module.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "digest"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module, the holder of " + sdkstep.StoreSeat}, + "fp": map[string]any{"type": "string", "description": "its build's fingerprint"}, + "digest": map[string]any{"type": "string", "description": "the artifact's sha256, 64 lowercase hex digits"}}}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + return m.keptArtifact(seat.KeptArtifactInput{Module: str("module"), Fp: str("fp"), Digest: str("digest")}), nil + }} +} diff --git a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go index 8419a7a08..e4b6b3d4c 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/manifest_test.go @@ -125,7 +125,7 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if want := []string{"keys", MirrorWaitsState, "pauses", "steps"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } raw, _ := os.ReadFile(filepath.Join(moduleDir, "module.json")) diff --git a/modules/mesh-build-image/module.json b/modules/mesh-build-image/module.json index be406b135..9190b8be7 100644 --- a/modules/mesh-build-image/module.json +++ b/modules/mesh-build-image/module.json @@ -21,7 +21,7 @@ { "name": "node-build-image", "scope": "node", - "serves": ["current", "drop", "pause", "resume", "steps"] + "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact"] } ], "config": { @@ -65,7 +65,8 @@ "seat:container-image-registry.publish", "seat:container-image-registry.publish-part", "seat:container-image-registry.mirror", - "seat:mesh-module-assigner.assignments" + "seat:mesh-module-assigner.assignments", + "seat:mesh-catalog.module" ], "consumes": [ "container-image-registry.published", diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks.go b/modules/mesh-build-node/cmd/mesh-build-node/asks.go index 5161802e3..da739e18a 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks.go @@ -87,7 +87,7 @@ func (m *Module) Puller(steps []StepTool, source buildstep.AskSource) *buildstep // SeatTools are the seat's verbs on this node (ADR 0525 §10). func (m *Module) SeatTools(p *buildstep.Puller) []stdio.Tool { - return append(buildstep.SeatTools(Seat, "Node", p), m.stepsTool()) + return append(buildstep.SeatTools(Seat, "Node", p), m.stepsTool(), m.keptArtifactVerb()) } // StepEndTool ends one step left running (issue 1890); the module's steps state is the one it reads and writes. diff --git a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go index 3b02e0b23..56546c5be 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/asks_test.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" ) const askCommit = "0123456789abcdef0123456789abcdef01234567" @@ -123,13 +124,31 @@ func TestManifestTakesTheBuildCapacity(t *testing.T) { t.Error("no settings file") } m0 := &Module{Node: "ace", Capacity: func() int { return 0 }} - served := map[string]bool{} + served := map[string]stdio.Tool{} for _, tool := range m0.SeatTools(m0.Puller(nil, buildstep.NewFakeAsks())) { - served[tool.Name] = true + served[tool.Name] = tool } - for _, verb := range []string{"current", "pause", "resume"} { - if !served[Seat+"."+verb] { + for _, verb := range []string{"current", "pause", "resume", "kept-artifact"} { + if _, ok := served[Seat+"."+verb]; !ok { t.Errorf("%s.%s is not served", Seat, verb) } } + // Design 47 lines 1888-1906 (test 60): kept-artifact is the domain SDK's judgement over no kept build here: the + // store's holder is not kept, any other module claims no oci-artifact-registry. + m0.Call = func(key string, body any) (json.RawMessage, error) { + mod, _ := body.(map[string]any)["module"].(string) + man := `{"module":"` + mod + `"}` + if mod == "holder" { + man = `{"module":"holder","claims":[{"name":"oci-artifact-registry","scope":"mesh"}]}` + } + return json.RawMessage(`{"module":{"module":"` + mod + `","manifest":` + man + `}}`), nil + } + for mod, refused := range map[string]string{"holder": "not kept", "web": "claims no oci-artifact-registry"} { + if v, ok := served[Seat+".kept-artifact"]; ok { + out, _ := v.Run(map[string]any{"module": mod, "fp": "fp1", "digest": strings.Repeat("a", 64)}) + if a, _ := out.(sdkstep.KeptArtifactAnswer); a.Refused != refused || !strings.Contains(a.Words, mod) { + t.Errorf("kept-artifact of %s answered %+v, want refused %q", mod, out, refused) + } + } + } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/keptartifact.go b/modules/mesh-build-node/cmd/mesh-build-node/keptartifact.go new file mode 100644 index 000000000..30dd76998 --- /dev/null +++ b/modules/mesh-build-node/cmd/mesh-build-node/keptartifact.go @@ -0,0 +1,45 @@ +package main + +// **The artifact store's holder takes its own new build even when its seat is the one broken** (novox/hq design 47, +// lines 1888-1906 at b229f5c4; test 60; milestone 47/G W12.5, issue 1966): the verb kept-artifact sits in every build +// seat's definition, so this module serves it too. It keeps no build of its own: its answer is the domain SDK's +// judgement (sdk/buildstep KeptArtifact, ADR 0634 §2) over no kept build, so a module that claims no +// oci-artifact-registry is refused as such and the store's holder is answered not kept here. Who may call it is the +// seat definition's (the node-engines alone), granted by the grants part (ADR 0666 §20): this module checks no caller. + +import ( + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/nodebuildnode" + sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// keptArtifact is the verb kept-artifact's answer for in: this node keeps no build. +func (m *Module) keptArtifact(in seat.KeptArtifactInput) sdkstep.KeptArtifactAnswer { + return sdkstep.KeptArtifact(in.Module, in.Fp, in.Digest, nil, "", func(module string) (bool, error) { + call := m.Call + if call == nil { + call = stdio.Ask + } + raw, err := call(sdkstep.CatalogModuleVerb, map[string]any{"module": module}) + if err != nil { + return false, err + } + return sdkstep.ClaimsStoreOf(module, raw) + }) +} + +// keptArtifactVerb serves `.kept-artifact`. +func (m *Module) keptArtifactVerb() stdio.Tool { + return stdio.Tool{Name: Seat + "." + seat.VerbKeptArtifact, + Description: "Where the build this node kept for the artifact store's holder (the holder of " + sdkstep.StoreSeat + + ") is. This build module keeps none: it refuses a module that claims no " + sdkstep.StoreSeat + + " and answers the holder's build not kept here, naming the module.", + Input: map[string]any{"type": "object", "required": []string{"module", "fp", "digest"}, "properties": map[string]any{ + "module": map[string]any{"type": "string", "description": "the module, the holder of " + sdkstep.StoreSeat}, + "fp": map[string]any{"type": "string", "description": "its build's fingerprint"}, + "digest": map[string]any{"type": "string", "description": "the artifact's sha256, 64 lowercase hex digits"}}}, + Run: func(args map[string]any) (any, error) { + str := func(k string) string { s, _ := args[k].(string); return s } + return m.keptArtifact(seat.KeptArtifactInput{Module: str("module"), Fp: str("fp"), Digest: str("digest")}), nil + }} +} diff --git a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go index 131d1c15c..288922bf5 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/manifest_test.go @@ -117,7 +117,7 @@ func TestManifestKeepsItsStatesClaimsItsBuildSeatAndDeclaresItsHealthTool(t *tes if want := []string{"keys", "pauses", "steps", "toolchains"}; !slices.Equal(states, want) { t.Errorf("state = %v, want %v", states, want) } - if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps"}) { + if len(m.Claims) != 1 || m.Claims[0].Name != Seat || !slices.Equal(m.Claims[0].Serves, []string{"current", "drop", "pause", "resume", "steps", "kept-artifact"}) { t.Errorf("claims = %v, want the seat %s alone", m.Claims, Seat) } if !slices.Contains(m.Tools, HealthTool) { diff --git a/modules/mesh-build-node/module.json b/modules/mesh-build-node/module.json index 4d9a5936b..e1cf7a539 100644 --- a/modules/mesh-build-node/module.json +++ b/modules/mesh-build-node/module.json @@ -21,7 +21,7 @@ { "name": "node-build-node", "scope": "node", - "serves": ["current", "drop", "pause", "resume", "steps"] + "serves": ["current", "drop", "pause", "resume", "steps", "kept-artifact"] } ], "config": { @@ -79,7 +79,8 @@ "seat:oci-artifact-registry.publish-part", "seat:npm-package-registry.publish", "seat:npm-package-registry.publish-part", - "seat:mesh-module-assigner.assignments" + "seat:mesh-module-assigner.assignments", + "seat:mesh-catalog.module" ], "own-secrets": { "broker": "${dir:state}/broker" -- 2.54.0 From cb5a8dd98ea7439a46eb8099c70843f14e27fb01 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:35:53 +0100 Subject: [PATCH 039/243] The gate keeps one delivery record per milestone's delivery on its ledger (milestone 47/G, W11.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0656 §7 and design 47 (The delivery record): the gate asks the bus module for a ledger `records` (template ledger, capped at 768 MiB) with two types: a record, keyed by its milestone and its heads, whose operations open it, write its group state, trace, outcomes, nodes away, pinned revisions and end, and remove it; and a month's list of the records opened, since a bundle cannot list a ledger's instances. An open of heads already recorded leaves the record as it is, an outcome applied twice writes nothing, and each open removes the records past the setting record-retention (90 days by default), naming each revision they pinned. The layout is the gate's own (design 20). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 84eda463.) Builds: design 47 G §141, §142, §143, §144, §145, §146, §155, §34 (GATE1) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/holder.go | 2 + .../cmd/mesh-delivery-gate/main.go | 1 + .../cmd/mesh-delivery-gate/record.go | 507 ++++++++++++++++++ .../cmd/mesh-delivery-gate/record_test.go | 305 +++++++++++ .../cmd/mesh-delivery-gate/settings.go | 25 + .../mesh-delivery-gate/files/settings.json | 3 +- modules/mesh-delivery-gate/module.json | 209 +++++++- 7 files changed, 1050 insertions(+), 2 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 36c28b901..5c91bee57 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -38,6 +38,8 @@ type Holder struct { Source SourceStore // Owed is the gate's bucket owed: what a transition owes the forge, written as keys (owed.go). Owed OwedStore + // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. + Records *Records // Pulls keeps the pull requests' keys on the bus (pullkey.go); nil keeps none. Pulls PullKeys // Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 6d86d423a..2d150687a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -73,6 +73,7 @@ func main() { Conditions: kvConditions{}, } h.init() + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention} l := &listening{now: "not yet: starting"} go run(h, l) if err := announceKept(stdio.Keeps); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go new file mode 100644 index 000000000..b838552d2 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -0,0 +1,507 @@ +package main + +// The delivery record (novox/hq ADR 0656 §7, design 47 *The delivery record*): one per milestone's delivery, a ledger +// instance of the gate's own (ADR 0595's template ledger, ADR 0597) that the gate alone writes. It holds the group's state +// (the heads, the verdicts, the checks and the asks, which `merges` held), the trace (the milestone, its issues, its pull +// requests, their commits and the graph revisions the delivery moved between) and what happened to each entry on each +// node. It is keyed by its milestone and its heads, so a switch or a delivery of milestone G again finds a record already +// there and leaves it as it is. Its layout is the gate's own (design 20 *Where a seat's code goes*: "logic and state +// layouts one module owns"); another module reads it through the seat's verbs, never the ledger (ADR 0596). + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "reflect" + "regexp" + "sort" + "strings" + "time" + + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// RecordsLedger is the local name the gate asks its ledger for in `requires`. +const RecordsLedger = "records" + +// RecordForm is the form of the records' layout (design 47 *A module's kept data on the bus*): raised whenever the gate +// changes how it writes them. +const RecordForm = 1 + +// The ledger's two types: a record, and the list of the records opened in one month (the ledger is listed through it, +// since a bundle has no listing of a ledger's instances; a month of about 600 records keeps it far under a value's 256 +// KiB). +const ( + TypeRecord = "record" + TypeHeld = "held" +) + +// The operations of a record, each one effect (ADR 0597 §2), and of the month's list. The manifest's `requires` declares +// exactly these (record_test.go). +const ( + OpOpen = "open" // set-fields: the milestone, the heads, when, forge-made, the form + OpGroup = "group" // set-fields: the group's state + OpTrace = "trace" // set-fields: the trace + OpOutcome = "outcome" // set-entry into outcomes, by the entry, the node and the step + OpAway = "away" // set-entry into away, by node + OpBack = "back" // remove from away, by node + OpPin = "pin" // set-entry into pinned, by revision + OpFree = "free" // remove from pinned, by revision + OpEnd = "end" // set-fields: ended + OpRemove = "remove" // remove the instance, at its retention + OpHold = "hold" // a month's list: set-entry into records, by instance + OpDrop = "drop" // a month's list: remove from records, by instance +) + +// RecordOperations are a record's operations, in the order the manifest's type allows them. +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpEnd, OpRemove} + +// HeldOperations are the month's list's operations. +var HeldOperations = []string{OpHold, OpDrop} + +// DefaultRecordRetention is how long a record is kept when the setting record-retention gives none (design 47: 90 days +// covers every trace the operator has asked of a past delivery so far). +const DefaultRecordRetention = 90 * 24 * time.Hour + +// Record is a delivery record's state, as apply derives it from its operations. +type Record struct { + // Instance is the record's path in the ledger; not in the state. + Instance string `json:"-"` + // Sequence is the last operation the state holds; not in the state. + Sequence uint64 `json:"-"` + + Milestone string `json:"milestone"` + // Heads are the heads the record is for: each member's id (owner/repository@commit) to its full commit. + Heads map[string]string `json:"heads"` + Opened time.Time `json:"opened"` + // ForgeMade is a record opened for a merge the forge made with no record's merge-pull (design 47 *What the gate owes + // the forge*): it has no Approve. + ForgeMade bool `json:"forge-made,omitempty"` + Form int `json:"form,omitempty"` + + Group *RecordGroup `json:"group,omitempty"` + Trace *RecordTrace `json:"trace,omitempty"` + // Outcomes are what happened to each entry on each node, by OutcomeKey. + Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` + // Away are the nodes away, by node, each with since when. + Away map[string]NodeAway `json:"away,omitempty"` + // Pinned are the graph revisions the record points at, by revision. + Pinned map[string]PinnedRevision `json:"pinned,omitempty"` + Ended *RecordEnded `json:"ended,omitempty"` +} + +// RecordGroup is the group's state as the record keeps it: what `merges` held for the group, and its members. +type RecordGroup struct { + Group string `json:"group"` + Members []string `json:"members,omitempty"` + // Ask is the key of the Approve's ask; Approved the heads it covers, `@` to the full head. + Ask string `json:"ask,omitempty"` + Approved map[string]string `json:"approved,omitempty"` + // Pulls are each member's merge, as merging, merged or refused, with the commit or the forge's words. + Pulls []RecordPull `json:"pulls,omitempty"` +} + +// RecordPull is one member's merge. +type RecordPull struct { + Repository string `json:"repository"` + Number int `json:"number"` + Head string `json:"head"` + State string `json:"state"` + Commit string `json:"commit,omitempty"` + Words string `json:"words,omitempty"` +} + +// RecordTrace is the trace: the milestone's issues and pull requests, their commits, and the graph revisions. +type RecordTrace struct { + Issues []int `json:"issues,omitempty"` + Pulls []string `json:"pulls,omitempty"` + Commits []string `json:"commits,omitempty"` + // Source is the source revision the merge moved; From and To the assignment graph revisions it moved between. + Source string `json:"source,omitempty"` + From string `json:"from,omitempty"` + To string `json:"to,omitempty"` +} + +// The outcomes an entry has on a node (ADR 0656 §7, design 47 *The delivery record*). +const ( + OutcomeTakenIn = "taken-in" + OutcomeBuilt = "built" + OutcomeChecked = "checked" + OutcomeComposed = "composed" + OutcomeWanted = "wanted" + OutcomeRunning = "running-healthy" + OutcomeHeld = "held" + OutcomeFailed = "failed" + OutcomeAway = "away" +) + +// RecordOutcome is what happened to one entry on one node at one step, with the event it was taken from. +type RecordOutcome struct { + Key string `json:"key"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint,omitempty"` + Node string `json:"node,omitempty"` + Step string `json:"step"` + Outcome string `json:"outcome"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` + // Event is the id of the event it was taken from (x-event-id), so an event applied twice is known. + Event string `json:"event,omitempty"` +} + +// OutcomeKey is the key of an entry's outcome: its module, its fingerprint, its node and its step, each a token. +func OutcomeKey(module, fingerprint, node, step string) string { + return strings.Join([]string{module, fingerprint, node, step}, "|") +} + +// NodeAway is a node away, and since when. +type NodeAway struct { + Node string `json:"node"` + Since time.Time `json:"since"` +} + +// PinnedRevision is a graph revision the record points at: its half (catalog or assignment) and its revision. +type PinnedRevision struct { + Revision string `json:"revision"` + Half string `json:"half"` +} + +// RecordEnded is how a record ended: when, why, by whom, and the nodes away it ended without. +type RecordEnded struct { + At time.Time `json:"at"` + Why string `json:"why"` + By string `json:"by,omitempty"` + Away []NodeAway `json:"away,omitempty"` +} + +// heldList is a month's list of records: each record opened that month, by instance. +type heldList struct { + Records map[string]heldEntry `json:"records,omitempty"` +} + +type heldEntry struct { + Instance string `json:"instance"` + Opened time.Time `json:"opened"` +} + +// segmentRe is what one part of an instance's path may hold (the tool runner's rule). +var segmentRe = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// milestoneToken is a milestone as one part of an instance's path: its letters, digits, - and _ kept, every other byte +// written _, and, when that changed it, a short digest of the milestone added, so two milestones never meet. +func milestoneToken(milestone string) string { + var b strings.Builder + for i := 0; i < len(milestone); i++ { + c := milestone[i] + if c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c == '-' { + b.WriteByte(c) + continue + } + b.WriteByte('_') + } + if b.String() == milestone && milestone != "" { + return milestone + } + sum := sha256.Sum256([]byte(milestone)) + return b.String() + "_" + hex.EncodeToString(sum[:4]) +} + +// HeadsDigest is the digest of a set of heads: each member's id and full commit, sorted, sixteen hex characters. +func HeadsDigest(heads map[string]string) string { + ids := make([]string, 0, len(heads)) + for id := range heads { + ids = append(ids, id) + } + sort.Strings(ids) + h := sha256.New() + for _, id := range ids { + fmt.Fprintf(h, "%s=%s\n", strings.ToLower(id), strings.ToLower(heads[id])) + } + return hex.EncodeToString(h.Sum(nil))[:16] +} + +// RecordInstance is the instance of the record of a milestone at its heads: `record//`. +func RecordInstance(milestone string, heads map[string]string) (string, error) { + if strings.TrimSpace(milestone) == "" { + return "", errors.New("a delivery record names its milestone") + } + if len(heads) == 0 { + return "", fmt.Errorf("the delivery record of %s names no heads", milestone) + } + for id, sha := range heads { + if id == "" || sha == "" { + return "", fmt.Errorf("the delivery record of %s names a head with no id or commit (%q=%q)", milestone, id, sha) + } + } + inst := TypeRecord + "/" + milestoneToken(milestone) + "/" + HeadsDigest(heads) + for _, p := range strings.Split(inst, "/") { + if !segmentRe.MatchString(p) { + return "", fmt.Errorf("the delivery record of %s cannot be named: %q", milestone, inst) + } + } + return inst, nil +} + +// heldInstance is the list of the records opened in a month: `held/2026-11`. +func heldInstance(t time.Time) string { return TypeHeld + "/" + t.UTC().Format("2006-01") } + +// RecordLedger is the gate's ledger: an operation applied to one instance, and one instance's state read. +type RecordLedger interface { + Apply(instance, operation string, body any) (uint64, error) + // Get is the instance's state and the sequence of its last operation; a nil state when it has none. + Get(instance string) (json.RawMessage, uint64, error) +} + +// sdkLedger is the gate's ledger through the runtime: the tool runner's ledger calls, on the gate's own ledger only +// (ADR 0597 §3, §10). Only the tool runner's ledger credential publishes on its history (§9). +type sdkLedger struct{} + +// ledgerCall is the runtime's call on the module's own bus state; a test's stand-in. +var ledgerCall = stdio.BusState + +func (sdkLedger) Apply(instance, operation string, body any) (uint64, error) { + raw, err := json.Marshal(body) + if err != nil { + return 0, err + } + out, err := ledgerCall(buscalls.LedgerApply, buscalls.ApplyRequest{Ledger: RecordsLedger, Instance: instance, + Operation: operation, Body: raw}) + if err != nil { + return 0, err + } + var got buscalls.Applied + if err := json.Unmarshal(out, &got); err != nil { + return 0, fmt.Errorf("the ledger's answer to %s on %s is not readable: %w", operation, instance, err) + } + if !got.Taken { + return 0, fmt.Errorf("the ledger did not take %s on %s", operation, instance) + } + return got.Sequence, nil +} + +func (sdkLedger) Get(instance string) (json.RawMessage, uint64, error) { + out, err := ledgerCall(buscalls.LedgerGet, buscalls.InstanceRequest{Ledger: RecordsLedger, Instance: instance}) + if err != nil { + return nil, 0, err + } + var got buscalls.InstanceState + if err := json.Unmarshal(out, &got); err != nil { + return nil, 0, fmt.Errorf("the ledger's state of %s is not readable: %w", instance, err) + } + if len(got.State) == 0 || string(got.State) == "null" { + return nil, got.Sequence, nil + } + return got.State, got.Sequence, nil +} + +// Records keeps the delivery records on the ledger: the one writer of them. +type Records struct { + Ledger RecordLedger + Now func() time.Time + // Retention is how long a record is kept; nil keeps DefaultRecordRetention. + Retention func() time.Duration + // Freed is told each revision a removed record pinned, to say as `freed` (design 47); nil says none. + Freed func(r *Record, p PinnedRevision) +} + +func (rs *Records) now() time.Time { + if rs.Now == nil { + return time.Now().UTC() + } + return rs.Now().UTC() +} + +func (rs *Records) retention() time.Duration { + if rs.Retention == nil { + return DefaultRecordRetention + } + if d := rs.Retention(); d > 0 { + return d + } + return DefaultRecordRetention +} + +// Read is one record, or nil when the ledger holds none at that instance (or one removed). +func (rs *Records) Read(instance string) (*Record, error) { + raw, seq, err := rs.Ledger.Get(instance) + if err != nil || raw == nil { + return nil, err + } + var r Record + if err := json.Unmarshal(raw, &r); err != nil { + return nil, fmt.Errorf("the delivery record %s is not readable: %w", instance, err) + } + if r.Milestone == "" { + return nil, nil + } + r.Instance, r.Sequence = instance, seq + return &r, nil +} + +// Open opens the record of a milestone at its heads, unless one is there already, which is left as it is (the switch +// and a delivery of milestone G again after a revert make no second record and ask nothing again). Opened says it was +// opened now. Each open removes the records past their retention first (the retention needs no timer: records open as +// deliveries do). +func (rs *Records) Open(milestone string, heads map[string]string, forgeMade bool) (r *Record, opened bool, err error) { + inst, err := RecordInstance(milestone, heads) + if err != nil { + return nil, false, err + } + if r, err = rs.Read(inst); err != nil || r != nil { + return r, false, err + } + now := rs.now() + if err := rs.Sweep(now); err != nil { + return nil, false, fmt.Errorf("the records past their retention cannot be removed, so no record is opened: %w", err) + } + body := map[string]any{"milestone": milestone, "heads": heads, "opened": now, "form": RecordForm} + if forgeMade { + body["forge-made"] = true + } + if _, err := rs.Ledger.Apply(inst, OpOpen, body); err != nil { + return nil, false, err + } + if _, err := rs.Ledger.Apply(heldInstance(now), OpHold, heldEntry{Instance: inst, Opened: now}); err != nil { + return nil, false, fmt.Errorf("the record %s is opened but not listed: %w", inst, err) + } + r, err = rs.Read(inst) + return r, err == nil && r != nil, err +} + +// List is every record kept, opened within the retention and a month more, oldest first. +func (rs *Records) List() ([]*Record, error) { + now := rs.now() + var out []*Record + for _, m := range rs.months(now) { + list, err := rs.heldIn(m) + if err != nil { + return nil, err + } + for _, e := range list { + r, err := rs.Read(e.Instance) + if err != nil { + return nil, err + } + if r != nil { + out = append(out, r) + } + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Opened.Before(out[j].Opened) }) + return out, nil +} + +// months are the months' lists a record within the retention can be in, oldest first, one more at each end. +func (rs *Records) months(now time.Time) []string { + from := now.Add(-rs.retention()).AddDate(0, -1, 0) + var out []string + for m := time.Date(from.Year(), from.Month(), 1, 0, 0, 0, 0, time.UTC); !m.After(now); m = m.AddDate(0, 1, 0) { + out = append(out, heldInstance(m)) + } + return out +} + +func (rs *Records) heldIn(month string) ([]heldEntry, error) { + raw, _, err := rs.Ledger.Get(month) + if err != nil || raw == nil { + return nil, err + } + var l heldList + if err := json.Unmarshal(raw, &l); err != nil { + return nil, fmt.Errorf("the list of records %s is not readable: %w", month, err) + } + out := make([]heldEntry, 0, len(l.Records)) + for _, e := range l.Records { + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { return out[i].Opened.Before(out[j].Opened) }) + return out, nil +} + +// Sweep removes every record opened longer ago than the retention, and says `freed` for each revision it pinned +// (design 47: "When the retention removes a record, the gate says freed for each revision it pinned"). Nothing else is +// removed to make room. +func (rs *Records) Sweep(now time.Time) error { + cut := now.Add(-rs.retention()) + for _, m := range rs.months(now) { + list, err := rs.heldIn(m) + if err != nil { + return err + } + for _, e := range list { + if !e.Opened.Before(cut) { + continue + } + r, err := rs.Read(e.Instance) + if err != nil { + return err + } + if r != nil { + if _, err := rs.Ledger.Apply(e.Instance, OpRemove, map[string]any{}); err != nil { + return err + } + if rs.Freed != nil { + for _, p := range sortedPins(r) { + rs.Freed(r, p) + } + } + } + if _, err := rs.Ledger.Apply(m, OpDrop, map[string]any{"instance": e.Instance}); err != nil { + return err + } + } + } + return nil +} + +func sortedPins(r *Record) []PinnedRevision { + out := make([]PinnedRevision, 0, len(r.Pinned)) + for _, p := range r.Pinned { + out = append(out, p) + } + sort.Slice(out, func(i, j int) bool { return out[i].Revision < out[j].Revision }) + return out +} + +// SetGroup writes the record's group state, unless it holds that state already. +func (rs *Records) SetGroup(r *Record, g RecordGroup) error { + if r.Group != nil && reflect.DeepEqual(*r.Group, g) { + return nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpGroup, map[string]any{"group": g}) + if err != nil { + return err + } + r.Group, r.Sequence = &g, seq + return nil +} + +// Outcome writes one entry's outcome on one node, unless the record holds it already: an event applied twice changes +// nothing (design 47). Applied says it was written now. +func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) { + if o.Module == "" || o.Step == "" || o.Outcome == "" { + return false, fmt.Errorf("an outcome names its module, its step and what happened (%+v)", o) + } + o.Key = OutcomeKey(o.Module, o.Fingerprint, o.Node, o.Step) + if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) { + return false, nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpOutcome, o) + if err != nil { + return false, err + } + if r.Outcomes == nil { + r.Outcomes = map[string]RecordOutcome{} + } + r.Outcomes[o.Key], r.Sequence = o, seq + return true, nil +} + +// sameOutcome is whether two outcomes say the same, whenever and from whichever event they were taken. +func sameOutcome(a, b RecordOutcome) bool { + a.At, b.At, a.Event, b.Event = time.Time{}, time.Time{}, "", "" + return a == b +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go new file mode 100644 index 000000000..b101b362f --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -0,0 +1,305 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// ledgerFake plays the tool runner's ledger for the gate's `records`, from the types the manifest declares: each operation +// checked against its type and folded by its effect as apply does (ADR 0597 §2), so a record the manifest's types could not +// hold fails here. +type ledgerFake struct { + t *testing.T + types map[string]catalogue.LedgerType + states map[string]json.RawMessage + seq map[string]uint64 + history []string + failOn string +} + +func newLedgerFake(t *testing.T) *ledgerFake { + t.Helper() + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + Requires []json.RawMessage `json:"requires"` + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + f := &ledgerFake{t: t, types: map[string]catalogue.LedgerType{}, states: map[string]json.RawMessage{}, seq: map[string]uint64{}} + for _, r := range m.Requires { + var e catalogue.BusStateEntry + if json.Unmarshal(r, &e) == nil && e.Name == RecordsLedger && e.Storage == catalogue.StorageLedger { + for _, ty := range e.Types { + f.types[ty.Name] = ty + } + } + } + if len(f.types) == 0 { + t.Fatalf("the manifest asks for no ledger %s in requires", RecordsLedger) + } + return f +} + +func (f *ledgerFake) Apply(instance, operation string, body any) (uint64, error) { + if f.failOn == operation { + return 0, fmt.Errorf("the ledger is away") + } + ty, ok := f.types[strings.SplitN(instance, "/", 2)[0]] + if !ok { + return 0, fmt.Errorf("no type for %s", instance) + } + var op *catalogue.LedgerOperation + for i := range ty.Operations { + if ty.Operations[i].Name == operation { + op = &ty.Operations[i] + } + } + if op == nil { + return 0, fmt.Errorf("the type %s allows no operation %q", ty.Name, operation) + } + raw, _ := json.Marshal(body) + in := map[string]any{} + if err := json.Unmarshal(raw, &in); err != nil { + return 0, err + } + for _, field := range op.Fields { + if _, known := ty.Schema[field]; !known { + return 0, fmt.Errorf("the operation %s copies %s, which the type %s's schema lacks", operation, field, ty.Name) + } + } + for name := range in { + if _, known := ty.Schema[name]; !known && !slices.Contains(op.Fields, name) { + f.t.Errorf("the gate's %s on %s carries %s, which the manifest's type %s neither holds nor copies", operation, instance, name, ty.Name) + } + } + state := map[string]any{} + if old := f.states[instance]; old != nil { + _ = json.Unmarshal(old, &state) + } + copied := map[string]any{} + for _, field := range op.Fields { + if v, ok := in[field]; ok { + copied[field] = v + } + } + f.seq[instance]++ + switch op.Effect { + case catalogue.EffectSetFields: + for k, v := range copied { + state[k] = v + } + case catalogue.EffectSetEntry: + key, ok := in[op.Key].(string) + if !ok { + return 0, fmt.Errorf("%s names no entry by %s", operation, op.Key) + } + m, _ := state[op.Into].(map[string]any) + if m == nil { + m = map[string]any{} + } + m[key] = copied + state[op.Into] = m + case catalogue.EffectRemove: + if op.Into == "" { + f.states[instance] = json.RawMessage("null") + f.history = append(f.history, instance+" "+operation) + return f.seq[instance], nil + } + if m, ok := state[op.Into].(map[string]any); ok { + delete(m, in[op.Key].(string)) + } + default: + return 0, fmt.Errorf("effect %s", op.Effect) + } + out, _ := json.Marshal(state) + f.states[instance] = out + f.history = append(f.history, instance+" "+operation) + return f.seq[instance], nil +} + +func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) { + s := f.states[instance] + if s == nil || string(s) == "null" { + return nil, f.seq[instance], nil + } + return s, f.seq[instance], nil +} + +// W11.1 (ADR 0656 §7, design 47 *The delivery record*): one record per milestone's delivery, a ledger instance keyed by its +// milestone and its heads, kept for its retention, its operations those the manifest's ledger types allow. +func TestTheDeliveryRecordIsALedgerInstanceKeyedByMilestoneAndHeads(t *testing.T) { + heads := map[string]string{"novox/app@111111111111": strings.Repeat("1", 40), "novox/lib@222222222222": strings.Repeat("2", 40)} + same := map[string]string{"novox/lib@222222222222": strings.Repeat("2", 40), "novox/app@111111111111": strings.Repeat("1", 40)} + other := map[string]string{"novox/app@111111111111": strings.Repeat("3", 40), "novox/lib@222222222222": strings.Repeat("2", 40)} + at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + + t.Run("an instance per milestone and heads, a path the tool runner takes", func(t *testing.T) { + a, err := RecordInstance("47/G", heads) + if err != nil { + t.Fatal(err) + } + b, _ := RecordInstance("47/G", same) + c, _ := RecordInstance("47/G", other) + d, _ := RecordInstance("47_G", heads) + if a != b || a == c || a == d { + t.Fatalf("instances %s %s %s %s: one per milestone and heads, whatever the heads' order", a, b, c, d) + } + for _, p := range strings.Split(a, "/") { + if !segmentRe.MatchString(p) { + t.Fatalf("%s has a part the tool runner refuses", a) + } + } + if _, err := RecordInstance("", heads); err == nil { + t.Fatal("a record with no milestone is named") + } + if _, err := RecordInstance("47/G", nil); err == nil { + t.Fatal("a record with no heads is named") + } + }) + + t.Run("opened once; a second open leaves it as it is", func(t *testing.T) { + l := newLedgerFake(t) + now := at + rs := &Records{Ledger: l, Now: func() time.Time { return now }} + r, opened, err := rs.Open("47/G", heads, false) + if err != nil || !opened || r == nil || r.Milestone != "47/G" || len(r.Heads) != 2 || !r.Opened.Equal(at) || r.Form != RecordForm { + t.Fatalf("open: %+v %v %v", r, opened, err) + } + now = at.Add(time.Hour) + again, opened, err := rs.Open("47/G", same, false) + if err != nil || opened || again.Instance != r.Instance || !again.Opened.Equal(at) { + t.Fatalf("a second open of the same heads: %+v %v %v", again, opened, err) + } + list, err := rs.List() + if err != nil || len(list) != 1 || list[0].Instance != r.Instance { + t.Fatalf("list %+v %v", list, err) + } + }) + + t.Run("an outcome applied twice changes nothing", func(t *testing.T) { + l := newLedgerFake(t) + rs := &Records{Ledger: l, Now: func() time.Time { return at }} + r, _, _ := rs.Open("47/G", heads, false) + o := RecordOutcome{Module: "mesh-delivery-gate", Fingerprint: "src2:abcd", Node: "ace", Step: "build", Outcome: OutcomeBuilt, At: at, Event: "e1"} + if applied, err := rs.Outcome(r, o); err != nil || !applied { + t.Fatalf("first: %v %v", applied, err) + } + n := len(l.history) + o.At = at.Add(time.Minute) + if applied, err := rs.Outcome(r, o); err != nil || applied || len(l.history) != n { + t.Fatalf("the same event again: %v %v, %d operations (was %d)", applied, err, len(l.history), n) + } + o.Event = "e2" + if applied, _ := rs.Outcome(r, o); applied { + t.Fatal("the same outcome from another event is written again") + } + o.Outcome, o.Why, o.Event = OutcomeFailed, "the build failed", "e3" + if applied, _ := rs.Outcome(r, o); !applied { + t.Fatal("a changed outcome is not written") + } + back, _ := rs.Read(r.Instance) + if got := back.Outcomes[OutcomeKey("mesh-delivery-gate", "src2:abcd", "ace", "build")]; got.Outcome != OutcomeFailed || got.Why == "" { + t.Fatalf("the ledger holds %+v", back.Outcomes) + } + g := RecordGroup{Group: "feat/x", Members: []string{"novox/app@111111111111"}, Ask: "merge.feat/x.h"} + if err := rs.SetGroup(back, g); err != nil { + t.Fatal(err) + } + n = len(l.history) + if err := rs.SetGroup(back, g); err != nil || len(l.history) != n { + t.Fatalf("the same group state is written again: %v", err) + } + }) + + t.Run("a record past its retention is removed at the next open and frees what it pinned", func(t *testing.T) { + l := newLedgerFake(t) + now := at + var freed []string + rs := &Records{Ledger: l, Now: func() time.Time { return now }, Retention: func() time.Duration { return 90 * 24 * time.Hour }, + Freed: func(r *Record, p PinnedRevision) { freed = append(freed, r.Milestone+" "+p.Revision) }} + old, _, _ := rs.Open("46/F", heads, false) + if _, err := l.Apply(old.Instance, OpPin, PinnedRevision{Revision: "cat-1", Half: "catalog"}); err != nil { + t.Fatal(err) + } + now = at.Add(89 * 24 * time.Hour) + kept, _, _ := rs.Open("47/G", heads, false) + if r, _ := rs.Read(old.Instance); r == nil || len(freed) != 0 { + t.Fatalf("a record within its retention is removed (%v)", freed) + } + now = at.Add(91 * 24 * time.Hour) + if _, _, err := rs.Open("47/H", heads, false); err != nil { + t.Fatal(err) + } + if r, _ := rs.Read(old.Instance); r != nil { + t.Fatal("a record past its retention is kept") + } + if !slices.Equal(freed, []string{"46/F cat-1"}) { + t.Fatalf("freed %v", freed) + } + list, _ := rs.List() + if len(list) != 2 || list[0].Instance != kept.Instance { + t.Fatalf("list %+v", list) + } + }) + + t.Run("the setting record-retention, in days", func(t *testing.T) { + dir := t.TempDir() + for _, c := range []struct { + file string + want time.Duration + bad bool + }{ + {`{}`, DefaultRecordRetention, false}, + {`{"record-retention": 30}`, 30 * 24 * time.Hour, false}, + {`{"record-retention": 0}`, 0, true}, + {`{"record-retention": 1.5}`, 0, true}, + {`{"record-retention": "90"}`, 0, true}, + } { + p := filepath.Join(dir, "settings.json") + if err := os.WriteFile(p, []byte(c.file), 0o600); err != nil { + t.Fatal(err) + } + s, err := readSettings(p) + if (err != nil) != c.bad { + t.Fatalf("%s: %v", c.file, err) + } + if c.bad { + continue + } + h := &Holder{Settings: func() Settings { return s }} + if got := h.recordRetention(); got != c.want { + t.Fatalf("%s: %s, wanted %s", c.file, got, c.want) + } + } + }) +} + +// W11.1: the manifest's ledger types allow exactly the operations the gate applies, and the ledger is capped at 768 MiB. +func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { + l := newLedgerFake(t) + for name, want := range map[string][]string{TypeRecord: RecordOperations, TypeHeld: HeldOperations} { + var got []string + for _, op := range l.types[name].Operations { + got = append(got, op.Name) + } + if !slices.Equal(got, want) { + t.Errorf("the type %s allows %v, the gate applies %v", name, got, want) + } + } + raw, _ := os.ReadFile(filepath.Join("..", "..", "module.json")) + if !strings.Contains(string(raw), `"max-mib": 768`) { + t.Error("the records ledger is not capped at 768 MiB") + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go index 9d0d016e5..c3217ae2e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go @@ -29,6 +29,9 @@ type Settings struct { // "run the check locally only, but not as a gate, it loses us too much time"): a reviewed pull request is ready and its // Approve asked without a verdict, a trunk's move puts nothing out of date, and a verdict is shown, never waited on. CheckGatesMerge *bool `json:"check-gates-merge"` + // RecordRetentionDays is how many days a delivery record is kept from when it opened (design 47 *The delivery + // record*); nil keeps DefaultRecordRetention. + RecordRetentionDays *int `json:"record-retention"` } // readSettings reads the settings file; a value that is not a whole number of minutes, one or more, is refused. @@ -59,6 +62,17 @@ func readSettings(path string) (Settings, error) { return Settings{}, fmt.Errorf("%s is not a number", name) } } + switch v := loose["record-retention"].(type) { + case nil: + case float64: + n := int(v) + if v < 1 || v > 3650 || v != float64(n) { + return Settings{}, errors.New("record-retention is not a whole number of days, from 1 to 3650") + } + s.RecordRetentionDays = &n + default: + return Settings{}, errors.New("record-retention is not a number") + } switch v := loose["check-gates-merge"].(type) { case nil: case bool: @@ -99,6 +113,17 @@ func (h *Holder) refusalFloor() time.Duration { return minutesOr(h.Settings().RefusalFloorMinutes, DefaultRefusalFloor) } +// recordRetention is how long a delivery record is kept from when it opened. +func (h *Holder) recordRetention() time.Duration { + if h.Settings == nil { + return DefaultRecordRetention + } + if n := h.Settings().RecordRetentionDays; n != nil { + return time.Duration(*n) * 24 * time.Hour + } + return DefaultRecordRetention +} + // authorBound is how long a delivery that needs its author's new head waits before the operator hears of it (§5). func (h *Holder) authorBound() time.Duration { if h.Settings == nil { diff --git a/modules/mesh-delivery-gate/files/settings.json b/modules/mesh-delivery-gate/files/settings.json index 66c66f877..e164ea53c 100644 --- a/modules/mesh-delivery-gate/files/settings.json +++ b/modules/mesh-delivery-gate/files/settings.json @@ -1,5 +1,6 @@ { "refusal-stall-floor-minutes": 10, "author-new-head-minutes": 120, - "check-gates-merge": true + "check-gates-merge": true, + "record-retention": 90 } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index ec98a4044..0c48589f9 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -38,6 +38,204 @@ "uses": [ "operator-channel" ], + "requires": [ + { + "name": "records", + "storage": "ledger", + "template": "ledger", + "options": { + "max-mib": 768 + }, + "types": [ + { + "name": "record", + "schema": { + "milestone": { + "type": "string" + }, + "heads": { + "type": "object" + }, + "opened": { + "type": "string" + }, + "forge-made": { + "type": "boolean" + }, + "form": { + "type": "integer" + }, + "group": { + "type": "object" + }, + "trace": { + "type": "object" + }, + "key": { + "type": "string" + }, + "module": { + "type": "string" + }, + "fingerprint": { + "type": "string" + }, + "node": { + "type": "string" + }, + "step": { + "type": "string" + }, + "outcome": { + "type": "string" + }, + "why": { + "type": "string" + }, + "at": { + "type": "string" + }, + "event": { + "type": "string" + }, + "since": { + "type": "string" + }, + "revision": { + "type": "string" + }, + "half": { + "type": "string" + }, + "ended": { + "type": "object" + } + }, + "operations": [ + { + "name": "open", + "effect": "set-fields", + "fields": [ + "milestone", + "heads", + "opened", + "forge-made", + "form" + ], + "sequence": true + }, + { + "name": "group", + "effect": "set-fields", + "fields": [ + "group" + ], + "sequence": true + }, + { + "name": "trace", + "effect": "set-fields", + "fields": [ + "trace" + ], + "sequence": true + }, + { + "name": "outcome", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node", + "step", + "outcome", + "why", + "at", + "event" + ], + "into": "outcomes", + "key": "key", + "sequence": true + }, + { + "name": "away", + "effect": "set-entry", + "fields": [ + "node", + "since" + ], + "into": "away", + "key": "node" + }, + { + "name": "back", + "effect": "remove", + "into": "away", + "key": "node" + }, + { + "name": "pin", + "effect": "set-entry", + "fields": [ + "revision", + "half" + ], + "into": "pinned", + "key": "revision" + }, + { + "name": "free", + "effect": "remove", + "into": "pinned", + "key": "revision" + }, + { + "name": "end", + "effect": "set-fields", + "fields": [ + "ended" + ], + "sequence": true + }, + { + "name": "remove", + "effect": "remove" + } + ] + }, + { + "name": "held", + "schema": { + "instance": { + "type": "string" + }, + "opened": { + "type": "string" + } + }, + "operations": [ + { + "name": "hold", + "effect": "set-entry", + "fields": [ + "instance", + "opened" + ], + "into": "records", + "key": "instance" + }, + { + "name": "drop", + "effect": "remove", + "into": "records", + "key": "instance" + } + ] + } + ] + } + ], "packages": { "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg": "0.1" }, @@ -105,6 +303,14 @@ "description": "Whether mesh/check gates the merge. True (the default): the operator's Approve is asked, and a merge handed, only on a passing verdict judged against the trunk's head, and a trunk's move puts a verdict out of date (ADR 0447, ADR 0505 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"; developers run the unit and integration levels locally before a pull request): a ready pull request (no review line is waited for, ADR 0659) has its Approve is asked once the check at its heads (a group: of all its heads together) has a verdict, a pass or a failure said in the ask, or after 60 minutes with none, said so (the operator, 2026-10-27 23:54: \"if you don't want me to approve it, don't send the approval request\"); an approved merge waits for no verdict; no merge is refused for a check judged on an older trunk; a later verdict is shown on the pull request and the delivery, and asks a pending Approve again in its words; a group still merges all its members in one pass. Set it false together with the gitea module's check-gates-merge, which lets its merge tool merge an approved head without a passing mesh/check.", "trusted": false, "default": true + }, + "record-retention": { + "type": "number", + "description": "days a delivery record is kept from when it opened (design 47 *The delivery record*): ninety covers every trace asked of a past delivery so far; when a record is removed at its retention, the gate says freed for each graph revision it pinned. Nothing else is removed to make room.", + "minimum": 1, + "maximum": 3650, + "trusted": false, + "default": 90 } }, "resources": [ @@ -123,7 +329,8 @@ "takes-config": [ "refusal-stall-floor-minutes", "author-new-head-minutes", - "check-gates-merge" + "check-gates-merge", + "record-retention" ], "merge": "json", "template": "files/settings.json" -- 2.54.0 From 3f25e5b4da2818e37d9a0e9871e67fb4b688142c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:35:57 +0100 Subject: [PATCH 040/243] The delivery record hears the step events through the gate's own consumer (milestone 47/G, W11.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 47 (The delivery record): the outcomes arrive as events through the module's durable consumer, each applied once and only from its owner's account. A family per step event names the pattern the manifest consumes, its owner and whether its subject names a node; an event is applied only when its key names its owner and, for a node's event, its x-node is the node its subject names. It is taken once its outcome is in every open record holding it (by a member's merge commit, or by the entry of the record's graph difference), so one the ledger did not take is offered again; an unreadable one is said and taken. The indexer's take-in.taken is the first family; each other step's event joins the table with its owner's contract. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 4c4d1239.) Builds: design 47 G §34, §141 to §146 (GATE2) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/consumer.go | 277 ++++++++++++++++++ .../cmd/mesh-delivery-gate/holder.go | 2 + .../cmd/mesh-delivery-gate/main.go | 9 +- .../cmd/mesh-delivery-gate/record.go | 106 ++++++- .../cmd/mesh-delivery-gate/record_test.go | 100 +++++++ modules/mesh-delivery-gate/go.mod | 1 + modules/mesh-delivery-gate/module.json | 16 +- 7 files changed, 505 insertions(+), 6 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go new file mode 100644 index 000000000..6c70dbc61 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go @@ -0,0 +1,277 @@ +package main + +// The delivery record's consumer (design 47 *The delivery record*: "The outcomes arrive as events, through a durable +// consumer of the gate's own"). The gate's manifest `consumes` each step event of *What each step says* and the waits' +// `wait-held` and `wait-ended`; the runtime binds the module's durable consumer for them (stdio.Subscribe), which resumes +// from the last event the gate took after a restart, and offers an event again until the gate took it. The gate takes an +// event only once its outcome is in every record that holds it, so an event the ledger could not take is offered again; +// one applied twice changes nothing (Records.Outcome); and one is applied only when it comes from its owner's account: +// the event's key is its owner and its event recovered from the subject, which the bus grants its owner's account alone +// (ADR 0363 §2), and an event a node publishes names that node in its subject and in its x-node header. + +import ( + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// StepOutcome is what one step event says of one entry, or of the record's merge when it names no module: the merge +// commits it is for (an event of the whole change), or the entry's module, fingerprint and node. +type StepOutcome struct { + Commits []string + Module string + Fingerprint string + Node string + Step string + Outcome string + Why string + At time.Time +} + +// stepFamily is one family of events the record is built from: the pattern its manifest consumes it by, its owner +// (the seat or module whose account alone publishes it), whether its subject names the node after the event, and how +// its body reads as outcomes. +type stepFamily struct { + Pattern string + Owner string + // Event is the event's name after its owner; the node is the token after it when NodeScoped. + Event string + NodeScoped bool + Decode func(body json.RawMessage, at time.Time) ([]StepOutcome, error) +} + +// The step events on main the record is built from. A family is added here, with its `consumes` line, when its owner +// publishes it (design 47 *What each step says*); each other step's event is its owner's to add in the same milestone. +var stepFamilies = []stepFamily{ + // An entry taken in (the indexer's own `take-in.taken`, ADR 0633 §1): the merges a source revision took in. + {Pattern: indexerModule + "." + takein.TakeInTakenEvent, Owner: indexerModule, Event: takein.TakeInTakenEvent, Decode: decodeTakenIn}, +} + +// indexerModule is the holder of the seat mesh-catalog whose own events the record hears. +const indexerModule = "mesh-delivery-indexer" + +// stepPatterns are the patterns of every family, in the order the manifest consumes them. +func stepPatterns() []string { + out := make([]string, 0, len(stepFamilies)) + for _, f := range stepFamilies { + out = append(out, f.Pattern) + } + return out +} + +// decodeTakenIn reads a `take-in.taken`: each merge taken in is an outcome of the record that merged it, unless the wait +// was dropped by hand, which took nothing in. +func decodeTakenIn(body json.RawMessage, at time.Time) ([]StepOutcome, error) { + t, err := takein.DecodeTakeIn(body) + if err != nil { + return nil, err + } + if t.Kind != takein.TakeInTaken { + return nil, fmt.Errorf("a take-in.taken of kind %q", t.Kind) + } + outcome, why := OutcomeTakenIn, t.Why + if t.Dropped { + outcome, why = OutcomeHeld, "its take-in was dropped: "+t.Why + } + var commits []string + for _, m := range t.Merges { + commits = append(commits, m.Merge) + } + if len(commits) == 0 { + return nil, nil + } + return []StepOutcome{{Commits: commits, Step: "take-in", Outcome: outcome, Why: why, At: at}}, nil +} + +// StepConsumer applies the step events to the records. +type StepConsumer struct { + Records *Records + Families []stepFamily + Now func() time.Time + Logf func(format string, a ...any) + + mu sync.Mutex + // heard counts, per family, the events applied, refused and the last applied, since this owner started. + heard map[string]*familyHeard +} + +type familyHeard struct { + Applied int `json:"applied"` + Refused int `json:"refused"` + Last time.Time `json:"last,omitzero"` + Why string `json:"last-refused,omitempty"` +} + +func (c *StepConsumer) now() time.Time { + if c.Now == nil { + return time.Now().UTC() + } + return c.Now().UTC() +} + +func (c *StepConsumer) logf(format string, a ...any) { + if c.Logf != nil { + c.Logf(format, a...) + } +} + +// familyOf is the family an event's key belongs to. +func (c *StepConsumer) familyOf(key string) (stepFamily, bool) { + for _, f := range c.Families { + if stdio.TopicMatches(f.Pattern, key) || (f.NodeScoped && strings.HasPrefix(key, f.Owner+"."+f.Event+".")) { + return f, true + } + } + return stepFamily{}, false +} + +// Takes is whether the consumer takes events of this key. +func (c *StepConsumer) Takes(key string) bool { _, ok := c.familyOf(key); return ok } + +// Hear applies one event. It answers an error only when an outcome could not be written, so the event is offered +// again; an event of no family, from another account than its owner's, or that cannot be read is said and taken, since +// offered again it would be refused again. +func (c *StepConsumer) Hear(e stdio.Envelope) error { + f, ok := c.familyOf(e.Key) + if !ok { + return nil + } + if why := ownerRefusal(f, e); why != "" { + c.count(f, false, why) + c.logf("[mesh-delivery] the record does not take %s: %s", e.Key, why) + return nil + } + at := c.now() + outcomes, err := f.Decode(e.Body, at) + if err != nil { + why := fmt.Sprintf("its body is not readable: %v", err) + c.count(f, false, why) + c.logf("[mesh-delivery] the record does not take %s: %s", e.Key, why) + return nil + } + id := e.Headers["x-event-id"] + for _, o := range outcomes { + if f.NodeScoped && o.Node == "" { + o.Node = e.Node + } + if err := c.apply(o, id); err != nil { + return fmt.Errorf("the outcome of %s is not recorded yet: %w", e.Key, err) + } + } + c.count(f, true, "") + return nil +} + +// ownerRefusal is why an event did not come from its owner's account, or "": its key names its owner, and an event of a +// node names the node its x-node header says. +func ownerRefusal(f stepFamily, e stdio.Envelope) string { + rest, ok := strings.CutPrefix(e.Key, f.Owner+".") + if !ok { + return fmt.Sprintf("it is not %s's own event", f.Owner) + } + if !f.NodeScoped { + return "" + } + after, ok := strings.CutPrefix(rest, f.Event+".") + node, _, _ := strings.Cut(after, ".") + if !ok || node == "" { + return "its subject names no node" + } + if e.Node != node { + return fmt.Sprintf("its subject names the node %s and it came from %q", node, e.Node) + } + return "" +} + +// apply writes an outcome into every open record that holds it: by its merge commits, or by its entry. +func (c *StepConsumer) apply(o StepOutcome, event string) error { + records, err := c.Records.Current() + if err != nil { + return err + } + for _, r := range records { + if r.Ended != nil || !holds(r, o) { + continue + } + ro := RecordOutcome{Module: o.Module, Fingerprint: o.Fingerprint, Node: o.Node, Step: o.Step, Outcome: o.Outcome, + Why: o.Why, At: o.At, Event: event} + if _, err := c.Records.Outcome(r, ro); err != nil { + return err + } + } + return nil +} + +// holds is whether a record is the one an outcome is for: one of its members merged as a commit the outcome names, or +// its difference holds the entry. +func holds(r *Record, o StepOutcome) bool { + if len(o.Commits) > 0 { + for _, m := range r.mergedAs() { + for _, c := range o.Commits { + if sameCommit(m, c) { + return true + } + } + } + return false + } + _, ok := r.Entries[EntryKey(o.Module, o.Fingerprint, o.Node)] + if !ok && o.Node != "" { + _, ok = r.Entries[EntryKey(o.Module, o.Fingerprint, "")] + } + return ok +} + +// mergedAs are the commits the record's members merged as. +func (r *Record) mergedAs() []string { + if r.Group == nil { + return nil + } + var out []string + for _, p := range r.Group.Pulls { + if p.Commit != "" { + out = append(out, p.Commit) + } + } + return out +} + +func (c *StepConsumer) count(f stepFamily, applied bool, why string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.heard == nil { + c.heard = map[string]*familyHeard{} + } + h := c.heard[f.Pattern] + if h == nil { + h = &familyHeard{} + c.heard[f.Pattern] = h + } + if applied { + h.Applied++ + h.Last = c.now() + } else { + h.Refused++ + h.Why = why + } +} + +// Heard is, per family, what the consumer took since this owner started. +func (c *StepConsumer) Heard() map[string]familyHeard { + c.mu.Lock() + defer c.mu.Unlock() + out := map[string]familyHeard{} + for _, f := range c.Families { + if h := c.heard[f.Pattern]; h != nil { + out[f.Pattern] = *h + } else { + out[f.Pattern] = familyHeard{} + } + } + return out +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 5c91bee57..470b68fde 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -40,6 +40,8 @@ type Holder struct { Owed OwedStore // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. Records *Records + // Consumer applies the step events to the records (consumer.go); nil applies none. + Consumer *StepConsumer // Pulls keeps the pull requests' keys on the bus (pullkey.go); nil keeps none. Pulls PullKeys // Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 2d150687a..939097202 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -32,9 +32,10 @@ func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() } func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now } // The events this owner follows: the forge's pull requests. The forge's status events say that a check answered (events.go). -var followed = []string{"gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed", +// The step events the delivery record is built from follow (consumer.go). +var followed = append([]string{"gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed", "gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed", - forgePattern, ConfigChangedEvent, writableEvent} + forgePattern, ConfigChangedEvent, writableEvent}, stepPatterns()...) // forgePattern is the forge bridge's messages on FORGE (novox/hq ADR 0437 §8): the gate follows a merge and the refusal of one. const forgePattern = "mesh-delivery-forge-bridge.forge.**" @@ -74,6 +75,7 @@ func main() { } h.init() h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention} + h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf} l := &listening{now: "not yet: starting"} go run(h, l) if err := announceKept(stdio.Keeps); err != nil { @@ -186,6 +188,9 @@ func handle(h *Holder, e stdio.Envelope) error { } func take(h *Holder, e stdio.Envelope) error { + if h.Consumer != nil && h.Consumer.Takes(e.Key) { + return h.Consumer.Hear(e) + } if strings.HasPrefix(e.Key, "mesh-delivery-forge-bridge.forge.") { h.Forged(e.Key, e.Body) return nil diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index b838552d2..aae76a812 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -18,6 +18,7 @@ import ( "regexp" "sort" "strings" + "sync" "time" stdio "git.novox.be/novox/mesh-sdk/go" @@ -45,6 +46,7 @@ const ( OpOpen = "open" // set-fields: the milestone, the heads, when, forge-made, the form OpGroup = "group" // set-fields: the group's state OpTrace = "trace" // set-fields: the trace + OpEntry = "entry" // set-entry into entries: one entry of the graph difference, by its module, fingerprint and node OpOutcome = "outcome" // set-entry into outcomes, by the entry, the node and the step OpAway = "away" // set-entry into away, by node OpBack = "back" // remove from away, by node @@ -57,7 +59,7 @@ const ( ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpEnd, OpRemove} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpEnd, OpRemove} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -84,6 +86,8 @@ type Record struct { Group *RecordGroup `json:"group,omitempty"` Trace *RecordTrace `json:"trace,omitempty"` + // Entries are the entries of the graph difference the record delivers, by EntryKey. + Entries map[string]RecordEntry `json:"entries,omitempty"` // Outcomes are what happened to each entry on each node, by OutcomeKey. Outcomes map[string]RecordOutcome `json:"outcomes,omitempty"` // Away are the nodes away, by node, each with since when. @@ -152,6 +156,20 @@ type RecordOutcome struct { Event string `json:"event,omitempty"` } +// RecordEntry is one entry of the graph difference: a module at a fingerprint, on a node (none for the catalog half's +// entry, which no node runs). +type RecordEntry struct { + Key string `json:"key"` + Module string `json:"module"` + Fingerprint string `json:"fingerprint"` + Node string `json:"node,omitempty"` +} + +// EntryKey is the key of an entry: its module, its fingerprint and its node. +func EntryKey(module, fingerprint, node string) string { + return strings.Join([]string{module, fingerprint, node}, "|") +} + // OutcomeKey is the key of an entry's outcome: its module, its fingerprint, its node and its step, each a token. func OutcomeKey(module, fingerprint, node, step string) string { return strings.Join([]string{module, fingerprint, node, step}, "|") @@ -305,6 +323,67 @@ type Records struct { Retention func() time.Duration // Freed is told each revision a removed record pinned, to say as `freed` (design 47); nil says none. Freed func(r *Record, p PinnedRevision) + + // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the + // gate is their one writer. + mu sync.Mutex + held map[string]*Record + loaded bool +} + +// Current is every record kept and not ended, oldest first: read from the ledger at the first call, then as the gate +// writes them. +func (rs *Records) Current() ([]*Record, error) { + if err := rs.load(); err != nil { + return nil, err + } + rs.mu.Lock() + defer rs.mu.Unlock() + var out []*Record + for _, r := range rs.held { + if r.Ended == nil { + out = append(out, r) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Opened.Before(out[j].Opened) }) + return out, nil +} + +func (rs *Records) load() error { + rs.mu.Lock() + loaded := rs.loaded + rs.mu.Unlock() + if loaded { + return nil + } + list, err := rs.List() + if err != nil { + return err + } + rs.mu.Lock() + defer rs.mu.Unlock() + if !rs.loaded { + rs.held = map[string]*Record{} + for _, r := range list { + rs.held[r.Instance] = r + } + rs.loaded = true + } + return nil +} + +func (rs *Records) keep(r *Record) { + rs.mu.Lock() + defer rs.mu.Unlock() + if rs.loaded { + rs.held[r.Instance] = r + } +} + +func (rs *Records) forget(instance string) { + rs.mu.Lock() + defer rs.mu.Unlock() + delete(rs.held, instance) } func (rs *Records) now() time.Time { @@ -368,6 +447,9 @@ func (rs *Records) Open(milestone string, heads map[string]string, forgeMade boo return nil, false, fmt.Errorf("the record %s is opened but not listed: %w", inst, err) } r, err = rs.Read(inst) + if err == nil && r != nil { + rs.keep(r) + } return r, err == nil && r != nil, err } @@ -443,6 +525,7 @@ func (rs *Records) Sweep(now time.Time) error { if _, err := rs.Ledger.Apply(e.Instance, OpRemove, map[string]any{}); err != nil { return err } + rs.forget(e.Instance) if rs.Freed != nil { for _, p := range sortedPins(r) { rs.Freed(r, p) @@ -482,8 +565,8 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { // Outcome writes one entry's outcome on one node, unless the record holds it already: an event applied twice changes // nothing (design 47). Applied says it was written now. func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) { - if o.Module == "" || o.Step == "" || o.Outcome == "" { - return false, fmt.Errorf("an outcome names its module, its step and what happened (%+v)", o) + if o.Step == "" || o.Outcome == "" { + return false, fmt.Errorf("an outcome names its step and what happened (%+v)", o) } o.Key = OutcomeKey(o.Module, o.Fingerprint, o.Node, o.Step) if had, ok := r.Outcomes[o.Key]; ok && (had.Event != "" && had.Event == o.Event || sameOutcome(had, o)) { @@ -500,6 +583,23 @@ func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) return true, nil } +// AddEntry writes one entry of the graph difference into the record, unless it holds it. +func (rs *Records) AddEntry(r *Record, e RecordEntry) error { + e.Key = EntryKey(e.Module, e.Fingerprint, e.Node) + if had, ok := r.Entries[e.Key]; ok && had == e { + return nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpEntry, e) + if err != nil { + return err + } + if r.Entries == nil { + r.Entries = map[string]RecordEntry{} + } + r.Entries[e.Key], r.Sequence = e, seq + return nil +} + // sameOutcome is whether two outcomes say the same, whenever and from whichever event they were taken. func sameOutcome(a, b RecordOutcome) bool { a.At, b.At, a.Event, b.Event = time.Time{}, time.Time{}, "", "" diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index b101b362f..3d1ac5f5c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -303,3 +304,102 @@ func TestTheManifestsLedgerTypesAreTheRecordsOperations(t *testing.T) { t.Error("the records ledger is not capped at 768 MiB") } } + +// W11.2 (design 47 *The delivery record*): the record's outcomes arrive as events through the gate's consumer: each +// applied once, only from its owner's account (its key's owner, and for a node's event the node its subject names), an +// unreadable one taken and said, and one the ledger could not take offered again. +func TestTheRecordsOutcomesArriveThroughTheGatesConsumer(t *testing.T) { + at := time.Date(2026, 11, 9, 21, 0, 0, 0, time.UTC) + merge := strings.Repeat("a", 40) + setup := func(t *testing.T) (*ledgerFake, *Records, *Record, *StepConsumer) { + l := newLedgerFake(t) + rs := &Records{Ledger: l, Now: func() time.Time { return at }} + r, _, err := rs.Open("47/G", map[string]string{"novox/app@111111111111": strings.Repeat("1", 40)}, false) + if err != nil { + t.Fatal(err) + } + if err := rs.SetGroup(r, RecordGroup{Group: "feat/x", Pulls: []RecordPull{{Repository: "novox/app", Number: 7, + Head: strings.Repeat("1", 40), State: "merged", Commit: merge}}}); err != nil { + t.Fatal(err) + } + if err := rs.AddEntry(r, RecordEntry{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace"}); err != nil { + t.Fatal(err) + } + running := stepFamily{Pattern: "node-engine.running.*.*", Owner: "node-engine", Event: "running", NodeScoped: true, + Decode: func(body json.RawMessage, at time.Time) ([]StepOutcome, error) { + var b struct{ Module, Fingerprint string } + if err := json.Unmarshal(body, &b); err != nil { + return nil, err + } + return []StepOutcome{{Module: b.Module, Fingerprint: b.Fingerprint, Step: "run", Outcome: OutcomeRunning, At: at}}, nil + }} + c := &StepConsumer{Records: rs, Families: append(slices.Clone(stepFamilies), running), Now: func() time.Time { return at }} + return l, rs, r, c + } + taken := func(id string) stdio.Envelope { + body := `{"v":1,"kind":"taken","key":"m.00000000000000000001","what":"revision","merges":[{"repository":"novox/app","number":7,"merge":"` + + merge + `"}],"since":"2026-11-09T20:00:00Z"}` + return stdio.Envelope{Key: "mesh-delivery-indexer.take-in.taken", Body: json.RawMessage(body), Headers: map[string]string{"x-event-id": id}} + } + t.Run("a take-in of the record's merge is its outcome, once", func(t *testing.T) { + l, rs, r, c := setup(t) + if err := c.Hear(taken("e1")); err != nil { + t.Fatal(err) + } + n := len(l.history) + if err := c.Hear(taken("e1")); err != nil || len(l.history) != n { + t.Fatalf("the same event again: %v, %d operations (was %d)", err, len(l.history), n) + } + back, _ := rs.Read(r.Instance) + if got := back.Outcomes[OutcomeKey("", "", "", "take-in")]; got.Outcome != OutcomeTakenIn || got.Event != "e1" { + t.Fatalf("outcomes %+v", back.Outcomes) + } + if c.Heard()["mesh-delivery-indexer.take-in.taken"].Applied != 2 { + t.Fatalf("heard %+v", c.Heard()) + } + }) + t.Run("a node's event from its own node is the entry's outcome; from another node it is refused", func(t *testing.T) { + _, rs, r, c := setup(t) + e := stdio.Envelope{Key: "node-engine.running.ace.mesh-x", Node: "g14", Body: json.RawMessage(`{"module":"mesh-x","fingerprint":"src2:f1"}`)} + if err := c.Hear(e); err != nil { + t.Fatal(err) + } + if back, _ := rs.Read(r.Instance); len(back.Outcomes) != 0 { + t.Fatalf("an event naming ace from g14 is applied: %+v", back.Outcomes) + } + if h := c.Heard()["node-engine.running.*.*"]; h.Refused != 1 || !strings.Contains(h.Why, "names the node ace") { + t.Fatalf("heard %+v", h) + } + e.Node = "ace" + if err := c.Hear(e); err != nil { + t.Fatal(err) + } + back, _ := rs.Read(r.Instance) + if got := back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "ace", "run")]; got.Outcome != OutcomeRunning { + t.Fatalf("outcomes %+v", back.Outcomes) + } + }) + t.Run("an unreadable event is taken and said; one the ledger cannot take is offered again", func(t *testing.T) { + l, _, _, c := setup(t) + bad := taken("e2") + bad.Body = json.RawMessage(`{"v":1,"kind":"pending"}`) + if err := c.Hear(bad); err != nil { + t.Fatalf("an unreadable event is offered again: %v", err) + } + if h := c.Heard()["mesh-delivery-indexer.take-in.taken"]; h.Refused != 1 { + t.Fatalf("heard %+v", h) + } + l.failOn = OpOutcome + if err := c.Hear(taken("e3")); err == nil { + t.Fatal("an outcome the ledger did not take is taken") + } + }) + t.Run("the manifest consumes every family", func(t *testing.T) { + m, _ := readManifest(t) + for _, f := range stepFamilies { + if !slices.Contains(m.Consumes, f.Pattern) { + t.Errorf("the manifest does not consume %s", f.Pattern) + } + } + }) +} diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 1ff0a9a76..11b37039e 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -9,6 +9,7 @@ require ( git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/sdk v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 0c48589f9..d7aa47e5f 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -28,7 +28,8 @@ "gitea.forge.pull-request.closed", "mesh-delivery-forge-bridge.forge.**", "mesh-config.config-changed", - "gitea.repo.writable" + "gitea.repo.writable", + "mesh-delivery-indexer.take-in.taken" ], "emits": [ "transition", @@ -140,6 +141,19 @@ ], "sequence": true }, + { + "name": "entry", + "effect": "set-entry", + "fields": [ + "key", + "module", + "fingerprint", + "node" + ], + "into": "entries", + "key": "key", + "sequence": true + }, { "name": "outcome", "effect": "set-entry", -- 2.54.0 From 11daf2f289cdd04552725cdac29d84fae9aa4f3d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:35:57 +0100 Subject: [PATCH 041/243] The switch carries every group in flight into its delivery record, once (milestone 47/G, W11.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 47 (The switch carries today's state in, once): at the gate's start, before any event is taken, each group in flight becomes a record keyed by its milestone and its heads (the heads its merge was approved at when merges holds one), with that merge as its group state; a second switch finds each record there and opens none, so a restart or a delivery of milestone G again after a revert asks nothing again. From then on every merge the gate hands is written into its record too, so the record's outcomes are found by its members' merge commits. Left for W11.17 and W11.14: the merge handed as an owed merge-pull instead of the merges key, the owed keys carried, and merges and turns no longer declared. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from e9415814.) Builds: design 47 G §156, §220 (GATE4) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/main.go | 17 +++ .../cmd/mesh-delivery-gate/record_test.go | 46 ++++++ .../cmd/mesh-delivery-gate/switch.go | 141 ++++++++++++++++++ 3 files changed, 204 insertions(+) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 939097202..e463c1359 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -75,6 +75,7 @@ func main() { } h.init() h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention} + h.Merges = recordedMerges{MergeStore: kvMerges{}, records: h.Records, members: h.groupMembers} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf} l := &listening{now: "not yet: starting"} go run(h, l) @@ -95,6 +96,7 @@ func main() { func run(h *Holder, l *listening) { <-stdio.Served() // Serve first: the state is reached through it, and an event is heard from as soon as it serves loadKept(h, l) + switchToRecords(h, l) go h.Work() for _, pattern := range followed { pattern := pattern @@ -171,6 +173,21 @@ func loadKept(h *Holder, l *listening) { } } +// switchToRecords carries the groups in flight into their delivery records (switch.go), retrying until it can: no event is +// taken before, so no outcome reaches a record the switch has not opened yet. +func switchToRecords(h *Holder, l *listening) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + opened, found, err := h.Switch() + if err == nil { + logf("[mesh-delivery] the switch to the delivery records: %d opened, %d already there", opened, found) + return + } + l.set("not yet: " + err.Error()) + logf("[mesh-delivery] the switch to the delivery records cannot be made yet (%v); asking again in %s", err, wait) + time.Sleep(wait) + } +} + // readOnStart reads once what happened while the gate was away: the forge's open pull requests (heads nobody checks) are // read by the first look, and the asks the router closed meanwhile are read here. func readOnStart(h *Holder) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 3d1ac5f5c..c1b11d87a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -403,3 +404,48 @@ func TestTheRecordsOutcomesArriveThroughTheGatesConsumer(t *testing.T) { } }) } + +// W11.3 (design 47 *The switch carries today's state in, once*): at the gate's start every group in flight becomes a +// record at its heads; a second switch (a restart, or milestone G delivered again after a revert) opens none and asks +// nothing; and from then on every merge the gate hands is in its record, whose merge commits the record's outcomes +// are found by. +func TestTheSwitchCarriesTheGroupsInFlightIntoRecordsOnce(t *testing.T) { + w := approvedGroup(t) + l := newLedgerFake(t) + w.h.Records = &Records{Ledger: l, Now: w.h.Now} + w.h.Merges = recordedMerges{MergeStore: w.merges, records: w.h.Records, members: w.h.groupMembers} + opened, found, err := w.h.Switch() + if err != nil || opened != 1 || found != 0 { + t.Fatalf("the first switch: %d opened, %d found, %v", opened, found, err) + } + asked := w.op.count() + opened, found, err = w.h.Switch() + if err != nil || opened != 0 || found != 1 || w.op.count() != asked { + t.Fatalf("the switch again: %d opened, %d found, %v, %d asks (was %d)", opened, found, err, w.op.count(), asked) + } + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.settleAll() + records, err := w.h.Records.Current() + if err != nil || len(records) != 1 { + t.Fatalf("records %+v %v: the handed merge opened another", records, err) + } + r := records[0] + if r.Milestone != "feat/g" || r.Group == nil || r.Group.Ask == "" || len(r.Group.Pulls) != 2 || len(r.Group.Members) != 2 { + t.Fatalf("the record's group state %+v", r.Group) + } + for _, p := range r.Group.Pulls { + if p.State != string(mdc.MergeMerged) || p.Commit == "" { + t.Fatalf("a member's merge is not in the record: %+v", p) + } + } + c := &StepConsumer{Records: w.h.Records, Families: stepFamilies, Now: w.h.Now} + body := `{"v":1,"kind":"taken","key":"m.00000000000000000001","what":"revision","merges":[{"repository":"` + gCat + + `","number":2,"merge":"` + r.Group.Pulls[1].Commit + `"}],"since":"2026-11-09T20:00:00Z"}` + if err := c.Hear(stdio.Envelope{Key: "mesh-delivery-indexer.take-in.taken", Body: json.RawMessage(body)}); err != nil { + t.Fatal(err) + } + if back, _ := w.h.Records.Read(r.Instance); back.Outcomes[OutcomeKey("", "", "", "take-in")].Outcome != OutcomeTakenIn { + t.Fatalf("the take-in of the group's merge is not its record's outcome: %+v", back.Outcomes) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go new file mode 100644 index 000000000..02f9893bc --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go @@ -0,0 +1,141 @@ +package main + +// The switch to the delivery record (design 47 *The delivery record*: "The switch carries today's state in, once"): at +// the new gate's start, every group in flight and its merge as `merges` holds it become records, keyed by the +// milestone and the heads, and from then on every merge the gate hands is written into its record. It is safe to run +// twice: a group in flight whose record is there already leaves it as it is (Records.Open), and a group state the +// record holds already is not written again (Records.SetGroup), so delivering milestone G again after a revert makes no +// second record and asks nothing again. + +import ( + "fmt" + "sort" + + mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" +) + +// milestoneOf is the milestone a delivery's record is kept under: the name its group's Approve is asked and its merge is +// handed by (askGroup), the one name the gate gives a group's delivery today. +func milestoneOf(d *Delivery) string { return askGroup(d) } + +// recordGroupOf is a merge, as `merges` holds it, as the record's group state. +func recordGroupOf(m Merge, members []string) RecordGroup { + g := RecordGroup{Group: m.Group, Members: members, Ask: m.Ask, Approved: m.Approved} + for _, p := range m.Pulls { + g.Pulls = append(g.Pulls, RecordPull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, State: string(p.State), + Commit: p.Commit, Words: p.Words}) + } + return g +} + +// recordedMerges is the gate's merges with every merge it writes also written into its delivery record: the record holds +// the group's state, which `merges` holds today. +type recordedMerges struct { + MergeStore + records *Records + members func(group string) []string +} + +func (m recordedMerges) Put(group string, v Merge) error { + if err := m.MergeStore.Put(group, v); err != nil { + return err + } + if m.records == nil || len(v.Approved) == 0 { + return nil + } + r, _, err := m.records.Open(group, v.Approved, false) + if err != nil { + return fmt.Errorf("the merge of %s is handed but not in its delivery record: %w", group, err) + } + var members []string + if m.members != nil { + members = m.members(group) + } + return m.records.SetGroup(r, recordGroupOf(v, members)) +} + +// switchGroup is one group in flight as the switch finds it: its milestone, its heads and its members. +type switchGroup struct { + milestone string + heads map[string]string + members []string +} + +// inFlight are the groups in flight, under the lock: every milestone with a delivery not final, its live members' heads +// as `@` to the full commit. +func (h *Holder) inFlight() []switchGroup { + by := map[string]*switchGroup{} + for _, d := range h.deliveries { + if d.State.Final() && !(d.State == Merged && !d.InSource) || d.Number == 0 || d.NewerHead != "" || d.ClosedUnmerged { + continue + } + name := milestoneOf(d) + g := by[name] + if g == nil { + g = &switchGroup{milestone: name, heads: map[string]string{}} + by[name] = g + } + g.heads[mdc.ApprovedKey(d.Repository, d.Number)] = d.Commit + g.members = append(g.members, d.ID) + } + out := make([]switchGroup, 0, len(by)) + for _, g := range by { + sort.Strings(g.members) + out = append(out, *g) + } + sort.Slice(out, func(i, j int) bool { return out[i].milestone < out[j].milestone }) + return out +} + +// Switch carries today's state into the records, once and safe to run again: each group in flight becomes a record at +// its heads, or at the heads its merge was approved at when `merges` holds one, with that merge as its group state. +// It answers how many records it opened and how many it found there. +func (h *Holder) Switch() (opened, found int, err error) { + if h.Records == nil { + return 0, 0, nil + } + h.mu.Lock() + groups := h.inFlight() + h.mu.Unlock() + for _, g := range groups { + var merge *Merge + if h.Merges != nil { + if merge, err = h.Merges.Get(g.milestone); err != nil { + return opened, found, fmt.Errorf("the merge of %s cannot be read for the switch: %w", g.milestone, err) + } + } + heads := g.heads + if merge != nil && len(merge.Approved) > 0 { + heads = merge.Approved + } + r, made, err := h.Records.Open(g.milestone, heads, false) + if err != nil { + return opened, found, fmt.Errorf("the switch cannot open the record of %s: %w", g.milestone, err) + } + if made { + opened++ + } else { + found++ + } + if merge != nil { + if err := h.Records.SetGroup(r, recordGroupOf(*merge, g.members)); err != nil { + return opened, found, fmt.Errorf("the switch cannot carry the merge of %s: %w", g.milestone, err) + } + } + } + return opened, found, nil +} + +// groupMembers are the members' ids of the milestone a merge is handed under, under the lock. +func (h *Holder) groupMembers(name string) []string { + h.mu.Lock() + defer h.mu.Unlock() + var out []string + for _, d := range h.deliveries { + if milestoneOf(d) == name && d.NewerHead == "" && !d.ClosedUnmerged { + out = append(out, d.ID) + } + } + sort.Strings(out) + return out +} -- 2.54.0 From 30a13c4ebb61e8a0df7dbba23a71a978beb12f78 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:35:58 +0100 Subject: [PATCH 042/243] The gate reads and hears the forge through the seat git (milestone 47/G, W11.18) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate named the forge module's tools and events and guessed their answers' shape (operator (d), F14). Its reads now are the seat git's verbs pull, statuses, open-pulls and contains, typed in the seat's contract, a failed read an error and never none; its forge events are the seat's pull.updated, pull.merged, pull.closed, forge.pull, forge.status and repo.writable, read into the gate's own forms in one place (gitseat.go). What the forge would merge now (the base's protection, its conflict word, whether the merging account writes) has no verb of the seat yet: it stays the one read of the forge's statuses tool, named as such (mergeStateTool), until the seat carries it. The bus tests' runtime now serves the gate's ledger, so the two whole-bundle tests that failed since W11.1 pass again. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 14338514.) Builds: design 47 G §252, §310, §312, test 51 (GATE14) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bus_test.go | 65 ++++++-- .../cmd/mesh-delivery-gate/checkerbus_test.go | 15 +- .../cmd/mesh-delivery-gate/events.go | 2 +- .../cmd/mesh-delivery-gate/events_test.go | 4 +- .../cmd/mesh-delivery-gate/fakes_test.go | 41 +++++ .../cmd/mesh-delivery-gate/gitseat.go | 153 ++++++++++++++++++ .../cmd/mesh-delivery-gate/gitseat_test.go | 104 ++++++++++++ .../cmd/mesh-delivery-gate/holder.go | 4 +- .../cmd/mesh-delivery-gate/issue1107_test.go | 14 +- .../cmd/mesh-delivery-gate/main.go | 52 +++--- .../cmd/mesh-delivery-gate/manifest_test.go | 8 +- .../cmd/mesh-delivery-gate/merges.go | 2 +- .../cmd/mesh-delivery-gate/model.go | 2 +- .../mesh-delivery-gate/onestatus_bus_test.go | 8 +- .../cmd/mesh-delivery-gate/ports.go | 143 ++++++++++------ .../cmd/mesh-delivery-gate/pullkey.go | 2 +- .../cmd/mesh-delivery-gate/unchecked.go | 4 +- .../unchecked_events_test.go | 18 ++- .../cmd/mesh-delivery-gate/writes.go | 23 ++- .../cmd/mesh-delivery-gate/writes_test.go | 4 +- modules/mesh-delivery-gate/go.mod | 1 + modules/mesh-delivery-gate/module.json | 21 +-- 22 files changed, 545 insertions(+), 145 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index a9a2847a8..428e57a4a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -3,6 +3,10 @@ package main import ( + "git.novox.be/novox/mesh-sdk/go/buscalls" + + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "bufio" "context" "encoding/json" @@ -95,7 +99,7 @@ func newTestBus(t *testing.T) *testBus { } // The streams the events are on (mesh.mod..event., mesh.seat..event.). if _, err := js.CreateStream(context.Background(), jetstream.StreamConfig{Name: "EVENTS", - Subjects: []string{"mesh.mod.gitea.event.>", "mesh.seat." + oc.Seat + ".event.>", + Subjects: []string{"mesh.seat.git.event.>", "mesh.seat." + oc.Seat + ".event.>", "mesh.mod." + forgeBridgeModule + ".event.>"}}); err != nil { t.Fatal(err) } @@ -112,6 +116,8 @@ type runtime struct { next int wait map[string]chan json.RawMessage + // the gate's ledger, as the tool runner keeps it. + ledger *ledgerFake // what the bundle subscribed to, and asked of the mesh. subs []string asked []askedOfMesh @@ -345,9 +351,27 @@ func (r *runtime) answer(m rpcMessage) { switch a.Key { case "": return nil, errors.New("an ask with no key") - case "gitea.gitea_open_pulls": - return toolAnswer(map[string]any{"pulls": pulls}), nil - case "gitea.gitea_commit_statuses": + case "seat:git.open-pulls": + var open []gitseat.OpenPull + for _, p := range pulls { + open = append(open, seatOpenPull(p)) + } + return gitseat.OpenPullsAnswer{Pulls: open}, nil + case "seat:git.pull": + number, _ := a.Body["number"].(float64) + repository, _ := a.Body["repository"].(string) + for _, p := range pulls { + if p.Owner+"/"+p.Repo == repository && p.Number == int(number) { + return gitseat.PullAnswer{Pull: &gitseat.Pull{Repository: repository, Number: int64(p.Number), Base: p.Base, + Head: p.HeadSHA, State: gitseat.PullStateOpen}}, nil + } + } + return gitseat.PullAnswer{Error: "no such pull request"}, nil + case "seat:git.statuses": + // Green: mesh/check a success at the head asked, as the seat answers. + commit, _ := a.Body["commit"].(string) + return gitseat.StatusesAnswer{Commit: commit, Statuses: []gitseat.Status{{Context: "mesh/check", State: "success"}}}, nil + case mergeStateTool: // Green: mesh/check a success at the head asked, as the gitea module answers. sha, _ := a.Body["sha"].(string) if sha == "" { @@ -367,6 +391,29 @@ func (r *runtime) answer(m rpcMessage) { } return nil, fmt.Errorf("the runtime answers no ask %s in this test", a.Key) } + switch m.Method { + case buscalls.LedgerApply, buscalls.LedgerGet: + // The gate's ledger `records` (W11.1), as the tool runner keeps it: the record test's ledger in memory. + r.mu.Lock() + defer r.mu.Unlock() + if r.ledger == nil { + r.ledger = newLedgerFake(r.t) + } + if m.Method == buscalls.LedgerGet { + var in buscalls.InstanceRequest + if err := json.Unmarshal(m.Params, &in); err != nil { + return nil, err + } + state, seq, err := r.ledger.Get(in.Instance) + return buscalls.InstanceState{Instance: in.Instance, State: state, Sequence: seq}, err + } + var in buscalls.ApplyRequest + if err := json.Unmarshal(m.Params, &in); err != nil { + return nil, err + } + seq, err := r.ledger.Apply(in.Instance, in.Operation, in.Body) + return buscalls.Applied{Taken: err == nil, Sequence: seq}, err + } kv := r.b.kvs[p.State] if kv == nil { return nil, fmt.Errorf("the state %q is not declared", p.State) @@ -518,7 +565,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB announced.BaseSHA = "a1a1a1a1a1a1" announced.Title = "Fix the apostrophe (issue 975)" announced.Body = "The description, with no review line yet." - b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) + b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(announced)) rt.until("the pull request's key to hold its head", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") var k PullKey @@ -576,7 +623,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB b.forgeSays(t, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"app": "t"}}) // The forge's own word that the pull request merged follows the bridge's (its webhook): the delivery ends merged. - b.publish(t, "mesh.mod.gitea.event.pull.merged", merged("novox/app", 7, head, strings.Repeat("b", 40))) + b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged("novox/app", 7, head, strings.Repeat("b", 40)))) rt.until("source.main to hold the merge", eventWithin, func() bool { e, err := b.kvs["source"].Get(ctx, mdc.SourceMainKey) if err != nil { @@ -704,7 +751,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")} { p.BaseSHA = "a1a1a1a1a1a1" p.Body = "Reviewed-by: lead@shanks at " + p.HeadSHA - b.publish(t, "mesh.mod.gitea.event.pull.updated", p) + b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(p)) } // Both members pass at the checker, whose one verdict covers both heads: the group is asked once, no composed check is asked // of the controller (issue 1318). @@ -761,8 +808,8 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe // The forge's own word that each pull request merged follows the bridge's (its webhook), after the second was handed (the // checker judges the members still open by the trunk the first merge moved, which this test does not play): both members // end merged, the group with them, and the controller was never called (R6-C3). - b.publish(t, "mesh.mod.gitea.event.pull.merged", merged(one.Repository, one.Number, one.Head.Head, strings.Repeat("a", 40))) - b.publish(t, "mesh.mod.gitea.event.pull.merged", merged(two.Repository, two.Number, two.Head.Head, strings.Repeat("b", 40))) + b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged(one.Repository, one.Number, one.Head.Head, strings.Repeat("a", 40)))) + b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged(two.Repository, two.Number, two.Head.Head, strings.Repeat("b", 40)))) for _, id := range []string{IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead)} { id := id rt.until(id+" to be merged", eventWithin, func() bool { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go index daf800108..c4ab1125b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go @@ -3,6 +3,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "context" "encoding/json" "testing" @@ -52,16 +54,7 @@ func (b *testBus) checkPasses(t *testing.T, rt *runtime, group, trunk string, pu Required: []string{ctxCheck}, Statuses: []string{ctxCheck}, Checks: map[string]HeadCheck{ctxCheck: {State: "success", Description: line}}}) } rt.mu.Unlock() - b.publish(t, "mesh.mod.gitea.event.forge.status", map[string]any{"owner": owner, "repo": repo, "sha": p.head, - "context": ctxCheck, "state": "success", "description": line}) + b.publish(t, "mesh.seat.git.event.forge.status", gitseat.ForgeStatusEvent{Repository: owner + "/" + repo, Commit: p.head, + Context: ctxCheck, State: "success", Description: line}) } } - -func splitRepository(r string) (owner, repo string) { - for i := range r { - if r[i] == '/' { - return r[:i], r[i+1:] - } - } - return "", r -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go index be95a0c7e..5b3e5b39e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go @@ -19,7 +19,7 @@ type ForgeStatusEvent struct { SHA string `json:"sha"` Context string `json:"context"` State string `json:"state"` - // Description is the status's words (the gitea module's status event carries them since issue 1302): mesh/check's + // Description is the status's words (the seat git's forge.status carries them, issue 1302): mesh/check's // line names the head and the trunk it judged. Description string `json:"description,omitempty"` } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go index 45a65698c..63609fe7d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go @@ -1,6 +1,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "encoding/json" "strings" "testing" @@ -55,7 +57,7 @@ func TestAPassingVerdictAsksTheOperatorAtOnceWithNoMark(t *testing.T) { t.Fatalf("something was written on the forge to ask: %v", w.forge.calls) } // Asked once for the head, however often it is looked at again. - hear(t, w, "gitea.forge.status", ForgeStatusEvent{Owner: "novox", Repo: "app", SHA: readyHead, Context: "mesh/merge-gate", State: "success"}) + hear(t, w, gitEvent(gitseat.EventForgeStatus), gitseat.ForgeStatusEvent{Repository: "novox/app", Commit: readyHead, Context: "mesh/merge-gate", State: "success"}) if w.op.count() != 1 { t.Fatalf("asked %d times for one head", w.op.count()) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go index 9b1e1f2a1..28842c05c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go @@ -1,6 +1,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "encoding/json" "errors" "fmt" @@ -373,6 +375,45 @@ func pr(repo string, number int, sha, branch string, paths ...string) PullEvent HTMLURL: fmt.Sprintf("https://forge.invalid/%s/pulls/%d", repo, number), Paths: paths} } +// seatUpdated is a pull request's new head as the seat git says it (pull.updated). +func seatUpdated(p PullEvent) gitseat.PullUpdatedEvent { + e := gitseat.PullUpdatedEvent{Repository: p.Owner + "/" + p.Repo, Number: int64(p.Number), Base: p.Base, Branch: p.Head, + Head: p.HeadSHA, URL: p.HTMLURL, Title: p.Title, BaseHead: p.BaseSHA, Author: p.Author} + if p.Paths != nil { + e.Change = &gitseat.Change{Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs} + } + return e +} + +// seatMerged is a pull request merged as the seat git says it (pull.merged). +func seatMerged(p PullEvent) gitseat.PullMergedEvent { + e := gitseat.PullMergedEvent{Repository: p.Owner + "/" + p.Repo, Number: int64(p.Number), Base: p.Base, Branch: p.Head, + Head: p.HeadSHA, MergeCommit: p.MergeCommit, URL: p.HTMLURL, HeadChecks: p.HeadChecks} + if at, err := time.Parse(time.RFC3339, p.MergedAt); err == nil { + e.MergedAt = &at + } + if p.Paths != nil { + e.Change = &gitseat.Change{Paths: p.Paths} + } + return e +} + +// seatOpenPull is an open pull request as the seat git's open-pulls answers it. +func seatOpenPull(p OpenPull) gitseat.OpenPull { + o := gitseat.OpenPull{Pull: gitseat.Pull{Repository: p.Owner + "/" + p.Repo, Number: int64(p.Number), Base: p.Base, + Head: p.HeadSHA, URL: p.HTMLURL, State: gitseat.PullStateOpen}, Required: p.Required, Checks: map[string]gitseat.Status{}} + for context, c := range p.Checks { + o.Checks[context] = gitseat.Status{Context: context, State: c.State, Description: c.Description} + } + return o +} + +// seatClosed is a pull request closed as the seat git says it (pull.closed). +func seatClosed(p PullEvent) gitseat.PullClosedEvent { + return gitseat.PullClosedEvent{Repository: p.Owner + "/" + p.Repo, Number: int64(p.Number), Base: p.Base, Branch: p.Head, + Head: p.HeadSHA} +} + // verdict is the controller's `checked` for a head. func verdict(repo string, number int, sha, gate string) CheckedEvent { owner, name, _ := strings.Cut(repo, "/") diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat.go new file mode 100644 index 000000000..421aa7c58 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat.go @@ -0,0 +1,153 @@ +package main + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "time" + + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" +) + +// The forge as the seat `git` says it (design 47 *The seat git*): its events heard as the seat's, by the key the runtime +// hands them under (`git.`), each read into the gate's own forms here and nowhere else. + +// gitEvent is the key a seat `git` event arrives under. +func gitEvent(event string) string { return gitseat.Seat + "." + event } + +// gitEvents are the seat's events the gate hears. +var gitEvents = []string{ + gitEvent(gitseat.EventPullUpdated), gitEvent(gitseat.EventPullMerged), gitEvent(gitseat.EventPullClosed), + gitEvent(gitseat.EventForgePull), gitEvent(gitseat.EventForgeStatus), gitEvent(gitseat.EventRepoWritable), +} + +// splitRepository is a repository's owner and name. +func splitRepository(repository string) (owner, repo string) { + owner, repo, _ = strings.Cut(repository, "/") + return owner, repo +} + +// timeText is a time as the gate keeps it, RFC 3339; empty when not said. +func timeText(t *time.Time) string { + if t == nil || t.IsZero() { + return "" + } + return t.UTC().Format(time.RFC3339) +} + +// withChange carries a pull request's changed paths into the gate's form; a list of module directories absent is one the +// forge could not read. +func withChange(p *PullEvent, c *gitseat.Change) { + if c == nil { + return + } + p.Paths, p.PathsTruncated, p.Removed = c.Paths, c.PathsTruncated, c.Removed + p.ModuleDirs, p.ModuleDirsSaid = c.ModuleDirs, c.ModuleDirs != nil +} + +// gitPullEvent reads a seat `git` event of a pull request into the gate's PullEvent. The event's head is its commit and its +// branch the head branch, as the gate's Head and HeadSHA name them. +func gitPullEvent(key string, body json.RawMessage) (PullEvent, error) { + var p PullEvent + switch key { + case gitEvent(gitseat.EventPullUpdated): + var e gitseat.PullUpdatedEvent + if err := json.Unmarshal(body, &e); err != nil { + return p, err + } + p = PullEvent{Number: int(e.Number), Title: e.Title, Base: e.Base, Head: e.Branch, HeadSHA: e.Head, HTMLURL: e.URL, + BaseSHA: e.BaseHead, Author: e.Author} + p.Owner, p.Repo = splitRepository(e.Repository) + withChange(&p, e.Change) + case gitEvent(gitseat.EventPullMerged): + var e gitseat.PullMergedEvent + if err := json.Unmarshal(body, &e); err != nil { + return p, err + } + p = PullEvent{Number: int(e.Number), Title: e.Title, Base: e.Base, Head: e.Branch, HeadSHA: e.Head, HTMLURL: e.URL, + MergeCommit: e.MergeCommit, MergedAt: timeText(e.MergedAt), HeadChecks: e.HeadChecks, Merged: true, Author: e.Author} + p.Owner, p.Repo = splitRepository(e.Repository) + withChange(&p, e.Change) + case gitEvent(gitseat.EventPullClosed): + var e gitseat.PullClosedEvent + if err := json.Unmarshal(body, &e); err != nil { + return p, err + } + p = PullEvent{Number: int(e.Number), Title: e.Title, Base: e.Base, Head: e.Branch, HeadSHA: e.Head, HTMLURL: e.URL, + Author: e.Author} + p.Owner, p.Repo = splitRepository(e.Repository) + case gitEvent(gitseat.EventForgePull): + var e gitseat.ForgePullEvent + if err := json.Unmarshal(body, &e); err != nil { + return p, err + } + p = PullEvent{Number: int(e.Number), Base: e.Base, Head: e.Branch, HeadSHA: e.Head, MergeCommit: e.MergeCommit, + Merged: e.Merged, Author: e.Author} + p.Owner, p.Repo = splitRepository(e.Repository) + default: + return p, fmt.Errorf("%s is no pull request event of the seat git", key) + } + if p.Owner == "" || p.Repo == "" { + return p, fmt.Errorf("names no repository /") + } + return p, nil +} + +// forgeAction is the forge's own word in a `forge.pull` event: what it did to the pull request. +func forgeAction(body json.RawMessage) string { + var e struct { + Action string `json:"action"` + } + _ = json.Unmarshal(body, &e) + return e.Action +} + +// gitStatusEvent reads the seat's `forge.status` into the gate's ForgeStatusEvent. +func gitStatusEvent(body json.RawMessage) (ForgeStatusEvent, error) { + var e gitseat.ForgeStatusEvent + if err := json.Unmarshal(body, &e); err != nil { + return ForgeStatusEvent{}, err + } + s := ForgeStatusEvent{SHA: e.Commit, Context: e.Context, State: e.State, Description: e.Description} + s.Owner, s.Repo = splitRepository(e.Repository) + if s.Owner == "" || s.Repo == "" || s.SHA == "" { + return s, fmt.Errorf("names no repository / or no commit") + } + return s, nil +} + +// forgePullOf is the seat's pull request in the gate's form: a merged one is closed and merged, as the forge said it before. +func forgePullOf(p gitseat.Pull) *ForgePull { + out := &ForgePull{Number: int(p.Number), Title: p.Title, State: p.State, Base: p.Base, Head: p.Branch, HeadSHA: p.Head, + MergeCommit: p.MergeCommit, MergedAt: timeText(p.MergedAt), HTMLURL: p.URL} + if p.State == gitseat.PullStateMerged { + out.State, out.Merged = gitseat.PullStateClosed, true + } + return out +} + +// forgeStatusOf is one status of the seat in the gate's form. +func forgeStatusOf(s gitseat.Status) ForgeStatus { + return ForgeStatus{Context: s.Context, State: s.State, Description: s.Description, TargetURL: s.Target, + UpdatedAt: timeText(s.At), Creator: s.By} +} + +// openPullOf is one of the seat's open pull requests in the gate's form. +func openPullOf(p gitseat.OpenPull) OpenPull { + out := OpenPull{Number: int(p.Pull.Number), Base: p.Pull.Base, HeadSHA: p.Pull.Head, HTMLURL: p.Pull.URL, + Required: p.Required, Statuses: []string{}} + out.Owner, out.Repo = splitRepository(p.Pull.Repository) + if out.Required == nil { + out.Required = []string{} + } + if len(p.Checks) > 0 { + out.Checks = map[string]HeadCheck{} + } + for context, s := range p.Checks { + out.Statuses = append(out.Statuses, context) + out.Checks[context] = HeadCheck{State: s.State, Description: s.Description, UpdatedAt: timeText(s.At)} + } + slices.Sort(out.Statuses) + return out +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go new file mode 100644 index 000000000..328b8de0c --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" +) + +// W11.18 (design 47 *The seat git*): every forge read of the gate is a verb of the seat git, its answer typed in the seat's +// contract, and a failed read is an error, never "none"; every forge event is heard as the seat's. +func TestTheGateReadsAndHearsTheForgeThroughTheSeatGit(t *testing.T) { + head := strings.Repeat("c", 40) + pull := gitseat.Pull{Repository: "novox/app", Number: 7, Base: "main", Branch: "feat/x", Head: head, State: "merged", + MergeCommit: strings.Repeat("d", 40), URL: "https://forge.invalid/novox/app/pulls/7"} + answers := map[string]any{ + "seat:git.open-pulls": gitseat.OpenPullsAnswer{Pulls: []gitseat.OpenPull{{Pull: pull, Required: []string{"mesh/check"}, + Checks: map[string]gitseat.Status{"mesh/check": {Context: "mesh/check", State: "pending"}}}}}, + "seat:git.contains": gitseat.ContainsAnswer{Contains: true}, + "seat:git.pull": gitseat.PullAnswer{Pull: &pull}, + "seat:git.statuses": gitseat.StatusesAnswer{Commit: head, Combined: "pending", + Statuses: []gitseat.Status{{Context: "mesh/check", State: "pending", By: "mesh"}}}, + mergeStateTool: map[string]any{"commit": head, "merge": map[string]any{"branch": "main", "mergeable": true}}, + } + for _, c := range []struct { + name string + failed string // the verb whose answer is an error + read func(seatForge) (any, error) + want any + asked []string + }{ + {name: "open pull requests", read: func(f seatForge) (any, error) { return f.OpenPulls() }, + want: []OpenPull{{Owner: "novox", Repo: "app", Number: 7, Base: "main", HeadSHA: head, HTMLURL: pull.URL, + Required: []string{"mesh/check"}, Statuses: []string{"mesh/check"}, + Checks: map[string]HeadCheck{"mesh/check": {State: "pending"}}}}, + asked: []string{"seat:git.open-pulls"}}, + {name: "open pull requests not read", failed: "seat:git.open-pulls", + read: func(f seatForge) (any, error) { return f.OpenPulls() }, asked: []string{"seat:git.open-pulls"}}, + {name: "contains", read: func(f seatForge) (any, error) { return f.Contains("novox", "app", "a", "b") }, want: true, + asked: []string{"seat:git.contains"}}, + {name: "contains not read", failed: "seat:git.contains", + read: func(f seatForge) (any, error) { return f.Contains("novox", "app", "a", "b") }, asked: []string{"seat:git.contains"}}, + {name: "a pull request by its number", read: func(f seatForge) (any, error) { + c, err := f.Statuses("novox", "app", 7, "", "") + if err != nil { + return nil, err + } + return []any{c.Commit, *c.Pull, c.Statuses, *c.Merge.Mergeable}, nil + }, want: []any{head, ForgePull{Number: 7, State: "closed", Merged: true, Base: "main", Head: "feat/x", HeadSHA: head, + MergeCommit: pull.MergeCommit, HTMLURL: pull.URL}, []ForgeStatus{{Context: "mesh/check", State: "pending", Creator: "mesh"}}, true}, + asked: []string{"seat:git.pull", "seat:git.statuses", mergeStateTool}}, + {name: "a pull request not read", failed: "seat:git.pull", + read: func(f seatForge) (any, error) { return f.Statuses("novox", "app", 7, "", "") }, + asked: []string{"seat:git.pull"}}, + {name: "statuses not read", failed: "seat:git.statuses", + read: func(f seatForge) (any, error) { return f.Statuses("novox", "app", 0, head, "main") }, + asked: []string{"seat:git.statuses"}}, + } { + t.Run(c.name, func(t *testing.T) { + var asked []string + f := seatForge{ask: func(key string, body any) (json.RawMessage, error) { + asked = append(asked, key) + if key == c.failed { + return json.Marshal(map[string]string{"error": "the forge answered 502"}) + } + a, ok := answers[key] + if !ok { + return nil, errors.New("no such verb: " + key) + } + return json.Marshal(a) + }} + got, err := c.read(f) + if !reflect.DeepEqual(asked, c.asked) { + t.Fatalf("asked %v, wanted %v", asked, c.asked) + } + if c.failed != "" { + if err == nil || !strings.Contains(err.Error(), "the forge answered 502") { + t.Fatalf("a failed read answered %v, %v; wanted its error", got, err) + } + return + } + if err != nil || !reflect.DeepEqual(got, c.want) { + t.Fatalf("read %#v, %v; wanted %#v", got, err, c.want) + } + }) + } + + t.Run("a merge heard", func(t *testing.T) { + body, _ := json.Marshal(gitseat.PullMergedEvent{Repository: "novox/app", Number: 7, Base: "main", Branch: "feat/x", + Head: head, MergeCommit: pull.MergeCommit, Author: "jo", Change: &gitseat.Change{Paths: []string{"a.go"}}}) + p, err := gitPullEvent(gitEvent(gitseat.EventPullMerged), body) + want := PullEvent{Owner: "novox", Repo: "app", Number: 7, Base: "main", Head: "feat/x", HeadSHA: head, + MergeCommit: pull.MergeCommit, Merged: true, Author: "jo", Paths: []string{"a.go"}} + if err != nil || !reflect.DeepEqual(p, want) { + t.Fatalf("heard %+v, %v; wanted %+v", p, err, want) + } + if _, err := gitPullEvent(gitEvent(gitseat.EventPullMerged), []byte(`{"number":7}`)); err == nil { + t.Fatal("a merge naming no repository was taken") + } + }) +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 470b68fde..64df2e448 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -313,9 +313,11 @@ type PullEvent struct { HeadChecks map[string]string `json:"head_checks,omitempty"` // BaseSHA is the base branch's head when the head was announced (novox/hq ADR 0323). BaseSHA string `json:"base_sha,omitempty"` - // Merged is the forge's own events' word that the pull request merged (gitea.forge.pull-request.*): its close is no + // Merged is the forge's own events' word that the pull request merged (the seat git's forge.pull): its close is no // close unmerged (issue 1107). Merged bool `json:"merged,omitempty"` + // Author is who opened it, as the forge's pull request record says (the seat git's events). + Author string `json:"author,omitempty"` } func (p PullEvent) repository() string { return strings.ToLower(p.Owner + "/" + p.Repo) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1107_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1107_test.go index f9dea49d5..0c99878bf 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1107_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1107_test.go @@ -1,6 +1,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "encoding/json" "testing" ) @@ -9,8 +11,8 @@ import ( // (gitea.forge.pull-request.closed), not only from the poll, which skips a repository whose update time did not move. // Its delivery leaves its group and stops; a merged one is ignored (the webhook says merged for it); a close heard // twice, from the event and the poll, changes nothing the second time. -func forgeClosed(owner, repo string, number int, merged bool) map[string]any { - return map[string]any{"owner": owner, "repo": repo, "number": number, "merged": merged} +func forgeClosed(owner, repo string, number int, merged bool) gitseat.ForgePullEvent { + return gitseat.ForgePullEvent{Action: "closed", Repository: owner + "/" + repo, Number: int64(number), Merged: merged} } func TestAForgeClosedEventTakesAGroupMembersDeliveryOutOfItsGroupAndStopsIt(t *testing.T) { @@ -18,7 +20,7 @@ func TestAForgeClosedEventTakesAGroupMembersDeliveryOutOfItsGroupAndStopsIt(t *t w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) id := IDOf(gCtl, gHc) - hear(t, w, "gitea.forge.pull-request.closed", forgeClosed("novox", "mesh-controller", 1, false)) + hear(t, w, gitEvent(gitseat.EventForgePull), forgeClosed("novox", "mesh-controller", 1, false)) d := w.delivery(id) if !d.ClosedUnmerged || !d.State.Final() || inGroup(w, "feat/g", id) { t.Fatalf("closed %v, state %s, in its group %v; wanted closed, stopped, out of its group", d.ClosedUnmerged, d.State, @@ -26,8 +28,8 @@ func TestAForgeClosedEventTakesAGroupMembersDeliveryOutOfItsGroupAndStopsIt(t *t } // Heard again, from the event and from the poll: nothing changes. before, _ := json.Marshal(d) - hear(t, w, "gitea.forge.pull-request.closed", forgeClosed("novox", "mesh-controller", 1, false)) - hear(t, w, "gitea.pull.closed", map[string]any{"owner": "novox", "repo": "mesh-controller", "number": 1}) + hear(t, w, gitEvent(gitseat.EventForgePull), forgeClosed("novox", "mesh-controller", 1, false)) + hear(t, w, gitEvent(gitseat.EventPullClosed), gitseat.PullClosedEvent{Repository: "novox/mesh-controller", Number: 1}) if after, _ := json.Marshal(w.delivery(id)); string(after) != string(before) { t.Fatalf("a second close changed it:\n%s\n%s", before, after) } @@ -38,7 +40,7 @@ func TestAForgeClosedEventOfAMergedPullRequestIsIgnored(t *testing.T) { w.h.PullUpdated(pr(gCtl, 1, gHc, "feat/g", "internal/a.go")) w.h.PullUpdated(pr(gCat, 2, gHk, "feat/g", "modules/app/module.json")) id := IDOf(gCtl, gHc) - hear(t, w, "gitea.forge.pull-request.closed", forgeClosed("novox", "mesh-controller", 1, true)) + hear(t, w, gitEvent(gitseat.EventForgePull), forgeClosed("novox", "mesh-controller", 1, true)) if d := w.delivery(id); d.ClosedUnmerged || !inGroup(w, "feat/g", id) || d.State.Final() { t.Fatalf("a merged pull request's close stopped it: closed %v, %s", d.ClosedUnmerged, d.State) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index e463c1359..068c09d19 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -17,6 +17,7 @@ import ( "sync" "time" + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -31,11 +32,9 @@ type listening struct { func (l *listening) set(s string) { l.mu.Lock(); l.now = s; l.mu.Unlock() } func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.now } -// The events this owner follows: the forge's pull requests. The forge's status events say that a check answered (events.go). -// The step events the delivery record is built from follow (consumer.go). -var followed = append([]string{"gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed", - "gitea.forge.status", "gitea.forge.pull-request.edited", "gitea.forge.pull-request.closed", - forgePattern, ConfigChangedEvent, writableEvent}, stepPatterns()...) +// The events this owner follows: the forge's pull requests, as the seat git says them (gitseat.go). The forge's status events +// say that a check answered (events.go). The step events the delivery record is built from follow (consumer.go). +var followed = append(append(append([]string{}, gitEvents...), forgePattern, ConfigChangedEvent), stepPatterns()...) // forgePattern is the forge bridge's messages on FORGE (novox/hq ADR 0437 §8): the gate follows a merge and the refusal of one. const forgePattern = "mesh-delivery-forge-bridge.forge.**" @@ -60,7 +59,7 @@ func main() { h := &Holder{ Settings: settings, Store: kvStore{}, - Forge: toolForge{ask: stdio.Ask}, + Forge: seatForge{ask: stdio.Ask}, Emit: func(event string, body any) error { return stdio.Emit(event, body) }, Logf: logf, Operator: kvOperator{}, @@ -213,45 +212,42 @@ func take(h *Holder, e stdio.Envelope) error { return nil } switch e.Key { - case "gitea.pull.updated", "gitea.pull.merged", "gitea.pull.closed": - var p PullEvent - if err := json.Unmarshal(e.Body, &p); err != nil { + case gitEvent(gitseat.EventPullUpdated), gitEvent(gitseat.EventPullMerged), gitEvent(gitseat.EventPullClosed): + p, err := gitPullEvent(e.Key, e.Body) + if err != nil { h.Refuse(fmt.Errorf("%s is not readable: %v", e.Key, err)) return nil } switch e.Key { - case "gitea.pull.updated": + case gitEvent(gitseat.EventPullUpdated): if h.PullUpdated(p) { h.wantForge() // a head not seen before: the forge shows whether anyone checks it } h.KeepPull(p) - case "gitea.pull.merged": + case gitEvent(gitseat.EventPullMerged): h.PullMerged(p) default: h.PullClosed(p) } - case "gitea.forge.pull-request.edited": - // The description edited after a head was announced (issue 1079): the lines in it are read again. - var p PullEvent - if err := json.Unmarshal(e.Body, &p); err != nil { + case gitEvent(gitseat.EventForgePull): + action := forgeAction(e.Body) + if action != "edited" && action != "closed" { + return nil // a head and a merge are the seat's pull.updated and pull.merged + } + p, err := gitPullEvent(e.Key, e.Body) + if err != nil { h.Refuse(fmt.Errorf("%s is not readable: %v", e.Key, err)) return nil } - h.PullEdited(p) - case "gitea.forge.pull-request.closed": - // The forge's own close event (issue 1107): the poll's gitea.pull.closed skips a repository whose update time did not - // move, and a close does not move it. A merged pull request is the webhook's merged event, never this one. - var p PullEvent - if err := json.Unmarshal(e.Body, &p); err != nil { - h.Refuse(fmt.Errorf("%s is not readable: %v", e.Key, err)) - return nil - } - if !p.Merged { + if action == "edited" { + h.PullEdited(p) // an edit after a head was announced takes over a stopped record (issue 1114) + } else if !p.Merged { + // The forge's own close (issue 1107): a merged pull request is the merged event, never this one. h.PullClosed(p) } - case "gitea.forge.status": - var s ForgeStatusEvent - if err := json.Unmarshal(e.Body, &s); err != nil { + case gitEvent(gitseat.EventForgeStatus): + s, err := gitStatusEvent(e.Body) + if err != nil { h.Refuse(fmt.Errorf("a commit status is not readable: %v", err)) return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 69acc6236..3c35d1126 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -111,9 +111,11 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if !reflect.DeepEqual(m.Reads, []string{tellerModule + ".answers", checkerVerdicts, viewModule + ".reach"}) { t.Fatalf("reads %v", m.Reads) } - // It calls no controller verb and reads the forge with three tools; the forge's writes are the bridge's (ADR 0437 §8). It - // uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks (ADR 0437 §6). - want := []string{"gitea.gitea_commit_statuses", "gitea.gitea_contains", "gitea.gitea_open_pulls"} + // It calls no controller verb and reads the forge through the seat git's verbs (design 47 *The seat git*), and the forge's + // merge state through the one statuses tool the seat does not carry yet (seatForge.mergeState); the forge's writes are the + // bridge's (ADR 0437 §8). It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks + // (ADR 0437 §6). + want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", mergeStateTool} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index 35e2649a2..2cbf862af 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -600,7 +600,7 @@ var trunkHead = regexp.MustCompile(`^[0-9a-f]{40}$`) // seedTrunks gives source.main the trunk head of a repository it names none of (issue 1422): one never merged through the // gate, whose open pull requests the checker could otherwise never judge. The head is the base_sha of the latest announcement of -// any open pull request into main (gitea.pull.updated; ADR 0323; lastAnnounced), the trunk's freshest head the gate heard. It is +// any open pull request into main (the seat git's pull.updated; ADR 0323; lastAnnounced), the trunk's freshest head the gate heard. It is // added in one compare-and-set write that never replaces a head source.main holds, so a merge always wins; the first merge // through the gate moves it as any other. source.main is per repository and names main's head only (ADR 0484 §11, MergeBranch): // a pull request into another base seeds nothing. A repository is weighed once per start after it was found or seeded. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 5ad6a565f..3b5b444ff 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -50,7 +50,7 @@ type Delivery struct { // Check is its own check's verdict. Check *Verdict `json:"check,omitempty"` - // Required are the statuses its base requires, as the forge last said them (gitea_open_pulls): mesh/check makes it + // Required are the statuses its base requires, as the forge last said them (the seat git's open-pulls): mesh/check makes it // the checker's (issue 1302). Empty until the forge said them. Required []string `json:"required,omitempty"` // CheckState is its head's mesh/check as the forge last said it (pending, success, failure, error): a pending one diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go index de48c147a..d5dfaa0c0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go @@ -3,6 +3,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "context" "encoding/json" "strings" @@ -38,7 +40,7 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) { rt.mu.Unlock() announced := pr("novox/app", 8, head, "fix/y", "src/b.go") announced.Body = "Reviewed-by: lead@shanks at " + head - b.publish(t, "mesh.mod.gitea.event.pull.updated", announced) + b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(announced)) // The checker's verdict at the head, then the forge's word that its bridge set mesh/check. hash, err := mdc.HeadsHash([]mdc.Head{{Repository: "novox/app", Number: 8, Head: head}}) @@ -55,8 +57,8 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) { rt.openPulls[0].Statuses = []string{ctxCheck} rt.openPulls[0].Checks = map[string]HeadCheck{ctxCheck: {State: "success", Description: line}} rt.mu.Unlock() - set := b.publish(t, "mesh.mod.gitea.event.forge.status", map[string]any{"owner": "novox", "repo": "app", "sha": head, - "context": ctxCheck, "state": "success", "description": line}) + set := b.publish(t, "mesh.seat.git.event.forge.status", gitseat.ForgeStatusEvent{Repository: "novox/app", Commit: head, + Context: ctxCheck, State: "success", Description: line}) var key string rt.until("the ask to be written on mesh/check alone", 10*time.Second, func() bool { key = onlyAsk(b); return key != "" }) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go index 76b40c283..5d57538e4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go @@ -7,12 +7,13 @@ import ( "sort" "strings" + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" stdio "git.novox.be/novox/mesh-sdk/go" ) // What mesh-delivery asks of others, and where it keeps what it owns. **It never sends to a machine, writes // a declaration, a grant or a bus object, and never registers a build** (novox/hq ADR 0239 decision 6): it -// asks the controller, through the controller's seat, and the forge's holder, through its tools. +// asks the controller, through the controller's seat, and the forge, through the seat `git`. // Forge is what the forge's holder does for a delivery. type Forge interface { @@ -26,7 +27,7 @@ type Forge interface { OpenPulls() ([]OpenPull, error) } -// ForgeChecks is what the forge says of a commit's checks (the gitea module's gitea_commit_statuses). +// ForgeChecks is what the forge says of a commit's checks (the seat git's pull and statuses, and the merge state). type ForgeChecks struct { Commit string `json:"commit"` Pull *ForgePull `json:"pull,omitempty"` @@ -48,7 +49,7 @@ type ForgePull struct { // MergedAt is when it merged, as the forge says it (RFC 3339); empty when not merged or not said. MergedAt string `json:"merged_at,omitempty"` HTMLURL string `json:"html_url,omitempty"` - // Mergeable is the forge's own word on whether it merges into its base now (gitea's mergeable): false while it is in + // Mergeable is the forge's own word on whether it merges into its base now (its mergeable): false while it is in // conflict with its base, or while the forge works that out after the base moved; nil when the forge did not say. Mergeable *bool `json:"mergeable,omitempty"` } @@ -137,80 +138,130 @@ func lastLine(s string) string { return strings.TrimSpace(lines[len(lines)-1]) } -// toolForge asks the forge's holder through its tools. -type toolForge struct{ ask asking } +// seatForge reads the forge through the seat `git` (design 47 *The seat git*): every read a verb of the seat, its answer +// typed in the seat's contract, a failed read an error and never "none". The forge module is never named. +type seatForge struct{ ask asking } -func (f toolForge) tool(name string, args map[string]any) error { - raw, err := f.ask("gitea."+name, args) +// gitAsk calls a verb of the seat `git` and reads its typed answer; an answer carrying `error` is a failed read. +func gitAsk[A any](f seatForge, verb string, in any, errOf func(*A) string) (*A, error) { + raw, err := f.ask("seat:"+gitseat.Seat+"."+verb, in) if err != nil { - return err + return nil, err } - if _, output, ok := answerOf(raw); !ok { - return fmt.Errorf("the forge refused %s: %s", name, lastLine(output)) + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("the seat git refused %s: %s", verb, lastLine(output)) } - return nil + var a A + if err := json.Unmarshal(answer, &a); err != nil { + return nil, fmt.Errorf("the seat git's %s is not readable: %s", verb, clip(string(answer), 200)) + } + if why := errOf(&a); why != "" { + return nil, fmt.Errorf("the seat git could not read %s: %s", verb, why) + } + return &a, nil } -func (f toolForge) Statuses(owner, repo string, number int, commit, base string) (*ForgeChecks, error) { - args := map[string]any{"owner": owner, "repo": repo} +// Statuses reads a pull request by its number through `pull` (its head when no commit is named) and the commit's statuses +// through `statuses`. What the forge would merge now (the base's protection, its conflict word, whether the merging +// account writes) is read by mergeState. +func (f seatForge) Statuses(owner, repo string, number int, commit, base string) (*ForgeChecks, error) { + repository := owner + "/" + repo + c := &ForgeChecks{Commit: commit} + if number > 0 { + a, err := gitAsk(f, gitseat.VerbPull, gitseat.PullInput{Repository: repository, Number: int64(number)}, + func(a *gitseat.PullAnswer) string { return a.Error }) + if err != nil { + return nil, err + } + if a.Pull == nil { + return nil, fmt.Errorf("the seat git's pull named no pull request for %s#%d", repository, number) + } + c.Pull = forgePullOf(*a.Pull) + if c.Commit == "" { + c.Commit = a.Pull.Head + } + if base == "" { + base = a.Pull.Base + } + } + if c.Commit == "" { + return nil, fmt.Errorf("the seat git named no head of %s#%d", repository, number) + } + s, err := gitAsk(f, gitseat.VerbStatuses, gitseat.StatusesInput{Repository: repository, Commit: c.Commit}, + func(a *gitseat.StatusesAnswer) string { return a.Error }) + if err != nil { + return nil, err + } + c.Combined = s.Combined + for _, st := range s.Statuses { + c.Statuses = append(c.Statuses, forgeStatusOf(st)) + } + merge, mergeable, err := f.mergeState(owner, repo, number, c.Commit, base) + if err != nil { + return nil, err + } + c.Merge = merge + if c.Pull != nil { + c.Pull.Mergeable = mergeable + } + return c, nil +} + +// mergeState is what the forge would merge now: the base's protection and its required statuses, the forge's own word on +// a conflict, and whether the merging account writes the repository. The seat `git` names none of them yet (dev-G-record's +// finding on W11.18): until it does, they are read through the forge holder's statuses tool, the one read left there. +func (f seatForge) mergeState(owner, repo string, number int, commit, base string) (ForgeMerge, *bool, error) { + args := map[string]any{"owner": owner, "repo": repo, "sha": commit} if number > 0 { args["number"] = number } - if commit != "" { - args["sha"] = commit - } if base != "" { args["base"] = base } - raw, err := f.ask("gitea.gitea_commit_statuses", args) + raw, err := f.ask(mergeStateTool, args) if err != nil { - return nil, err + return ForgeMerge{}, nil, err } answer, output, ok := answerOf(raw) if !ok { - return nil, fmt.Errorf("the forge refused gitea_commit_statuses: %s", lastLine(output)) + return ForgeMerge{}, nil, fmt.Errorf("the forge refused its merge state: %s", lastLine(output)) } var c ForgeChecks if err := json.Unmarshal(answer, &c); err != nil || c.Commit == "" { - return nil, fmt.Errorf("the forge's gitea_commit_statuses is not readable: %s", clip(string(answer), 200)) + return ForgeMerge{}, nil, fmt.Errorf("the forge's merge state is not readable: %s", clip(string(answer), 200)) } - return &c, nil + var mergeable *bool + if c.Pull != nil { + mergeable = c.Pull.Mergeable + } + return c.Merge, mergeable, nil } -func (f toolForge) OpenPulls() ([]OpenPull, error) { - raw, err := f.ask("gitea.gitea_open_pulls", map[string]any{}) +// mergeStateTool is the one forge read the seat `git` does not carry yet (see mergeState). +const mergeStateTool = "gitea.gitea_commit_statuses" + +// OpenPulls reads every open pull request through `open-pulls`. +func (f seatForge) OpenPulls() ([]OpenPull, error) { + a, err := gitAsk(f, gitseat.VerbOpenPulls, gitseat.OpenPullsInput{}, func(a *gitseat.OpenPullsAnswer) string { return a.Error }) if err != nil { return nil, err } - answer, output, ok := answerOf(raw) - if !ok { - return nil, fmt.Errorf("the forge refused gitea_open_pulls: %s", lastLine(output)) + out := make([]OpenPull, 0, len(a.Pulls)) + for _, p := range a.Pulls { + out = append(out, openPullOf(p)) } - var open struct { - Pulls []OpenPull `json:"pulls"` - } - if err := json.Unmarshal(answer, &open); err != nil || open.Pulls == nil { - return nil, fmt.Errorf("the forge's gitea_open_pulls is not readable: %s", clip(string(answer), 200)) - } - return open.Pulls, nil + return out, nil } -func (f toolForge) Contains(owner, repo, commit, in string) (bool, error) { - raw, err := f.ask("gitea.gitea_contains", map[string]any{"owner": owner, "repo": repo, "commit": commit, "in": in}) +// Contains reads through `contains` whether in contains commit. +func (f seatForge) Contains(owner, repo, commit, in string) (bool, error) { + a, err := gitAsk(f, gitseat.VerbContains, gitseat.ContainsInput{Repository: owner + "/" + repo, Commit: commit, In: in}, + func(a *gitseat.ContainsAnswer) string { return a.Error }) if err != nil { return false, err } - answer, output, ok := answerOf(raw) - if !ok { - return false, fmt.Errorf("the forge refused gitea_contains: %s", lastLine(output)) - } - var a struct { - Contains *bool `json:"contains"` - } - if err := json.Unmarshal(answer, &a); err != nil || a.Contains == nil { - return false, fmt.Errorf("the forge's gitea_contains is not readable: %s", clip(string(answer), 200)) - } - return *a.Contains, nil + return a.Contains, nil } // carriedKey is a key an earlier owner's carry-over wrote into the bucket `deliveries`: no delivery, never read as one. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/pullkey.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/pullkey.go index 4b2b77017..e208c0fac 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/pullkey.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/pullkey.go @@ -14,7 +14,7 @@ import ( // It writes a new head when the forge says one (`pull.updated`, which carries the head and the trunk commit it was // announced against), and a new trunk commit only when the check is asked again against one — the merge tool's ask is // that same announcement. A merge of a group's member, and a trunk move that leaves an inactive pull request unasked, -// are announced as nothing (the gitea module announces `pull.updated` only for a head and for the merge tool's ask), so +// are announced as nothing (the forge announces `pull.updated` only for a head and for the merge tool's ask), so // they write nothing. A verdict for that head and trunk commit writes the verdict. The build seat reads the key. // PullKey is what the bus keeps of one pull request. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go index 368b3271f..1990de8cf 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go @@ -61,7 +61,7 @@ const UncheckedAfter = 10 * time.Minute // requiring any is one the mesh is meant to check; only those a base requires are waited on. var mergeChecks = []string{ctxCheck, "mesh/merge-gate", "mesh/repo-check"} -// OpenPull is an open pull request as the forge says it (the gitea module's gitea_open_pulls): its head, its base, +// OpenPull is an open pull request as the forge says it (the seat git's open-pulls): its head, its base, // the contexts the base's protection requires, the contexts its head has a status of, in any state, and each of // those statuses with its state. type OpenPull struct { @@ -183,7 +183,7 @@ func (h *Holder) SeeOpenPulls() error { return nil } -// openPullsMost is the most open pull requests the forge's gitea_open_pulls answers: it reads 20 pages of 50 and says +// openPullsMost is the most open pull requests the seat git's open-pulls answers: it reads 20 pages of 50 and says // nothing of the rest (mesh-catalog modules/gitea client.ts searchOpenPulls). A list that long may be cut, so what is // missing from it is not taken as closed. const openPullsMost = 20 * 50 diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked_events_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked_events_test.go index b2999e7c5..4ed5fb1db 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked_events_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked_events_test.go @@ -1,6 +1,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "testing" "time" ) @@ -9,18 +11,18 @@ import ( // the forge's open pull requests are read when this owner starts and when an event says something they show may have // changed — a merge check's status, a new head, a ready mark — and once more at the bound a head's check is said. -func statusEvent(sha, context, state string) ForgeStatusEvent { - return ForgeStatusEvent{Owner: "novox", Repo: "hq", SHA: sha, Context: context, State: state} +func statusEvent(sha, context, state string) gitseat.ForgeStatusEvent { + return gitseat.ForgeStatusEvent{Repository: "novox/hq", Commit: sha, Context: context, State: state} } func TestAMergeCheckStatusReadsTheOpenPullRequestsAndAnotherContextDoesNot(t *testing.T) { w := newWorld(t) before := w.forge.openCalls - hear(t, w, "gitea.forge.status", statusEvent("cccc", "mesh/delivery", "success")) + hear(t, w, gitEvent(gitseat.EventForgeStatus), statusEvent("cccc", "mesh/delivery", "success")) if w.forge.openCalls != before { t.Fatalf("a status of a context that is no merge check read the forge %d times", w.forge.openCalls-before) } - hear(t, w, "gitea.forge.status", statusEvent("cccc", "mesh/merge-gate", "pending")) + hear(t, w, gitEvent(gitseat.EventForgeStatus), statusEvent("cccc", "mesh/merge-gate", "pending")) if w.forge.openCalls != before+1 { t.Fatalf("a merge check's status read the forge %d times, wanted once", w.forge.openCalls-before) } @@ -31,7 +33,7 @@ func TestAHeadWhoseCheckNeverAnsweredIsSaidFromTheEventsAndOneReadAtTheBound(t * w.forge.open = []OpenPull{{Owner: "novox", Repo: "hq", Number: 1, Base: "main", HeadSHA: "cccc", Required: []string{"mesh/merge-gate"}, Statuses: []string{"mesh/merge-gate"}, Checks: map[string]HeadCheck{"mesh/merge-gate": {State: "pending", UpdatedAt: "2026-10-06T12:00:00Z"}}}} - hear(t, w, "gitea.forge.status", statusEvent("cccc", "mesh/merge-gate", "pending")) + hear(t, w, gitEvent(gitseat.EventForgeStatus), statusEvent("cccc", "mesh/merge-gate", "pending")) if len(w.h.Stalled()) != 0 { t.Fatal("a head just seen waiting is said already") } @@ -55,11 +57,11 @@ func TestAHeadWhoseCheckNeverAnsweredIsSaidFromTheEventsAndOneReadAtTheBound(t * func TestANewlyAnnouncedHeadReadsTheForgeAndAKnownOneDoesNot(t *testing.T) { w := newWorld(t) before := w.forge.openCalls - hear(t, w, "gitea.pull.updated", pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + hear(t, w, gitEvent(gitseat.EventPullUpdated), seatUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))) if w.forge.openCalls != before+2 { // the unchecked heads, and the head's standing verdict before its check is asked (issue 1115) t.Fatalf("a new head read the forge %d times, wanted twice", w.forge.openCalls-before) } - hear(t, w, "gitea.pull.updated", pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + hear(t, w, gitEvent(gitseat.EventPullUpdated), seatUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go"))) if w.forge.openCalls != before+3 { // the look of a delivery awaiting its verdict reads its standing one t.Fatalf("the same head announced again read the forge again (%d reads)", w.forge.openCalls-before) } @@ -88,7 +90,7 @@ func TestAClosedOrMergedPullRequestIsNoLongerSaidUnchecked(t *testing.T) { t.Fatal("not said") } closed := pr("novox/hq", 1, "cccc", "feat/x") - hear(t, w, "gitea.pull.closed", closed) + hear(t, w, gitEvent(gitseat.EventPullClosed), seatClosed(closed)) if lines := w.h.Stalled(); len(lines) != 0 { t.Fatalf("a closed pull request is still said: %+v", lines) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes.go index cb142cbe4..4421db95f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes.go @@ -5,24 +5,26 @@ import ( "fmt" "strings" "time" + + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" ) // **The operator is asked only for a merge the forge account can carry out** (novox/hq issue 1945). car-hunter #2, a // repository of a user, was approved, and its first act after the Approve, the commit status, was refused 403: the mesh's // merging account (the gitea module's tools, mesh-tools) only read that repository. Before the gate asks the operator's -// Approve it reads, through the forge holder's gitea_commit_statuses, whether that account writes each repository the +// Approve it reads, with the forge's merge state (seatForge.mergeState), whether that account writes each repository the // ask covers (merge.writes), as the ask is about to be written. Where it does not, nothing is asked: the delivery waits, // saying "the forge account cannot write : gitea_grant_write", and the gate raises it as a condition at once. // -// A repository not written is kept so, and the ask is not composed again, until the gitea module's event -// gitea.repo.writable (its grant, read back from the forge) or, as a safety net, writeReadLasts after the read. A forge +// A repository not written is kept so, and the ask is not composed again, until the seat git's event +// repo.writable (its grant, read back from the forge) or, as a safety net, writeReadLasts after the read. A forge // that does not say merge.writes (an older gitea module) is taken as writing: the merge says its own refusal, as before. // writeReadLasts is how long a read of whether the forge account writes a repository holds. const writeReadLasts = 10 * time.Minute -// writableEvent is the gitea module's event of an account given write on a repository. -const writableEvent = "gitea.repo.writable" +// writableEvent is the seat git's event of an account given write on a repository. +var writableEvent = gitEvent(gitseat.EventRepoWritable) // writeRead is what the forge last said of the merging account on one repository it cannot write. type writeRead struct { @@ -88,17 +90,14 @@ func (h *Holder) cannotWrite(reads []memberRead) string { return "" } -// WriteGranted takes the gitea module's gitea.repo.writable: the repository's read is dropped, so the next look reads it +// WriteGranted takes the seat git's repo.writable: the repository's read is dropped, so the next look reads it // again, and an approved merge of it that the forge refused is asked again, its cause gone. func (h *Holder) WriteGranted(body json.RawMessage) error { - var e struct { - Owner string `json:"owner"` - Repo string `json:"repo"` - } - if err := json.Unmarshal(body, &e); err != nil || e.Owner == "" || e.Repo == "" { + var e gitseat.RepoWritableEvent + if err := json.Unmarshal(body, &e); err != nil || !strings.Contains(strings.Trim(e.Repository, "/"), "/") { return fmt.Errorf("%s names no repository: %s", writableEvent, string(body)) } - repo := strings.ToLower(e.Owner + "/" + e.Repo) + repo := strings.ToLower(e.Repository) h.mu.Lock() defer h.mu.Unlock() delete(h.writeReads, repo) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes_test.go index 286242307..e198541c3 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes_test.go @@ -1,6 +1,8 @@ package main import ( + gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "strings" "testing" ) @@ -33,7 +35,7 @@ func TestAnUnwritableRepositoryIsNotAskedAndSaysSoUntilTheWriteIsGranted(t *test } delete(w.forge.unwritable, "novox/app") - hear(t, w, writableEvent, map[string]string{"owner": "novox", "repo": "app", "account": "mesh-tools"}) + hear(t, w, writableEvent, gitseat.RepoWritableEvent{Repository: "novox/app", Account: "mesh-tools"}) w.h.React() if w.op.count() != 1 { t.Fatalf("after the write was granted the operator was asked %d times, wanted once", w.op.count()) diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index 11b37039e..e352605ad 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -7,6 +7,7 @@ require ( git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/seats/git v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index d7aa47e5f..a619a7d8e 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -20,15 +20,14 @@ } ], "consumes": [ - "gitea.pull.updated", - "gitea.pull.merged", - "gitea.pull.closed", - "gitea.forge.status", - "gitea.forge.pull-request.edited", - "gitea.forge.pull-request.closed", + "git.pull.updated", + "git.pull.merged", + "git.pull.closed", + "git.forge.pull", + "git.forge.status", + "git.repo.writable", "mesh-delivery-forge-bridge.forge.**", "mesh-config.config-changed", - "gitea.repo.writable", "mesh-delivery-indexer.take-in.taken" ], "emits": [ @@ -254,9 +253,11 @@ "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg": "0.1" }, "invokes": [ - "gitea.gitea_commit_statuses", - "gitea.gitea_contains", - "gitea.gitea_open_pulls" + "seat:git.pull", + "seat:git.statuses", + "seat:git.open-pulls", + "seat:git.contains", + "gitea.gitea_commit_statuses" ], "state": [ "deliveries", -- 2.54.0 From 8a6827dfe24da10788620ac68d5533ae747dfce2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:33 +0100 Subject: [PATCH 043/243] The delivery record says pinned, freed, heads-replaced and record-ended; it ends once every present node runs its entries (milestone 47/G, W11.9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node away is named with since when and holds nothing; a node away that wakes adds its outcomes and is no longer shown away. freed is said before a record past its retention is removed, so no revision stays pinned by nobody. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 52b629c6.) Builds: design 47 G §147, §149, §153 (GATE3) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/consumer.go | 18 +- .../cmd/mesh-delivery-gate/holder.go | 15 + .../cmd/mesh-delivery-gate/main.go | 3 +- .../cmd/mesh-delivery-gate/record.go | 40 ++- .../cmd/mesh-delivery-gate/record_test.go | 179 ++++++++++- .../cmd/mesh-delivery-gate/recordevents.go | 293 ++++++++++++++++++ 6 files changed, 532 insertions(+), 16 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go index 6c70dbc61..33f55c13d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go @@ -188,14 +188,18 @@ func ownerRefusal(f stepFamily, e stdio.Envelope) string { return "" } -// apply writes an outcome into every open record that holds it: by its merge commits, or by its entry. +// apply writes an outcome into every record that holds it, by its merge commits or by its entry: an open record, or one +// that ended without the node the outcome comes from (a node away that wakes adds its outcomes, and is no longer shown +// away). An open record whose present nodes then all run their entries ends. func (c *StepConsumer) apply(o StepOutcome, event string) error { - records, err := c.Records.Current() + records, err := c.Records.Kept() if err != nil { return err } for _, r := range records { - if r.Ended != nil || !holds(r, o) { + _, fromAway := r.Away[o.Node] + fromAway = fromAway && o.Node != "" + if (r.Ended != nil && !fromAway) || !holds(r, o) { continue } ro := RecordOutcome{Module: o.Module, Fingerprint: o.Fingerprint, Node: o.Node, Step: o.Step, Outcome: o.Outcome, @@ -203,6 +207,14 @@ func (c *StepConsumer) apply(o StepOutcome, event string) error { if _, err := c.Records.Outcome(r, ro); err != nil { return err } + if fromAway { + if err := c.Records.NodeBack(r, o.Node); err != nil { + return err + } + } + if _, err := c.Records.EndIfRun(r); err != nil { + return err + } } return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 64df2e448..b22b758d5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -42,6 +42,8 @@ type Holder struct { Records *Records // Consumer applies the step events to the records (consumer.go); nil applies none. Consumer *StepConsumer + // SaySeat says an event of the seat mesh-delivery (recordevents.go): heads-replaced here; nil says none. + SaySeat SeatSay // Pulls keeps the pull requests' keys on the bus (pullkey.go); nil keeps none. Pulls PullKeys // Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none. @@ -416,6 +418,7 @@ func (h *Holder) supersedeOlder(p PullEvent) (replaced string) { if old.Repository == p.repository() && old.Number == p.Number && !sameCommit(old.Commit, p.HeadSHA) && !old.State.Final() && old.MergedAs == "" { old.NewerHead = p.HeadSHA + h.sayHeadsReplaced(askGroup(old), h.candidateOf(old), old, p.HeadSHA, false) h.settleAndKeep(old) if old.Group != "" { h.leaveGroup(old) @@ -426,6 +429,17 @@ func (h *Holder) supersedeOlder(p PullEvent) (replaced string) { return replaced } +// candidateOf are the members whose heads make the candidate a delivery is judged in: its group's, or itself alone. +// Under the lock. +func (h *Holder) candidateOf(d *Delivery) []*Delivery { + if g := h.groups[d.Group]; d.Group != "" && g != nil { + if members := h.membersOf(g); len(members) > 0 { + return members + } + } + return []*Delivery{d} +} + // fillFromAnnouncement fills a delivery made from its check's verdict before its pull request's head was heard (issue // 1726: no base, branch, title or URLs) from an announcement or an edit of that pull request, and then // does what a first announcement does: the older heads of the pull request are superseded, and the delivery takes @@ -880,6 +894,7 @@ func (h *Holder) PullClosed(p PullEvent) { } if !d.State.Final() && d.MergedAs == "" { d.ClosedUnmerged = true + h.sayHeadsReplaced(askGroup(d), h.candidateOf(d), d, "", true) h.settleAndKeep(d) if d.Group != "" { h.leaveGroup(d) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 068c09d19..74f682a7e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -73,7 +73,8 @@ func main() { Conditions: kvConditions{}, } h.init() - h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention} + h.SaySeat = stdio.SeatPublish + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish} h.Merges = recordedMerges{MergeStore: kvMerges{}, records: h.Records, members: h.groupMembers} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf} l := &listening{now: "not yet: starting"} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index aae76a812..0c0934532 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -321,8 +321,8 @@ type Records struct { Now func() time.Time // Retention is how long a record is kept; nil keeps DefaultRecordRetention. Retention func() time.Duration - // Freed is told each revision a removed record pinned, to say as `freed` (design 47); nil says none. - Freed func(r *Record, p PinnedRevision) + // Say says the seat mesh-delivery's events of the records: pinned, freed, record-ended (recordevents.go); nil says none. + Say SeatSay // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -349,6 +349,21 @@ func (rs *Records) Current() ([]*Record, error) { return out, nil } +// Kept is every record kept, ended or not, oldest first. +func (rs *Records) Kept() ([]*Record, error) { + if err := rs.load(); err != nil { + return nil, err + } + rs.mu.Lock() + defer rs.mu.Unlock() + out := make([]*Record, 0, len(rs.held)) + for _, r := range rs.held { + out = append(out, r) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].Opened.Before(out[j].Opened) }) + return out, nil +} + func (rs *Records) load() error { rs.mu.Lock() loaded := rs.loaded @@ -522,15 +537,13 @@ func (rs *Records) Sweep(now time.Time) error { return err } if r != nil { + if err := rs.sayFreedAll(r, "the record is past its retention"); err != nil { + return err + } if _, err := rs.Ledger.Apply(e.Instance, OpRemove, map[string]any{}); err != nil { return err } rs.forget(e.Instance) - if rs.Freed != nil { - for _, p := range sortedPins(r) { - rs.Freed(r, p) - } - } } if _, err := rs.Ledger.Apply(m, OpDrop, map[string]any{"instance": e.Instance}); err != nil { return err @@ -562,6 +575,19 @@ func (rs *Records) SetGroup(r *Record, g RecordGroup) error { return nil } +// SetTrace writes the record's trace, unless it holds that trace already. +func (rs *Records) SetTrace(r *Record, t RecordTrace) error { + if r.Trace != nil && reflect.DeepEqual(*r.Trace, t) { + return nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"trace": t}) + if err != nil { + return err + } + r.Trace, r.Sequence = &t, seq + return nil +} + // Outcome writes one entry's outcome on one node, unless the record holds it already: an event applied twice changes // nothing (design 47). Applied says it was written now. func (rs *Records) Outcome(r *Record, o RecordOutcome) (applied bool, err error) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index c1b11d87a..9e141b073 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -5,12 +5,14 @@ import ( "fmt" "os" "path/filepath" + "reflect" "slices" "strings" "testing" "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -230,9 +232,15 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByMilestoneAndHeads(t *testing.T now := at var freed []string rs := &Records{Ledger: l, Now: func() time.Time { return now }, Retention: func() time.Duration { return 90 * 24 * time.Hour }, - Freed: func(r *Record, p PinnedRevision) { freed = append(freed, r.Milestone+" "+p.Revision) }} + Say: func(subject string, body any, id string) error { + if subject == meshdelivery.SubjectFreed { + p := body.(meshdelivery.Pins) + freed = append(freed, fmt.Sprint(p.Milestone, " ", p.Catalog, " ", p.Why)) + } + return nil + }} old, _, _ := rs.Open("46/F", heads, false) - if _, err := l.Apply(old.Instance, OpPin, PinnedRevision{Revision: "cat-1", Half: "catalog"}); err != nil { + if err := rs.Pin(old, HalfCatalog, 1); err != nil { t.Fatal(err) } now = at.Add(89 * 24 * time.Hour) @@ -247,7 +255,7 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByMilestoneAndHeads(t *testing.T if r, _ := rs.Read(old.Instance); r != nil { t.Fatal("a record past its retention is kept") } - if !slices.Equal(freed, []string{"46/F cat-1"}) { + if !slices.Equal(freed, []string{"46/F [1] the record is past its retention"}) { t.Fatalf("freed %v", freed) } list, _ := rs.List() @@ -256,6 +264,60 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByMilestoneAndHeads(t *testing.T } }) + // W11.9: pinned and freed, said under an id of what they say; only a catalog or an assignment revision is pinned. + t.Run("a pin is written once and said as pinned; a free says freed with why", func(t *testing.T) { + l := newLedgerFake(t) + type said struct { + subject, id string + pins meshdelivery.Pins + } + var got []said + rs := &Records{Ledger: l, Now: func() time.Time { return at }, Say: func(subject string, body any, id string) error { + got = append(got, said{subject, id, body.(meshdelivery.Pins)}) + return nil + }} + r, _, _ := rs.Open("47/G", heads, false) + for range 2 { + if err := rs.Pin(r, HalfAssignment, 9); err != nil { + t.Fatal(err) + } + } + if err := rs.Pin(r, HalfCatalog, 9); err != nil { + t.Fatal(err) + } + pins := 0 + for _, op := range l.history { + if strings.HasSuffix(op, " "+OpPin) { + pins++ + } + } + back, _ := rs.Read(r.Instance) + if pins != 2 || len(back.Pinned) != 2 { + t.Fatalf("%d pins written, the record holds %+v", pins, back.Pinned) + } + if len(got) != 3 || !reflect.DeepEqual(got[0], got[1]) || got[0].subject != meshdelivery.SubjectPinned || + !slices.Equal(got[0].pins.Assignment, []int64{9}) || got[2].pins.Catalog[0] != 9 || got[0].pins.Record != r.Instance { + t.Fatalf("said %+v", got) + } + got = nil + if err := rs.Free(back, HalfAssignment, 9, "removed by hand"); err != nil { + t.Fatal(err) + } + if err := rs.Free(back, HalfAssignment, 9, "again"); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].subject != meshdelivery.SubjectFreed || got[0].pins.Why != "removed by hand" || + !slices.Equal(got[0].pins.Assignment, []int64{9}) { + t.Fatalf("said %+v", got) + } + if again, _ := rs.Read(r.Instance); len(again.Pinned) != 1 { + t.Fatalf("pinned after the free %+v", again.Pinned) + } + if err := rs.Pin(r, "trunk", 1); err == nil { + t.Fatal("a pin of no half is taken") + } + }) + t.Run("the setting record-retention, in days", func(t *testing.T) { dir := t.TempDir() for _, c := range []struct { @@ -332,7 +394,7 @@ func TestTheRecordsOutcomesArriveThroughTheGatesConsumer(t *testing.T) { if err := json.Unmarshal(body, &b); err != nil { return nil, err } - return []StepOutcome{{Module: b.Module, Fingerprint: b.Fingerprint, Step: "run", Outcome: OutcomeRunning, At: at}}, nil + return []StepOutcome{{Module: b.Module, Fingerprint: b.Fingerprint, Step: StepRunning, Outcome: OutcomeRunning, At: at}}, nil }} c := &StepConsumer{Records: rs, Families: append(slices.Clone(stepFamilies), running), Now: func() time.Time { return at }} return l, rs, r, c @@ -376,10 +438,67 @@ func TestTheRecordsOutcomesArriveThroughTheGatesConsumer(t *testing.T) { t.Fatal(err) } back, _ := rs.Read(r.Instance) - if got := back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "ace", "run")]; got.Outcome != OutcomeRunning { + if got := back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "ace", StepRunning)]; got.Outcome != OutcomeRunning { t.Fatalf("outcomes %+v", back.Outcomes) } }) + // W11.9: a record ends when every present node runs its entries, naming the nodes away, and says record-ended; a node + // away that wakes adds its outcomes and is no longer shown away. + t.Run("it ends once every present node runs its entries, and a node away that wakes adds its outcomes", func(t *testing.T) { + _, rs, r, c := setup(t) + var ended []recordEnded + rs.Say = func(subject string, body any, id string) error { + if subject == meshdelivery.SubjectRecordEnded && id == "record-ended."+r.Instance { + ended = append(ended, body.(recordEnded)) + } + return nil + } + if err := rs.AddEntry(r, RecordEntry{Module: "mesh-x", Fingerprint: "src2:f1", Node: "g14"}); err != nil { + t.Fatal(err) + } + if err := rs.AddEntry(r, RecordEntry{Module: "mesh-x", Fingerprint: "src2:f1"}); err != nil { + t.Fatal(err) // the catalog half's entry, which no node runs + } + running := func(node string) stdio.Envelope { + return stdio.Envelope{Key: "node-engine.running." + node + ".mesh-x", Node: node, + Body: json.RawMessage(`{"module":"mesh-x","fingerprint":"src2:f1"}`), Headers: map[string]string{"x-event-id": "r-" + node}} + } + if err := c.Hear(running("ace")); err != nil { + t.Fatal(err) + } + if back, _ := rs.Read(r.Instance); back.Ended != nil || len(ended) != 0 { + t.Fatalf("a record whose difference is not known ended: %+v", back.Ended) + } + kept, _ := rs.Kept() // the records as the consumer holds them + r = kept[0] + if err := rs.SetTrace(r, RecordTrace{From: "assignment-8", To: "assignment-9"}); err != nil { + t.Fatal(err) + } + if done, err := rs.EndIfRun(r); err != nil || done || allRun(r) != "waiting for mesh-x on g14" { + t.Fatalf("ended %v (%v) while g14 runs nothing: %q", done, err, allRun(r)) + } + if err := rs.NodeAway(r, "g14", at.Add(-time.Hour)); err != nil { + t.Fatal(err) + } + back, _ := rs.Read(r.Instance) + if back.Ended == nil || back.Ended.Why != WhyAllRun || len(back.Ended.Away) != 1 || back.Ended.Away[0].Node != "g14" || + !back.Ended.Away[0].Since.Equal(at.Add(-time.Hour)) { + t.Fatalf("ended %+v", back.Ended) + } + if len(ended) != 1 || ended[0].Milestone != "47/G" || len(ended[0].Away) != 1 || ended[0].Away[0].Node != "g14" { + t.Fatalf("record-ended said %+v", ended) + } + if err := c.Hear(running("g14")); err != nil { + t.Fatal(err) + } + back, _ = rs.Read(r.Instance) + if got := back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "g14", StepRunning)]; got.Outcome != OutcomeRunning || len(back.Away) != 0 { + t.Fatalf("the node that woke: outcomes %+v, away %+v", back.Outcomes, back.Away) + } + if back.Ended == nil || len(back.Ended.Away) != 1 { + t.Fatalf("how it ended is not kept: %+v", back.Ended) + } + }) t.Run("an unreadable event is taken and said; one the ledger cannot take is offered again", func(t *testing.T) { l, _, _, c := setup(t) bad := taken("e2") @@ -449,3 +568,53 @@ func TestTheSwitchCarriesTheGroupsInFlightIntoRecordsOnce(t *testing.T) { t.Fatalf("the take-in of the group's merge is not its record's outcome: %+v", back.Outcomes) } } + +// W11.9 (design 47 *The delivery record*): the gate says heads-replaced when a candidate's heads were replaced or its pull +// request closed, keyed as the candidate's revisions are (the heads' hash), so the indexer and the assigner drop its graphs. +func TestACandidatesHeadsReplacedOrClosedAreSaid(t *testing.T) { + a, b, c := strings.Repeat("a", 40), strings.Repeat("b", 40), strings.Repeat("c", 40) + setup := func(t *testing.T) (*world, *[]headsReplaced) { + w := newWorld(t) + var said []headsReplaced + w.h.SaySeat = func(subject string, body any, id string) error { + if subject != meshdelivery.SubjectHeadsReplaced { + t.Errorf("said on %s", subject) + } + e := body.(headsReplaced) + if id != "heads-replaced."+e.Group+"."+e.Heads { + t.Errorf("id %s for %+v", id, e) + } + said = append(said, e) + return nil + } + w.h.PullUpdated(pr("novox/app", 7, a, "feat/x", "src/a.go")) + w.h.PullUpdated(pr("novox/lib", 3, b, "feat/x", "src/b.go")) + w.settleAll() + return w, &said + } + hashOf := func(heads ...mdc.Head) string { + h, err := mdc.HeadsHash(heads) + if err != nil { + t.Fatal(err) + } + return h + } + group := hashOf(mdc.Head{Repository: "novox/app", Number: 7, Head: a}, mdc.Head{Repository: "novox/lib", Number: 3, Head: b}) + t.Run("a member's new head replaces the group's candidate", func(t *testing.T) { + w, said := setup(t) + w.h.PullUpdated(pr("novox/app", 7, c, "feat/x", "src/a.go")) + w.settleAll() + if len(*said) != 1 || (*said)[0].Group != "feat/x" || (*said)[0].Heads != group || (*said)[0].Head != c || + (*said)[0].Closed || (*said)[0].Repository != "novox/app" || (*said)[0].Number != 7 { + t.Fatalf("said %+v (the candidate %s)", *said, group) + } + }) + t.Run("a member's pull request closed replaces the group's candidate", func(t *testing.T) { + w, said := setup(t) + w.h.PullClosed(pr("novox/lib", 3, b, "feat/x")) + w.settleAll() + if len(*said) != 1 || (*said)[0].Heads != group || !(*said)[0].Closed || (*said)[0].Head != "" || (*said)[0].Number != 3 { + t.Fatalf("said %+v (the candidate %s)", *said, group) + } + }) +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go new file mode 100644 index 000000000..5e2d7978a --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordevents.go @@ -0,0 +1,293 @@ +package main + +// What a delivery record says on the seat `mesh-delivery` (design 47 *The delivery record*): `pinned` and `freed` for the +// graph revisions it starts and stops pointing at, `heads-replaced` for a candidate whose heads were replaced or whose pull +// request closed, and `record-ended` once every present node runs its entries. Only the gate's account publishes them +// (ADR 0363 §2); each is said under an id of what it says, so said again it is kept once, and a say that failed is said +// again at the next write of the record. + +import ( + "fmt" + "sort" + "strconv" + "strings" + "time" + + mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" +) + +// The halves of the revision view a record pins (design 47 *Two graphs, one source of truth*). +const ( + HalfCatalog = "catalog" + HalfAssignment = "assignment" +) + +// StepRunning is the step whose outcome OutcomeRunning says an entry runs healthy on its node: from the node-engine seat's +// `running` event, never from a have read across the seam (ADR 0596). +const StepRunning = "running" + +// WhyAllRun is why a record ends by itself. +const WhyAllRun = "every present node runs its entries" + +// SeatSay says one event of the seat mesh-delivery on its subject, under an id the bus keeps once (stdio.SeatPublish). +type SeatSay func(subject string, body any, id string) error + +func (rs *Records) say(subject string, body any, id string) error { + if rs.Say == nil { + return nil + } + return rs.Say(subject, body, id) +} + +// pinKey is a pinned revision's key in the record: its half and its number, so the two halves' numbers never meet. +func pinKey(half string, revision int64) string { return half + "-" + strconv.FormatInt(revision, 10) } + +// number is the revision's number on its half's seat. +func (p PinnedRevision) number() (int64, bool) { + n, err := strconv.ParseInt(strings.TrimPrefix(p.Revision, p.Half+"-"), 10, 64) + return n, err == nil && strings.HasPrefix(p.Revision, p.Half+"-") +} + +// pinsOf is the seat's Pins of a record for these revisions. +func pinsOf(r *Record, pins []PinnedRevision, why string) meshdelivery.Pins { + out := meshdelivery.Pins{Record: r.Instance, Milestone: r.Milestone, Why: why} + for _, p := range pins { + n, ok := p.number() + if !ok { + continue + } + switch p.Half { + case HalfCatalog: + out.Catalog = append(out.Catalog, n) + case HalfAssignment: + out.Assignment = append(out.Assignment, n) + } + } + return out +} + +func checkHalf(half string) error { + if half != HalfCatalog && half != HalfAssignment { + return fmt.Errorf("a record pins a catalog or an assignment graph revision, not a %q one", half) + } + return nil +} + +// Pin writes that the record points at a graph revision, unless it does already, and says `pinned` (again, when written +// before: the id keeps it once). +func (rs *Records) Pin(r *Record, half string, revision int64) error { + if err := checkHalf(half); err != nil { + return err + } + p := PinnedRevision{Revision: pinKey(half, revision), Half: half} + if _, ok := r.Pinned[p.Revision]; !ok { + seq, err := rs.Ledger.Apply(r.Instance, OpPin, p) + if err != nil { + return err + } + if r.Pinned == nil { + r.Pinned = map[string]PinnedRevision{} + } + r.Pinned[p.Revision], r.Sequence = p, seq + } + return rs.say(meshdelivery.SubjectPinned, pinsOf(r, []PinnedRevision{p}, ""), "pinned."+r.Instance+"."+p.Revision) +} + +// Free writes that the record no longer points at a graph revision and says `freed` with why; a revision it does not +// point at frees nothing and says nothing. +func (rs *Records) Free(r *Record, half string, revision int64, why string) error { + if err := checkHalf(half); err != nil { + return err + } + p, ok := r.Pinned[pinKey(half, revision)] + if !ok { + return nil + } + if err := rs.say(meshdelivery.SubjectFreed, pinsOf(r, []PinnedRevision{p}, why), "freed."+r.Instance+"."+p.Revision); err != nil { + return err + } + seq, err := rs.Ledger.Apply(r.Instance, OpFree, map[string]any{"revision": p.Revision}) + if err != nil { + return err + } + delete(r.Pinned, p.Revision) + r.Sequence = seq + return nil +} + +// sayFreedAll says `freed` for every revision a record pins, before the record is removed: said first, so a removal never +// leaves a revision pinned by nobody that the indexer or the assigner still keeps. +func (rs *Records) sayFreedAll(r *Record, why string) error { + pins := sortedPins(r) + if len(pins) == 0 { + return nil + } + return rs.say(meshdelivery.SubjectFreed, pinsOf(r, pins, why), "freed."+r.Instance) +} + +// NodeAway writes a node away since when, unless the record holds it already, and ends the record when every other node +// runs its entries: a node away holds nothing (ADR 0603 §7). +func (rs *Records) NodeAway(r *Record, node string, since time.Time) error { + if node == "" { + return fmt.Errorf("a node away is named") + } + if _, ok := r.Away[node]; !ok { + a := NodeAway{Node: node, Since: since.UTC()} + seq, err := rs.Ledger.Apply(r.Instance, OpAway, a) + if err != nil { + return err + } + if r.Away == nil { + r.Away = map[string]NodeAway{} + } + r.Away[node], r.Sequence = a, seq + } + _, err := rs.EndIfRun(r) + return err +} + +// NodeBack writes that a node away is heard again: the record no longer shows it away, also once it ended. +func (rs *Records) NodeBack(r *Record, node string) error { + if _, ok := r.Away[node]; !ok { + return nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpBack, map[string]any{"node": node}) + if err != nil { + return err + } + delete(r.Away, node) + r.Sequence = seq + return nil +} + +// allRun is why a record does not end yet, or "" when every present node runs its entries: the difference it delivers is +// known (the trace names the assignment graph revision it moved to), and every entry on a node not away has its running +// healthy outcome. An entry of the catalog half names no node and is run by none. +func allRun(r *Record) string { + if r.Trace == nil || r.Trace.To == "" { + return "its difference is not known yet" + } + var waiting []string + for _, e := range r.Entries { + if e.Node == "" { + continue + } + if _, away := r.Away[e.Node]; away { + continue + } + if o, ok := r.Outcomes[OutcomeKey(e.Module, e.Fingerprint, e.Node, StepRunning)]; !ok || o.Outcome != OutcomeRunning { + waiting = append(waiting, e.Module+" on "+e.Node) + } + } + if len(waiting) > 0 { + sort.Strings(waiting) + return "waiting for " + strings.Join(waiting, ", ") + } + return "" +} + +// awayList is the record's nodes away, by node. +func awayList(r *Record) []NodeAway { + out := make([]NodeAway, 0, len(r.Away)) + for _, a := range r.Away { + out = append(out, a) + } + sort.Slice(out, func(i, j int) bool { return out[i].Node < out[j].Node }) + return out +} + +// EndIfRun ends the record when every present node runs its entries, naming the nodes away with since when, and says +// `record-ended`. Ended says it ended now; a record ended before is said again (the id keeps it once). +func (rs *Records) EndIfRun(r *Record) (ended bool, err error) { + if r.Ended == nil { + if allRun(r) != "" { + return false, nil + } + if err := rs.end(r, WhyAllRun, ""); err != nil { + return false, err + } + ended = true + } + return ended, rs.sayEnded(r) +} + +// End ends a record by hand, with the reason and who (gate_record_end). +func (rs *Records) End(r *Record, why, by string) error { + if r.Ended != nil { + return rs.sayEnded(r) + } + if strings.TrimSpace(why) == "" { + return fmt.Errorf("a record ended by hand says why") + } + if err := rs.end(r, why, by); err != nil { + return err + } + return rs.sayEnded(r) +} + +func (rs *Records) end(r *Record, why, by string) error { + e := RecordEnded{At: rs.now(), Why: why, By: by, Away: awayList(r)} + seq, err := rs.Ledger.Apply(r.Instance, OpEnd, map[string]any{"ended": e}) + if err != nil { + return err + } + r.Ended, r.Sequence = &e, seq + rs.keep(r) + return nil +} + +// recordEnded is the seat's `record-ended` body. +type recordEnded struct { + Record string `json:"record"` + Milestone string `json:"milestone"` + ForgeMade bool `json:"forge-made,omitempty"` + Away []NodeAway `json:"away,omitempty"` + At time.Time `json:"at"` +} + +func (rs *Records) sayEnded(r *Record) error { + if r.Ended == nil { + return nil + } + return rs.say(meshdelivery.SubjectRecordEnded, recordEnded{Record: r.Instance, Milestone: r.Milestone, + ForgeMade: r.ForgeMade, Away: r.Ended.Away, At: r.Ended.At}, "record-ended."+r.Instance) +} + +// headsReplaced is the seat's `heads-replaced` body: a candidate's heads replaced or its pull request closed. +type headsReplaced struct { + Group string `json:"group"` + Heads string `json:"heads"` + Repository string `json:"repository"` + Number int `json:"number"` + Head string `json:"head,omitempty"` + Closed bool `json:"closed,omitempty"` + At time.Time `json:"at"` +} + +// sayHeadsReplaced says that the candidate of these members' heads means nothing any more, since one member's head moved +// to head or (closed) its pull request closed; the heads are keyed as the candidate's revisions are (HeadsHash). Under the +// lock; the say itself is the work's, outside it. +func (h *Holder) sayHeadsReplaced(group string, members []*Delivery, moved *Delivery, head string, closed bool) { + if h.SaySeat == nil || len(members) == 0 { + return + } + hash, err := mdc.HeadsHash(headsOf(members)) + if err != nil { + h.refuse(fmt.Errorf("the candidate of %s cannot be said replaced: %v", group, err)) + return + } + e := headsReplaced{Group: group, Heads: hash, Repository: strings.ToLower(moved.Repository), Number: moved.Number, + Closed: closed, At: h.Now().UTC()} + if !closed { + e.Head = strings.ToLower(head) + } + say := h.SaySeat + id := "heads-replaced." + group + "." + hash + h.cancels = append(h.cancels, func() { + if err := say(meshdelivery.SubjectHeadsReplaced, e, id); err != nil { + h.Logf("[mesh-delivery] the candidate %s of %s is not said replaced: %v", hash, group, err) + } + }) + h.Poke() +} -- 2.54.0 From eb7a25622df51d55032966529e49510086b4d0a2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:33 +0100 Subject: [PATCH 044/243] The delivery record's tools read a record whole, list them with their fill and end one by hand; a removal that kept its operations is said (milestone 47/G, W11.13) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit gate_record, gate_records, gate_consumer and gate_record_end (recorded in the hand-act log of the seat mesh-hand-acts before anything is ended). gate_record_repair waits for the owners' read verbs. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 12032a28.) Builds: design 47 G §159 (GATE13) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/main.go | 2 + .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/record.go | 35 +++- .../cmd/mesh-delivery-gate/record_test.go | 103 ++++++++++ .../cmd/mesh-delivery-gate/recordtools.go | 193 ++++++++++++++++++ modules/mesh-delivery-gate/go.mod | 2 + modules/mesh-delivery-gate/module.json | 9 +- 7 files changed, 336 insertions(+), 10 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 74f682a7e..1fd9bcaa4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -17,6 +17,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -466,6 +467,7 @@ func tools(h *Holder, l *listening) []stdio.Tool { return out, nil }}, } + list = append(list, recordTools(h, meshhandactsclient.Recorder{Ask: stdio.Ask})...) // A verb that changes a delivery asks for a look at what it makes possible (react.go). for i := range list { run := list[i].Run diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 3c35d1126..9d0edb064 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -115,7 +115,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { // merge state through the one statuses tool the seat does not carry yet (seatForge.mergeState); the forge's writes are the // bridge's (ADR 0437 §8). It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks // (ADR 0437 §6). - want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", mergeStateTool} + want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", mergeStateTool, "seat:mesh-hand-acts.hand-act"} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 0c0934532..4bf027ce0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -271,6 +271,8 @@ type RecordLedger interface { Apply(instance, operation string, body any) (uint64, error) // Get is the instance's state and the sequence of its last operation; a nil state when it has none. Get(instance string) (json.RawMessage, uint64, error) + // Remove removes a record with its operations and its snapshot (OpRemove); an error when any of them stays. + Remove(instance string) error } // sdkLedger is the gate's ledger through the runtime: the tool runner's ledger calls, on the gate's own ledger only @@ -281,23 +283,42 @@ type sdkLedger struct{} var ledgerCall = stdio.BusState func (sdkLedger) Apply(instance, operation string, body any) (uint64, error) { + got, err := applyCall(instance, operation, body) + return got.Sequence, err +} + +// applyCall is one operation applied through the runtime, and the ledger's answer. +func applyCall(instance, operation string, body any) (buscalls.Applied, error) { + var got buscalls.Applied raw, err := json.Marshal(body) if err != nil { - return 0, err + return got, err } out, err := ledgerCall(buscalls.LedgerApply, buscalls.ApplyRequest{Ledger: RecordsLedger, Instance: instance, Operation: operation, Body: raw}) if err != nil { - return 0, err + return got, err } - var got buscalls.Applied if err := json.Unmarshal(out, &got); err != nil { - return 0, fmt.Errorf("the ledger's answer to %s on %s is not readable: %w", operation, instance, err) + return got, fmt.Errorf("the ledger's answer to %s on %s is not readable: %w", operation, instance, err) } if !got.Taken { - return 0, fmt.Errorf("the ledger did not take %s on %s", operation, instance) + return got, fmt.Errorf("the ledger did not take %s on %s", operation, instance) } - return got.Sequence, nil + return got, nil +} + +// Remove applies OpRemove and reads the ledger's answer: a removal that left the record's operations, its snapshot or its +// state is an error, so the retention says it rather than leaving a record's history to grow unseen. +func (sdkLedger) Remove(instance string) error { + got, err := applyCall(instance, OpRemove, map[string]any{}) + if err != nil { + return err + } + if !got.Removed { + return fmt.Errorf("the ledger removed %s but kept its operations or its snapshot (removed false at sequence %d)", instance, got.Sequence) + } + return nil } func (sdkLedger) Get(instance string) (json.RawMessage, uint64, error) { @@ -540,7 +561,7 @@ func (rs *Records) Sweep(now time.Time) error { if err := rs.sayFreedAll(r, "the record is past its retention"); err != nil { return err } - if _, err := rs.Ledger.Apply(e.Instance, OpRemove, map[string]any{}); err != nil { + if err := rs.Ledger.Remove(e.Instance); err != nil { return err } rs.forget(e.Instance) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 9e141b073..e9fb985e6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -11,9 +12,11 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" ) @@ -133,6 +136,11 @@ func (f *ledgerFake) Apply(instance, operation string, body any) (uint64, error) return f.seq[instance], nil } +func (f *ledgerFake) Remove(instance string) error { + _, err := f.Apply(instance, OpRemove, map[string]any{}) + return err +} + func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) { s := f.states[instance] if s == nil || string(s) == "null" { @@ -618,3 +626,98 @@ func TestACandidatesHeadsReplacedOrClosedAreSaid(t *testing.T) { } }) } + +// handActsFake is the hand-act log: what it took, or a refusal. +type handActsFake struct { + took []meshhandactsclient.Request + refuse error +} + +func (f *handActsFake) Record(q meshhandactsclient.Request) (meshhandactsclient.Taken, error) { + if f.refuse != nil { + return meshhandactsclient.Taken{}, f.refuse + } + f.took = append(f.took, q) + return meshhandactsclient.Taken{Log: "hand-acts"}, nil +} + +// W11.13 (design 47 *The delivery record*, its tools): gate_record reads a record whole, gate_records lists them with +// their fill, gate_record_end ends one by hand after the hand-act log took it; and a removal that keeps the record's +// operations is said, never taken quietly (buscalls.Applied.Removed). +func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { + at := time.Date(2026, 11, 9, 22, 0, 0, 0, time.UTC) + heads := map[string]string{"novox/app@111111111111": strings.Repeat("1", 40)} + setup := func(t *testing.T) (*Records, *Record, *[]string) { + var said []string + rs := &Records{Ledger: newLedgerFake(t), Now: func() time.Time { return at }, + Say: func(subject string, body any, id string) error { said = append(said, subject+" "+id); return nil }} + r, _, err := rs.Open("47/G", heads, false) + if err != nil { + t.Fatal(err) + } + if err := rs.AddEntry(r, RecordEntry{Module: "mesh-x", Fingerprint: "src2:f1", Node: "ace"}); err != nil { + t.Fatal(err) + } + if err := rs.SetTrace(r, RecordTrace{To: "assignment-9"}); err != nil { + t.Fatal(err) + } + return rs, r, &said + } + t.Run("gate_record and gate_records read a record whole and its fill", func(t *testing.T) { + rs, r, _ := setup(t) + got, err := rs.RecordOf(r.Instance, "") + if err != nil || len(got) != 1 || got[0].Record.Milestone != "47/G" || got[0].Waits != "waiting for mesh-x on ace" || + got[0].Bytes == 0 || got[0].Of != 256<<10 { + t.Fatalf("%+v %v", got, err) + } + if byMilestone, err := rs.RecordOf("", "47/G"); err != nil || len(byMilestone) != 1 { + t.Fatalf("by milestone %+v %v", byMilestone, err) + } + if _, err := rs.RecordOf("", "46/F"); err == nil { + t.Fatal("a milestone with no record answers nothing") + } + lines, err := rs.Lines() + if err != nil || lines["open"] != 1 || len(lines["records"].([]recordLine)) != 1 || lines["records"].([]recordLine)[0].Entries != 1 { + t.Fatalf("%+v %v", lines, err) + } + }) + t.Run("gate_record_end ends a record only once the hand-act log took it", func(t *testing.T) { + rs, r, said := setup(t) + log := &handActsFake{refuse: errors.New("no responders")} + if _, err := rs.EndByHand(r.Instance, "", "jochen@shanks", log); err == nil { + t.Fatal("ended with no reason") + } + if _, err := rs.EndByHand(r.Instance, "g14 is gone for good", "jochen@shanks", log); err == nil { + t.Fatal("ended though the hand-act log refused it") + } + if back, _ := rs.Read(r.Instance); back.Ended != nil { + t.Fatal("a record is ended while no hand act was recorded") + } + log.refuse = nil + v, err := rs.EndByHand(r.Instance, "g14 is gone for good", "jochen@shanks", log) + if err != nil || v.Record.Ended == nil || v.Record.Ended.By != "jochen@shanks" || len(log.took) != 1 { + t.Fatalf("%+v %v (log %+v)", v, err, log.took) + } + if !slices.Contains(*said, meshdelivery.SubjectRecordEnded+" record-ended."+r.Instance) { + t.Fatalf("said %v", *said) + } + if _, err := rs.EndByHand(r.Instance, "again", "jochen@shanks", log); err == nil || len(log.took) != 1 { + t.Fatal("a record ended is ended again") + } + }) + t.Run("a removal that keeps the record's operations is an error", func(t *testing.T) { + was := ledgerCall + t.Cleanup(func() { ledgerCall = was }) + removed := false + ledgerCall = func(method string, body any) (json.RawMessage, error) { + return json.Marshal(buscalls.Applied{Taken: true, Sequence: 4, Removed: removed}) + } + if err := (sdkLedger{}).Remove("record/47_G/0011223344556677"); err == nil || !strings.Contains(err.Error(), "kept its operations") { + t.Fatalf("a removal that kept its operations: %v", err) + } + removed = true + if err := (sdkLedger{}).Remove("record/47_G/0011223344556677"); err != nil { + t.Fatal(err) + } + }) +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go new file mode 100644 index 000000000..72faf22d1 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -0,0 +1,193 @@ +package main + +// The delivery record's tools (design 47 *The delivery record*, its tools against the four lines a module's tools answer): +// gate_record reads one record whole, gate_records lists them with their fill, gate_consumer reads the record's consumer, +// and gate_record_end ends a record left open, recorded first in the hand-act log of the seat mesh-hand-acts. + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// valueLimit is the bus's limit of one value, which a record's state is held under (design 47: a record of 30 modules on +// four nodes is about 30 KiB of it). +const valueLimit = 256 << 10 + +// recordView is one record as gate_record answers it: its state, the sequence of its last operation, its size against +// the value limit, and while open why it has not ended. +type recordView struct { + Instance string `json:"instance"` + Sequence uint64 `json:"sequence"` + Bytes int `json:"bytes"` + Of int `json:"of-bytes"` + Waits string `json:"waits,omitempty"` + Record *Record `json:"record"` +} + +func viewOf(r *Record) recordView { + raw, _ := json.Marshal(r) + v := recordView{Instance: r.Instance, Sequence: r.Sequence, Bytes: len(raw), Of: valueLimit, Record: r} + if r.Ended == nil { + v.Waits = allRun(r) + } + return v +} + +// RecordOf is one record whole, by its instance, or every record of a milestone. +func (rs *Records) RecordOf(instance, milestone string) ([]recordView, error) { + if instance != "" { + r, err := rs.Read(instance) + if err != nil { + return nil, err + } + if r == nil { + return nil, fmt.Errorf("no delivery record %s is kept", instance) + } + return []recordView{viewOf(r)}, nil + } + if milestone == "" { + return nil, errors.New("name a record's instance or a milestone") + } + list, err := rs.List() + if err != nil { + return nil, err + } + var out []recordView + for _, r := range list { + if r.Milestone == milestone { + out = append(out, viewOf(r)) + } + } + if len(out) == 0 { + return nil, fmt.Errorf("no delivery record of the milestone %s is kept", milestone) + } + return out, nil +} + +// recordLine is one record as gate_records lists it. +type recordLine struct { + Instance string `json:"instance"` + Milestone string `json:"milestone"` + Opened string `json:"opened"` + Ended string `json:"ended,omitempty"` + ForgeMade bool `json:"forge-made,omitempty"` + Entries int `json:"entries"` + Outcomes int `json:"outcomes"` + Away int `json:"away,omitempty"` + Pinned int `json:"pinned,omitempty"` + Bytes int `json:"bytes"` + Of int `json:"of-bytes"` + Waits string `json:"waits,omitempty"` +} + +// Lines are every record kept, oldest first, each with its fill against the value limit. +func (rs *Records) Lines() (map[string]any, error) { + list, err := rs.List() + if err != nil { + return nil, err + } + lines := make([]recordLine, 0, len(list)) + open := 0 + for _, r := range list { + v := viewOf(r) + l := recordLine{Instance: r.Instance, Milestone: r.Milestone, Opened: timeText(&r.Opened), ForgeMade: r.ForgeMade, + Entries: len(r.Entries), Outcomes: len(r.Outcomes), Away: len(r.Away), Pinned: len(r.Pinned), Bytes: v.Bytes, + Of: v.Of, Waits: v.Waits} + if r.Ended != nil { + l.Ended = timeText(&r.Ended.At) + ": " + r.Ended.Why + } else { + open++ + } + lines = append(lines, l) + } + return map[string]any{"records": lines, "open": open, "kept": len(lines), "retention-days": int(rs.retention().Hours() / 24)}, nil +} + +// EndByHand ends one open record by hand, with the reason, recorded in the hand-act log first: nothing is ended when the +// log did not take it. +func (rs *Records) EndByHand(instance, why, by string, acts handActs) (recordView, error) { + if strings.TrimSpace(why) == "" { + return recordView{}, errors.New("say why the record is ended by hand, such as a node gone for good") + } + r, err := rs.Read(instance) + if err != nil { + return recordView{}, err + } + if r == nil { + return recordView{}, fmt.Errorf("no delivery record %s is kept", instance) + } + if r.Ended != nil { + return viewOf(r), fmt.Errorf("the record %s ended before, at %s: %s", instance, timeText(&r.Ended.At), r.Ended.Why) + } + if acts == nil { + return recordView{}, errors.New("the hand-act log cannot be reached here, so nothing was ended") + } + if _, err := acts.Record(meshhandactsclient.Request{What: "ended the delivery record " + instance + " by hand", + Why: why, Cause: "gate_record_end", Verb: "record-end"}); err != nil { + return recordView{}, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was ended: %w", err) + } + if err := rs.End(r, why, by); err != nil { + return recordView{}, err + } + rs.keep(r) + return viewOf(r), nil +} + +// handActs records an act in the hand-act log (meshhandactsclient.Recorder). +type handActs interface { + Record(q meshhandactsclient.Request) (meshhandactsclient.Taken, error) +} + +// recordTools are the record's tools; acts is the hand-act log. +func recordTools(h *Holder, acts handActs) []stdio.Tool { + none := func() (any, error) { return nil, errors.New("this gate keeps no delivery records") } + return []stdio.Tool{ + {Name: "gate_record", + Description: "One delivery record whole (design 47, the delivery record): its group's state, its trace, each " + + "entry's outcome per node, the nodes away with since when, the revisions it pins, how it ended, its size " + + "against the bus's value limit, and while open what it waits for. By its instance, or every record of a milestone.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "instance": str("a record's instance, record//"), "milestone": str("a milestone")}}, + Run: func(a map[string]any) (any, error) { + if h.Records == nil { + return none() + } + return h.Records.RecordOf(strArg(a, "instance"), strArg(a, "milestone")) + }}, + {Name: "gate_records", + Description: "Every delivery record kept, oldest first: its milestone, when it opened and ended, its entries, " + + "outcomes, nodes away and pins, its size against the bus's value limit, and while open what it waits for.", + Run: func(map[string]any) (any, error) { + if h.Records == nil { + return none() + } + return h.Records.Lines() + }}, + {Name: "gate_consumer", + Description: "The delivery record's consumer of the step events: per family the events applied and refused " + + "since the gate started, the last applied, and why the last was refused.", + Run: func(map[string]any) (any, error) { + if h.Consumer == nil { + return nil, errors.New("this gate applies no step events") + } + return map[string]any{"families": h.Consumer.Heard()}, nil + }}, + {Name: "gate_record_end", + Description: "End a delivery record left open by hand (a node gone for good, an entry that never runs), with the " + + "reason; recorded in the hand-act log of the seat mesh-hand-acts first, and said as record-ended.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "instance": str("the record's instance"), "why": str("why it is ended by hand, in plain words")}, + "required": []string{"instance", "why"}}, + Run: func(a map[string]any) (any, error) { + if h.Records == nil { + return none() + } + return h.Records.EndByHand(strArg(a, "instance"), strArg(a, "why"), stdio.CallerOf(a), acts) + }}, + } +} diff --git a/modules/mesh-delivery-gate/go.mod b/modules/mesh-delivery-gate/go.mod index e352605ad..2b9665c4d 100644 --- a/modules/mesh-delivery-gate/go.mod +++ b/modules/mesh-delivery-gate/go.mod @@ -3,6 +3,7 @@ module mesh-delivery-gate go 1.26.0 require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient v0.0.0-00010101000000-000000000000 @@ -32,6 +33,7 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index a619a7d8e..b10a9275a 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -257,7 +257,8 @@ "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", - "gitea.gitea_commit_statuses" + "gitea.gitea_commit_statuses", + "seat:mesh-hand-acts.hand-act" ], "state": [ "deliveries", @@ -294,7 +295,11 @@ "delivery_hand_now", "delivery_merge_again", "delivery_status", - "delivery_turns" + "delivery_turns", + "gate_consumer", + "gate_record", + "gate_record_end", + "gate_records" ], "config": { "refusal-stall-floor-minutes": { -- 2.54.0 From a9fa00475866fe8de4e2999dc1e2be874158195f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:33 +0100 Subject: [PATCH 045/243] The gate reads what the forge would merge now from the seat git's pull, so it names no forge tool (milestone 47/G, W11.18 follow-up) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The base's protection and required contexts, the forge's conflict word and the merging account's write access come from the pull answer's merging (issues 1275, 1945; ADR 0441 §1); a required context not a success at the commit blocks. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 13e12d2c.) Builds: design 47 G §252, §310, §312 (GATE14) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bus_test.go | 13 +-- .../cmd/mesh-delivery-gate/gitseat_test.go | 23 ++++-- .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/ports.go | 82 +++++++++++-------- modules/mesh-delivery-gate/module.json | 1 - 5 files changed, 67 insertions(+), 54 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 428e57a4a..2bfb10adf 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -363,7 +363,8 @@ func (r *runtime) answer(m rpcMessage) { for _, p := range pulls { if p.Owner+"/"+p.Repo == repository && p.Number == int(number) { return gitseat.PullAnswer{Pull: &gitseat.Pull{Repository: repository, Number: int64(p.Number), Base: p.Base, - Head: p.HeadSHA, State: gitseat.PullStateOpen}}, nil + Head: p.HeadSHA, State: gitseat.PullStateOpen}, Merging: &gitseat.Merging{Mergeable: true, Protected: true, + Required: []string{"mesh/check"}, Writes: true}}, nil } } return gitseat.PullAnswer{Error: "no such pull request"}, nil @@ -371,16 +372,6 @@ func (r *runtime) answer(m rpcMessage) { // Green: mesh/check a success at the head asked, as the seat answers. commit, _ := a.Body["commit"].(string) return gitseat.StatusesAnswer{Commit: commit, Statuses: []gitseat.Status{{Context: "mesh/check", State: "success"}}}, nil - case mergeStateTool: - // Green: mesh/check a success at the head asked, as the gitea module answers. - sha, _ := a.Body["sha"].(string) - if sha == "" { - sha = "0000000" - } - return toolAnswer(map[string]any{"commit": sha, "statuses": []any{ - map[string]any{"context": "mesh/check", "state": "success"}}, - "merge": map[string]any{"branch": "main", "protected": true, "required": []string{"mesh/check"}, - "blocking": []any{}, "mergeable": true, "says": "main requires mesh/check"}}), nil case "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", "gitea.gitea_merge_pull_request", "gitea.gitea_add_comment": return toolAnswer(map[string]any{}), nil diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go index 328b8de0c..12b94b45b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go @@ -20,10 +20,10 @@ func TestTheGateReadsAndHearsTheForgeThroughTheSeatGit(t *testing.T) { "seat:git.open-pulls": gitseat.OpenPullsAnswer{Pulls: []gitseat.OpenPull{{Pull: pull, Required: []string{"mesh/check"}, Checks: map[string]gitseat.Status{"mesh/check": {Context: "mesh/check", State: "pending"}}}}}, "seat:git.contains": gitseat.ContainsAnswer{Contains: true}, - "seat:git.pull": gitseat.PullAnswer{Pull: &pull}, + "seat:git.pull": gitseat.PullAnswer{Pull: &pull, Merging: &gitseat.Merging{Mergeable: true, Protected: true, + Required: []string{"mesh/check"}, Writes: true}}, "seat:git.statuses": gitseat.StatusesAnswer{Commit: head, Combined: "pending", Statuses: []gitseat.Status{{Context: "mesh/check", State: "pending", By: "mesh"}}}, - mergeStateTool: map[string]any{"commit": head, "merge": map[string]any{"branch": "main", "mergeable": true}}, } for _, c := range []struct { name string @@ -48,10 +48,23 @@ func TestTheGateReadsAndHearsTheForgeThroughTheSeatGit(t *testing.T) { if err != nil { return nil, err } - return []any{c.Commit, *c.Pull, c.Statuses, *c.Merge.Mergeable}, nil + p := *c.Pull + conflict := *p.Mergeable + p.Mergeable = nil + return []any{c.Commit, p, c.Statuses, conflict, *c.Merge.Mergeable, c.Merge.Blocking[0].Context + " " + + c.Merge.Blocking[0].State, c.Merge.Says, *c.Merge.Writes}, nil }, want: []any{head, ForgePull{Number: 7, State: "closed", Merged: true, Base: "main", Head: "feat/x", HeadSHA: head, - MergeCommit: pull.MergeCommit, HTMLURL: pull.URL}, []ForgeStatus{{Context: "mesh/check", State: "pending", Creator: "mesh"}}, true}, - asked: []string{"seat:git.pull", "seat:git.statuses", mergeStateTool}}, + MergeCommit: pull.MergeCommit, HTMLURL: pull.URL}, []ForgeStatus{{Context: "mesh/check", State: "pending", Creator: "mesh"}}, + true, false, "mesh/check pending", "main requires mesh/check", true}, + asked: []string{"seat:git.pull", "seat:git.statuses"}}, + // What the forge would merge now comes from the pull answer's merging alone: none for a commit no pull request names. + {name: "no merge state for a commit alone", read: func(f seatForge) (any, error) { + c, err := f.Statuses("novox", "app", 0, head, "main") + if err != nil { + return nil, err + } + return []any{c.Merge.Mergeable == nil, c.Merge.Writes == nil}, nil + }, want: []any{true, true}, asked: []string{"seat:git.statuses"}}, {name: "a pull request not read", failed: "seat:git.pull", read: func(f seatForge) (any, error) { return f.Statuses("novox", "app", 7, "", "") }, asked: []string{"seat:git.pull"}}, diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 9d0edb064..28891258d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -115,7 +115,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { // merge state through the one statuses tool the seat does not carry yet (seatForge.mergeState); the forge's writes are the // bridge's (ADR 0437 §8). It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks // (ADR 0437 §6). - want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", mergeStateTool, "seat:mesh-hand-acts.hand-act"} + want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", "seat:mesh-hand-acts.hand-act"} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go index 5d57538e4..3f103955b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ports.go @@ -163,11 +163,13 @@ func gitAsk[A any](f seatForge, verb string, in any, errOf func(*A) string) (*A, } // Statuses reads a pull request by its number through `pull` (its head when no commit is named) and the commit's statuses -// through `statuses`. What the forge would merge now (the base's protection, its conflict word, whether the merging -// account writes) is read by mergeState. +// through `statuses`. What the forge would merge now (the base's protection and its required contexts, the forge's +// conflict word, whether the merging account writes) is the pull answer's `merging` (mergeState); a commit no pull +// request names has none. func (f seatForge) Statuses(owner, repo string, number int, commit, base string) (*ForgeChecks, error) { repository := owner + "/" + repo c := &ForgeChecks{Commit: commit} + var merging *gitseat.Merging if number > 0 { a, err := gitAsk(f, gitseat.VerbPull, gitseat.PullInput{Repository: repository, Number: int64(number)}, func(a *gitseat.PullAnswer) string { return a.Error }) @@ -177,7 +179,7 @@ func (f seatForge) Statuses(owner, repo string, number int, commit, base string) if a.Pull == nil { return nil, fmt.Errorf("the seat git's pull named no pull request for %s#%d", repository, number) } - c.Pull = forgePullOf(*a.Pull) + c.Pull, merging = forgePullOf(*a.Pull), a.Merging if c.Commit == "" { c.Commit = a.Pull.Head } @@ -197,50 +199,58 @@ func (f seatForge) Statuses(owner, repo string, number int, commit, base string) for _, st := range s.Statuses { c.Statuses = append(c.Statuses, forgeStatusOf(st)) } - merge, mergeable, err := f.mergeState(owner, repo, number, c.Commit, base) - if err != nil { - return nil, err - } - c.Merge = merge - if c.Pull != nil { - c.Pull.Mergeable = mergeable + c.Merge = mergeState(repository, base, merging, c.Statuses) + if c.Pull != nil && merging != nil { + mergeable := merging.Mergeable + c.Pull.Mergeable = &mergeable } return c, nil } -// mergeState is what the forge would merge now: the base's protection and its required statuses, the forge's own word on -// a conflict, and whether the merging account writes the repository. The seat `git` names none of them yet (dev-G-record's -// finding on W11.18): until it does, they are read through the forge holder's statuses tool, the one read left there. -func (f seatForge) mergeState(owner, repo string, number int, commit, base string) (ForgeMerge, *bool, error) { - args := map[string]any{"owner": owner, "repo": repo, "sha": commit} - if number > 0 { - args["number"] = number +// mergeState is what the forge would merge now, from the seat git's `merging` (issues 1275, 1945; ADR 0441 §1): the base's +// protection and its required contexts, each required context not a success at the commit blocking, and whether the +// merging account writes. Without `merging` (a commit no pull request names, or a pull request merged or closed) whether +// it merges is not known. +func mergeState(repository, base string, m *gitseat.Merging, statuses []ForgeStatus) ForgeMerge { + out := ForgeMerge{Branch: base, Required: []string{}} + if m == nil { + out.Says = "the seat git says what merges only for an open pull request" + return out } - if base != "" { - args["base"] = base + out.Protected, out.Required = m.Protected, append([]string{}, m.Required...) + newest := map[string]string{} + for _, s := range statuses { + if _, seen := newest[s.Context]; !seen { + newest[s.Context] = s.State + } } - raw, err := f.ask(mergeStateTool, args) - if err != nil { - return ForgeMerge{}, nil, err + for _, context := range out.Required { + if state := newest[context]; state != "success" { + if state == "" { + state = "missing" + } + out.Blocking = append(out.Blocking, struct { + Context string `json:"context"` + State string `json:"state"` + }{context, state}) + } } - answer, output, ok := answerOf(raw) - if !ok { - return ForgeMerge{}, nil, fmt.Errorf("the forge refused its merge state: %s", lastLine(output)) + mergeable := len(out.Blocking) == 0 + out.Mergeable = &mergeable + switch { + case len(out.Required) == 0: + out.Says = base + " requires no status" + default: + out.Says = base + " requires " + strings.Join(out.Required, ", ") } - var c ForgeChecks - if err := json.Unmarshal(answer, &c); err != nil || c.Commit == "" { - return ForgeMerge{}, nil, fmt.Errorf("the forge's merge state is not readable: %s", clip(string(answer), 200)) + writes := m.Writes + out.Writes = &writes + if !writes { + out.CannotWrite = "the forge's merging account may not write " + repository + ": its statuses and its merge are refused" } - var mergeable *bool - if c.Pull != nil { - mergeable = c.Pull.Mergeable - } - return c.Merge, mergeable, nil + return out } -// mergeStateTool is the one forge read the seat `git` does not carry yet (see mergeState). -const mergeStateTool = "gitea.gitea_commit_statuses" - // OpenPulls reads every open pull request through `open-pulls`. func (f seatForge) OpenPulls() ([]OpenPull, error) { a, err := gitAsk(f, gitseat.VerbOpenPulls, gitseat.OpenPullsInput{}, func(a *gitseat.OpenPullsAnswer) string { return a.Error }) diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index b10a9275a..d766a5f54 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -257,7 +257,6 @@ "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", - "gitea.gitea_commit_statuses", "seat:mesh-hand-acts.hand-act" ], "state": [ -- 2.54.0 From 00b95f1f6811889c0eee99e4e0ef9cb3cd6b277f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:33 +0100 Subject: [PATCH 046/243] Every act owed to the forge is said as the seat's event owed under one id and stays on its delivery's or group's state until its owed.ended is recorded (milestone 47/G, W11.17a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At the gate's start every unanswered act is said again under the same id; an outcome is written into the record that holds the pull request, then answered; an outcome other than performed is said; the checker's are not the gate's. Needs the forge bridge's forgemsg owed.ended (feat/47-G-forge). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from cf3715ea.) Builds: design 47 G §162, §163, §164, §165, §166 (GATE5) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bridge_test.go | 85 +++++++--- .../cmd/mesh-delivery-gate/bus_test.go | 16 +- .../cmd/mesh-delivery-gate/effects.go | 76 +++++++-- .../cmd/mesh-delivery-gate/fakes_test.go | 1 + .../cmd/mesh-delivery-gate/holder.go | 6 +- .../cmd/mesh-delivery-gate/issue1349_test.go | 2 +- .../cmd/mesh-delivery-gate/main.go | 7 +- .../cmd/mesh-delivery-gate/model.go | 3 + .../cmd/mesh-delivery-gate/owed.go | 89 ++++++---- .../cmd/mesh-delivery-gate/owed_test.go | 107 ++++++++++++ .../cmd/mesh-delivery-gate/owedended.go | 156 ++++++++++++++++++ .../cmd/mesh-delivery-gate/record.go | 7 +- modules/mesh-delivery-gate/module.json | 36 ++++ 13 files changed, 514 insertions(+), 77 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go index 39eed29a2..b3e2c98b4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go @@ -2,12 +2,15 @@ package main import ( "errors" + "fmt" "sort" "strconv" "strings" "sync" + "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) @@ -137,57 +140,95 @@ func (s *fakeSource) Move(update func(prev *Source) (Source, bool)) error { func sortStrings(s []string) { sort.Strings(s) } func itoa(n int) string { return strconv.Itoa(n) } -// fakeOwed is the gate's bucket owed with the forge bridge behind it: every act written is performed on the fake forge, as the -// bridge performs it (mesh-delivery modules/mesh-delivery-forge-bridge internal/bridge acts.go OwedAct), once for each -// revision. With down set the bucket cannot be written (the bus is away); the forge's own refusals are the bridge's to say, -// not the gate's to retry. +// fakeOwed is the seat's event owed with the forge bridge behind it: every act said is performed on the fake forge, as the +// bridge performs it through the seat git (mesh-delivery modules/mesh-delivery-forge-bridge internal/bridge owed.go), once +// for each id, and answered as owed.ended performed unless silent is set. With down set nothing can be said (the bus is +// away); the forge's own refusals are the bridge's to say, not the gate's to retry. type fakeOwed struct { - mu sync.Mutex - forge *fakeForge - down bool - // keys are the latest value of each key written, and writes every revision in order. + mu sync.Mutex + forge *fakeForge + h *Holder + down bool + silent bool + // keys are the latest act said under each key, in the gate's working form, writes every key said in order, and said + // every event in order. keys map[string]Owed writes []string + said []seat.Owed + ids map[string]bool } -func newFakeOwed(f *fakeForge) *fakeOwed { return &fakeOwed{forge: f, keys: map[string]Owed{}} } +func newFakeOwed(f *fakeForge) *fakeOwed { + return &fakeOwed{forge: f, keys: map[string]Owed{}, ids: map[string]bool{}} +} -func (o *fakeOwed) Put(key string, v Owed) error { - if _, err := mdc.EncodeOwed(v); err != nil { - return err +func (o *fakeOwed) Say(v seat.Owed) error { + key, _, found := strings.Cut(v.ID, "@") + if !found { + return fmt.Errorf("the act's id %q holds no @", v.ID) } - repository, number, head, act, err := mdc.ParseOwedKey(key) + repository, number, head, _, err := mdc.ParseOwedKey(key) if err != nil { return err } + if repository != v.Repository || int64(number) != v.Number || head != v.Head && v.Commit != head { + return fmt.Errorf("the act %s names %s#%d at %s", v.ID, v.Repository, v.Number, v.Head) + } o.mu.Lock() if o.down { defer o.mu.Unlock() return errors.New("the bus is away") } - if kept, ok := o.keys[key]; ok && mdc.SameOwed(kept, v) { + o.said = append(o.said, v) + if o.ids[v.ID] { + // Said again: the bridge performs it once and answers it again from what it performed. + silent, h := o.silent, o.h o.mu.Unlock() + if !silent && h != nil { + return h.OwedEnded(owedEndedBody(v.ID, forgemsg.OwedPerformed, "")) + } return nil } - o.keys[key] = v + o.ids[v.ID] = true + working := Owed{Act: v.Act, Context: v.Context, State: v.State, Target: v.Target, Text: v.Text} + switch v.Act { + case seat.OwedActStatus: + working.Text = v.Description + case seat.OwedActNote: + working.Text = v.Line + } + o.keys[key] = working o.writes = append(o.writes, key) + silent, h := o.silent, o.h o.mu.Unlock() owner, repo, _ := strings.Cut(repository, "/") f := o.forge switch v.Act { - case mdc.OwedNote: - _ = f.Note(owner, repo, head, v.Text) - case mdc.OwedView: + case seat.OwedActNote: + _ = f.Note(owner, repo, head, v.Line) + case seat.OwedActView: _ = f.View(owner, repo, number, v.Text) - case mdc.OwedStatus: - _ = f.Status(owner, repo, head, v.Context, v.State, v.Text, v.Target) - case mdc.OwedComment: + case seat.OwedActStatus: + _ = f.Status(owner, repo, head, v.Context, v.State, v.Description, v.Target) + case seat.OwedActComment: _ = f.Comment(owner, repo, number, v.Text) } - _ = act + if !silent && h != nil { + return h.OwedEnded(owedEndedBody(v.ID, forgemsg.OwedPerformed, "")) + } return nil } +// owedEndedBody is the forge bridge's owed.ended for one of the gate's ids. +func owedEndedBody(id, outcome, words string) []byte { + raw, err := forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{Of: seat.Seat, ID: id, Outcome: outcome, Words: words, + At: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC)}) + if err != nil { + panic(err) + } + return raw +} + // acts are the keys written whose act is of a kind, in order. func (o *fakeOwed) acts(kind string) []string { o.mu.Lock() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 2bfb10adf..7eaeaa47d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -127,7 +127,9 @@ type runtime struct { operatorAsks []oc.Ask // operatorCancels are the ids of the asks taken back from the router, in order (issue 1079). operatorCancels []string - openPulls []OpenPull + // owedSaid are the subjects of the seat events owed the gate said, in order (W11.17). + owedSaid []string + openPulls []OpenPull } type askedOfMesh struct { @@ -311,6 +313,11 @@ func (r *runtime) answer(m rpcMessage) { r.operatorAsks = append(r.operatorAsks, a) r.mu.Unlock() } + if strings.HasPrefix(p.Subject, "mesh.seat.mesh-delivery.event.owed.") { + r.mu.Lock() + r.owedSaid = append(r.owedSaid, p.Subject) + r.mu.Unlock() + } if strings.Contains(p.Subject, ".accept.cancel.") { var c struct { ID string `json:"id"` @@ -593,9 +600,10 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB t.Fatalf("the gate asked %s %d times: the bridge writes to the forge", tool, n) } } - rt.until("the owed keys to be written", eventWithin, func() bool { - keys, err := b.kvs["owed"].Keys(ctx) - return err == nil && len(keys) > 0 + rt.until("the acts owed to be said as the seat's event owed", eventWithin, func() bool { + rt.mu.Lock() + defer rt.mu.Unlock() + return len(rt.owedSaid) > 0 && strings.HasSuffix(rt.owedSaid[0], ".7") }) // The operator approves; the teller holds the warrant and writes the answer: the merge is handed to the bridge within // seconds, with the Approve's heads, and the gate asks the forge nothing. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/effects.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/effects.go index dcd1762fb..00bdb349d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/effects.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/effects.go @@ -1,6 +1,8 @@ package main import ( + mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "encoding/json" "fmt" "sort" @@ -173,15 +175,29 @@ func (h *Holder) Flush() { ids = append(ids, id) } sort.Strings(ids) + // An act said and not yet answered by its owed.ended waits for that answer; at this owner's start it is said once more, + // under the same id (design 47: every unanswered act said again at its start). + again := !h.saidAtStart + h.saidAtStart = true + waiting := func(e Effect) bool { + return e.Next != nil && now.Before(*e.Next) || e.Said != nil && !again + } for _, id := range ids { d := h.deliveries[id] if h.dirty[id] { continue } - for _, e := range d.Owed { - if e.Next != nil && now.Before(*e.Next) { + for i, e := range d.Owed { + if waiting(e) { continue } + if e.ID == "" { + // The id is kept before the act is said, so its owed.ended finds it however soon it is heard. + if e.ID = actIDOf(e, d.Repository, d.Number, d.Commit, d.HTMLURL, false); e.ID != "" { + d.Owed[i].ID = e.ID + h.keep(d) + } + } j := job{id: id, e: e, repo: d.Repository, num: d.Number, head: d.Commit, url: d.HTMLURL} if e.Kind == EffectView { j.body = ViewBody(d, h.groups[d.Group]) @@ -193,17 +209,23 @@ func (h *Holder) Flush() { if h.dirtyGroup[id] { continue } - for _, e := range g.Owed { - if e.Next != nil && now.Before(*e.Next) { + for i, e := range g.Owed { + if waiting(e) { continue } + if e.ID == "" { + if e.ID = actIDOf(e, "", 0, "", "", true); e.ID != "" { + g.Owed[i].ID = e.ID + h.keepGroup(g) + } + } jobs = append(jobs, job{id: id, group: true, e: e}) } } h.mu.Unlock() for _, j := range jobs { - err := h.do(j.e, j.repo, j.num, j.head, j.body, j.url, j.group) + said, err := h.do(j.e, j.repo, j.num, j.head, j.body, j.url, j.group) h.mu.Lock() // Why it is given up, when it is: a failure of the bus tried retryLimit times, the last ones retryCap apart. dropped := "" @@ -214,6 +236,15 @@ func (h *Holder) Flush() { for i, o := range list { if o.Kind == j.e.Kind && o.Commit == j.e.Commit && o.Context == j.e.Context && o.Line == j.e.Line && o.Since.Equal(j.e.Since) { + if err == nil && said != "" { + // Said: it stays until its owed.ended is heard (OwedEnded). + at := h.Now() + if o.Said != nil { + at = *o.Said + } + list[i].ID, list[i].Said, list[i].Next, list[i].Tries = said, &at, nil, 0 + return list + } if err == nil || dropped != "" { return append(list[:i:i], list[i+1:]...) } @@ -307,13 +338,14 @@ func retryAfter(tries int) time.Duration { return min(wait, retryCap) } -// do is one owed effect: said on the bus, or written as the key the forge bridge performs. -func (h *Holder) do(e Effect, repository string, number int, head, body, url string, group bool) error { +// do is one owed effect: said on the bus, or said as the seat's event owed, which the forge bridge performs; said is the +// act's id when it was said as owed, and empty when nothing waits for an answer. +func (h *Holder) do(e Effect, repository string, number int, head, body, url string, group bool) (said string, err error) { switch e.Kind { case EffectEmit: var payload any if err := json.Unmarshal([]byte(e.Line), &payload); err != nil { - return nil // nothing sayable: dropped + return "", nil // nothing sayable: dropped } event := "transition" if group { @@ -322,10 +354,10 @@ func (h *Holder) do(e Effect, repository string, number int, head, body, url str if e.Event != "" { event = e.Event } - return h.Emit(event, payload) + return "", h.Emit(event, payload) case EffectView: if number == 0 { - return nil + return "", nil } } page := url @@ -336,7 +368,27 @@ func (h *Holder) do(e Effect, repository string, number int, head, body, url str } act, value, num, hd := owedOf(e, repository, number, head, body, page) if act == "" { - return nil + return "", nil } - return h.owe(repository, num, hd, act, value) + return h.owe(repository, num, hd, act, value, e.Since, actSays(e), e.ID) +} + +// actIDOf is the id an effect is said under as owed, as do says it; empty for an effect said otherwise or not at all. +func actIDOf(e Effect, repository string, number int, head, url string, group bool) string { + switch { + case e.Kind == EffectEmit, e.Kind == EffectView && number == 0: + return "" + case group && e.Kind == EffectStatus: + repository, number, url = memberOfLast(e.Last) + head = e.Commit + } + act, _, num, hd := owedOf(e, repository, number, head, "", url) + if act == "" { + return "" + } + key, err := mdc.OwedKey(repository, num, hd, act) + if err != nil { + return "" + } + return actID(key, e.Since, actSays(e)) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go index 28842c05c..d966a7b8d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go @@ -323,6 +323,7 @@ func newWorldOver(t *testing.T, store *memStore, forge *fakeForge, now time.Time w.h.Merges, w.h.Source = w.merges, w.source w.merges.h = w.h w.owed = newFakeOwed(forge) + w.owed.h = w.h w.h.Owed = w.owed w.h.init() if err := w.h.Load(); err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index b22b758d5..14a0cf246 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -36,8 +36,8 @@ type Holder struct { Operator Operator Merges MergeStore Source SourceStore - // Owed is the gate's bucket owed: what a transition owes the forge, written as keys (owed.go). - Owed OwedStore + // Owed says what a transition owes the forge as the seat's event owed (owed.go). + Owed OwedSayer // Records keeps the delivery records on the gate's ledger (record.go, ADR 0656 §7); nil keeps none. Records *Records // Consumer applies the step events to the records (consumer.go); nil applies none. @@ -102,6 +102,8 @@ type Holder struct { reach map[string]viewstate.Reach // tried counts the owed effects Flush tried since this owner started, by how each ended (novox/hq issue 407). tried owedTries + // saidAtStart is whether this owner has said again, once, every act still unanswered (Flush). + saidAtStart bool // forgeWanted is whether the forge's open pull requests are to be read at the next React: a check may have // answered (events.go). reactMu keeps one React at a time. forgeWanted bool diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1349_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1349_test.go index 4ea57265d..bc3160bfa 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1349_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1349_test.go @@ -38,7 +38,7 @@ func TestATransitionWritesItsStatusViewAndNoteAsOwedKeys(t *testing.T) { if v := w.owed.keys[viewKey]; v.Act != mdc.OwedView || !strings.Contains(v.Text, id) { t.Fatalf("the owed view is %+v", v) } - if n := w.owed.keys[noteKey]; n.Act != mdc.OwedNote || n.Ref != "mesh-plan" || !strings.Contains(n.Text, "proposed") { + if n := w.owed.keys[noteKey]; n.Act != mdc.OwedNote || !strings.Contains(n.Text, "proposed") { t.Fatalf("the owed note is %+v", n) } // What the bridge would perform is what the forge holds. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 1fd9bcaa4..50a28b098 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -35,7 +35,7 @@ func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.n // The events this owner follows: the forge's pull requests, as the seat git says them (gitseat.go). The forge's status events // say that a check answered (events.go). The step events the delivery record is built from follow (consumer.go). -var followed = append(append(append([]string{}, gitEvents...), forgePattern, ConfigChangedEvent), stepPatterns()...) +var followed = append(append(append([]string{}, gitEvents...), forgePattern, owedEndedEvent, ConfigChangedEvent), stepPatterns()...) // forgePattern is the forge bridge's messages on FORGE (novox/hq ADR 0437 §8): the gate follows a merge and the refusal of one. const forgePattern = "mesh-delivery-forge-bridge.forge.**" @@ -67,7 +67,7 @@ func main() { Merges: kvMerges{}, Source: kvSource{}, Turns: kvTurns{}, - Owed: kvOwed{}, + Owed: seatOwed{say: stdio.SeatPublish}, Pulls: kvPulls{}, // The gate says what stalled to the operator itself (issue 1275, ADR 0437 §9). Say: sayOnSeat, @@ -209,6 +209,9 @@ func take(h *Holder, e stdio.Envelope) error { if h.Consumer != nil && h.Consumer.Takes(e.Key) { return h.Consumer.Hear(e) } + if e.Key == owedEndedEvent { + return h.OwedEnded(e.Body) + } if strings.HasPrefix(e.Key, "mesh-delivery-forge-bridge.forge.") { h.Forged(e.Key, e.Body) return nil diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 3b5b444ff..37e27c0bd 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -217,6 +217,9 @@ type Effect struct { Since time.Time `json:"since"` // Next is when a failed effect is tried again: later after every failure, up to retryCap. Next *time.Time `json:"next,omitempty"` + // ID and Said are the act's id and when it was first said as the event owed: it stays until its owed.ended is heard. + ID string `json:"id,omitempty"` + Said *time.Time `json:"said,omitempty"` } // The kinds of effect. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go index 2d5763d8d..9ee4587e9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go @@ -1,22 +1,25 @@ package main import ( - "encoding/json" "errors" "fmt" + "hash/fnv" "strconv" "strings" + "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - stdio "git.novox.be/novox/mesh-sdk/go" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/keys" ) -// The gate writes nothing to the forge: what a transition owes it — a status, the view on the pull request, a line of the -// commit's note, a comment — is one key of the gate's bucket owed, `...` (meshdeliveryclient -// OwedKey), which the forge bridge performs once for each revision (novox/hq ADR 0437 §8, ADR 0450 §4). The number is 0 for -// a commit no pull request names; a comment's act carries a sequence number, so two comments on one head are two acts. -// What the gate keeps is the effect owed until its key is written (effects.go); the forge's own refusal is the bridge's -// to say on FORGE, and the gate has no retry for it. +// The gate writes nothing to the forge: each act a transition owes it — a status, the view on the pull request, a line of +// the commit's note, a comment — is said as the event owed of the seat mesh-delivery, on +// `mesh.seat.mesh-delivery.event.owed..`, under one id, which the forge bridge performs through the +// seat git and answers as owed.ended (design 47 *What the gate owes the forge*). The act stays on its delivery's or group's +// state, already on the bus, until its owed.ended is heard; at the gate's start every act still unanswered is said again +// under the same id, and the forge holds it once. The number is 0 for a commit no pull request names; a comment's act +// carries a sequence number, so two comments on one head are two acts. // noPage is the target of a status whose delivery has no page to point at. const noPage = "none" @@ -24,34 +27,35 @@ const noPage = "none" // notesRef is the notes ref the commit's note is kept under. const notesRef = "mesh-plan" -// OwedStore is the gate's bucket owed. -type OwedStore interface { - // Put writes the act under its key, unless that key holds the same act already (a revision is an act to perform). - Put(key string, o Owed) error +// OwedSayer says one act owed to the forge as the seat mesh-delivery's event owed. +type OwedSayer interface { + Say(o seat.Owed) error } -// kvOwed is owed over the runtime. -type kvOwed struct{} +// seatOwed says owed on the seat's subject, under the act's id, which the bus keeps once. +type seatOwed struct{ say SeatSay } -func (kvOwed) Put(key string, o Owed) error { - raw, err := mdc.EncodeOwed(o) - if err != nil { - return err - } - if e, err := stdio.State("owed").Get(key); err == nil && e != nil { - if kept, err := mdc.DecodeOwed(e.Value); err == nil && mdc.SameOwed(kept, o) { - return nil - } - } - _, err = stdio.State("owed").Put(key, json.RawMessage(raw)) - return err +func (s seatOwed) Say(o seat.Owed) error { + return s.say(seat.SubjectOwed(keys.EscapeName(o.Repository), strconv.FormatInt(o.Number, 10)), o, "owed."+o.ID) } -// owe writes one owed act as its key. An act that has no key (a name the key refuses, a status with no page to point at) is -// said and dropped, since writing it again would fail the same. -func (h *Holder) owe(repository string, number int, head, act string, o Owed) error { +// actID is the id of an act owed: its key, when it was first owed and a digest of what it says, so it is the same each time +// it is said again and two acts of one key (a status set twice, two lines of the note in one moment) are two ids. It holds +// an @, which a switched act's id never holds. +func actID(key string, since time.Time, says string) string { + d := fnv.New32a() + _, _ = d.Write([]byte(says)) + return key + "@" + strconv.FormatInt(since.UnixNano(), 10) + "." + strconv.FormatUint(uint64(d.Sum32()), 16) +} + +// actSays is what an effect says, as its id's digest reads it. +func actSays(e Effect) string { return e.Kind + "\n" + e.Context + "\n" + e.State + "\n" + e.Line } + +// owe says one owed act under its id. An act that has no key (a name the key refuses, a status with no page to point at) +// is said as refused and dropped (an empty id), since saying it again would fail the same. +func (h *Holder) owe(repository string, number int, head, act string, o Owed, since time.Time, says, want string) (id string, err error) { if h.Owed == nil { - return errors.New("no owed bucket") + return "", errors.New("no seat to say owed on") } key, err := mdc.OwedKey(repository, number, head, act) if err == nil { @@ -60,9 +64,30 @@ func (h *Holder) owe(repository string, number int, head, act string, o Owed) er } if err != nil { h.Refuse(fmt.Errorf("the act %q on %s#%d at %s cannot be owed and is dropped: %v", act, repository, number, shortOf(head, 12), err)) - return nil + return "", nil } - return h.Owed.Put(key, o) + if id = actID(key, since, says); want != "" { + id = want + } + said := since.UTC() + return id, h.Owed.Say(seatOwedOf(o, id, repository, number, head, &said)) +} + +// seatOwedOf is an act in the gate's working form as the seat's event owed says it. +func seatOwedOf(o Owed, id, repository string, number int, head string, said *time.Time) seat.Owed { + out := seat.Owed{ID: id, Repository: repository, Number: int64(number), Head: head, Said: said, Text: o.Text} + switch o.Act { + case mdc.OwedStatus: + out.Act, out.Context, out.State, out.Description, out.Target, out.Commit = seat.OwedActStatus, o.Context, o.State, o.Text, o.Target, head + out.Text = "" + case mdc.OwedNote: + out.Act, out.Line, out.Commit, out.Text = seat.OwedActNote, o.Text, head, "" + case mdc.OwedView: + out.Act = seat.OwedActView + case mdc.OwedComment: + out.Act = seat.OwedActComment + } + return out } // actKind is the kind of an owed act: `status=` is a status, `comment.` a comment. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go new file mode 100644 index 000000000..6c1546f18 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" +) + +// W11.17 (design 47 *What the gate owes the forge*): every act owed is said as the seat's event owed under one id and stays +// on its delivery's state until its owed.ended is heard; at the owner's start every unanswered act is said again under the +// same id; an outcome is written into the record holding the pull request before the act is answered, once; an outcome +// other than performed is said; the checker's outcomes are not the gate's. +func TestAnActOwedStaysUntilItsOutcomeIsRecorded(t *testing.T) { + w := newWorld(t) + w.owed.silent = true + l := newLedgerFake(t) + id := IDOf("novox/app", head) + w.h.Records = &Records{Ledger: l, Now: w.h.Now} + if _, _, err := w.h.Records.Open("feat/x", map[string]string{id: head}, false); err != nil { + t.Fatal(err) + } + w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) + w.settleAll() + w.settleAll() + held := map[string]bool{} + for _, e := range w.delivery(id).Owed { + if e.Said != nil && e.ID != "" { + held[e.ID] = true + } + } + if len(held) < 3 { + t.Fatalf("the acts said and held on the delivery: %v; its owed %+v", held, w.delivery(id).Owed) + } + saidOnce := map[string]int{} + for _, o := range w.owed.said { + saidOnce[o.ID]++ + if !held[o.ID] && strings.Contains(o.ID, ".view@") { + t.Errorf("the view %s was said but is not held", o.ID) + } + } + for heldID := range held { + if saidOnce[heldID] != 1 { + t.Errorf("%s was said %d times before any answer, want once", heldID, saidOnce[heldID]) + } + } + + // The owner starts again over the same state: every unanswered act is said again, under the same id. + w2 := newWorldOver(t, w.store, w.forge, w.now) + w2.owed.silent = true + w2.h.Records = &Records{Ledger: l, Now: w2.h.Now} + w2.h.Flush() + again := map[string]bool{} + for _, o := range w2.owed.said { + again[o.ID] = true + } + for heldID := range held { + if !again[heldID] { + t.Errorf("%s was not said again at the start; said %v", heldID, again) + } + } + w2.h.Flush() + if len(w2.owed.said) != len(again) { + t.Errorf("a second flush said %d acts again, want none", len(w2.owed.said)-len(again)) + } + + var one string + for heldID := range held { + one = heldID + break + } + // The checker's outcome of an id is not the gate's: nothing is answered. + checkers, _ := forgemsg.EncodeOwedEnded(forgemsg.OwedEnded{Of: "mesh-delivery-checker", ID: one, Outcome: forgemsg.OwedPerformed, At: w.now}) + if err := w2.h.OwedEnded(checkers); err != nil || !owedHolds(w2, id, one) { + t.Fatalf("the checker's outcome answered the gate's act: %v", err) + } + // The ledger away: the act is not answered, and the event is heard again. + l.failOn = OpAct + if err := w2.h.OwedEnded(owedEndedBody(one, forgemsg.OwedRefused, "the branch is protected")); err == nil || !owedHolds(w2, id, one) { + t.Fatalf("an outcome the record could not take answered the act: %v", err) + } + l.failOn = "" + if err := w2.h.OwedEnded(owedEndedBody(one, forgemsg.OwedRefused, "the branch is protected")); err != nil || owedHolds(w2, id, one) { + t.Fatalf("the refused outcome did not answer the act: %v", err) + } + records, _ := w2.h.Records.Current() + if a := records[0].Acts[one]; a.Outcome != forgemsg.OwedRefused || a.Words != "the branch is protected" || a.Head != head || a.Number != 7 { + t.Fatalf("the record holds the act as %+v", a) + } + if !strings.Contains(w2.logs(), "was refused: the branch is protected") { + t.Errorf("a refused act was not said: %s", w2.logs()) + } + seq := records[0].Sequence + if err := w2.h.OwedEnded(owedEndedBody(one, forgemsg.OwedRefused, "the branch is protected")); err != nil || records[0].Sequence != seq { + t.Errorf("an outcome heard twice changed the record (%d, was %d): %v", records[0].Sequence, seq, err) + } +} + +// owedHolds is whether the delivery still holds the act unanswered. +func owedHolds(w *world, id, act string) bool { + for _, e := range w.delivery(id).Owed { + if e.ID == act { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go new file mode 100644 index 000000000..2f76733ad --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -0,0 +1,156 @@ +package main + +import ( + "fmt" + "strings" + "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" +) + +// owedEndedEvent is the forge bridge's word on an act owed, owed.ended on its stream forge (design 47 *What the gate owes +// the forge*): the outcome of one id, the gate's own or the checker's. +const owedEndedEvent = forgeBridgeModule + "." + forgemsg.OwedEndedEvent + +// OwedEnded takes the forge bridge's outcome of one act this owner said as owed. The outcome is written into the delivery +// record that holds the pull request at that head, and only then is the act answered: taken off its delivery's or group's +// state. An outcome heard twice, or of an id this owner holds no longer, changes nothing. An outcome other than performed +// is said. An error leaves the event to be heard again: the record could not take it. +func (h *Holder) OwedEnded(body []byte) error { + e, err := forgemsg.DecodeOwedEnded(body) + if err != nil { + h.Refuse(fmt.Errorf("an owed act's outcome the forge bridge said is not readable: %v", err)) + return nil + } + if e.Of != seat.Seat { + return nil // the checker's act + } + if e.ID == "" { + h.Refuse(fmt.Errorf("the forge bridge could not read an act this owner said: %s", e.Words)) + return nil + } + h.mu.Lock() + held, ok := h.heldAct(e.ID) + h.mu.Unlock() + if !ok { + return nil + } + if e.Outcome != forgemsg.OwedPerformed { + h.Refuse(fmt.Errorf("%s: the %s owed on %s#%d at %s was %s: %s", held.owner, held.effect.Kind, held.repository, + held.number, shortOf(held.head, 12), e.Outcome, e.Words)) + } + if err := h.recordAct(held, e); err != nil { + return err + } + h.mu.Lock() + defer h.mu.Unlock() + h.answer(e.ID) + return nil +} + +// heldAct is an act said and not yet answered, with where it is held. Under the lock. +type heldAct struct { + owner string // the delivery's or the group's id + effect Effect + repository, head string + number int +} + +func (h *Holder) heldAct(id string) (heldAct, bool) { + for did, d := range h.deliveries { + for _, o := range d.Owed { + if o.ID == id { + head := d.Commit + if o.Commit != "" { + head = o.Commit + } + return heldAct{owner: did, effect: o, repository: d.Repository, head: head, number: d.Number}, true + } + } + } + for gid, g := range h.groups { + for _, o := range g.Owed { + if o.ID == id { + repository, number, _ := memberOfLast(o.Last) + return heldAct{owner: gid, effect: o, repository: repository, head: o.Commit, number: number}, true + } + } + } + return heldAct{}, false +} + +// answer takes an answered act off its delivery's or group's state. Under the lock. +func (h *Holder) answer(id string) { + drop := func(list []Effect) ([]Effect, bool) { + for i, o := range list { + if o.ID == id { + return append(list[:i:i], list[i+1:]...), true + } + } + return list, false + } + for _, d := range h.deliveries { + var dropped bool + if d.Owed, dropped = drop(d.Owed); dropped { + h.keep(d) + } + } + for _, g := range h.groups { + var dropped bool + if g.Owed, dropped = drop(g.Owed); dropped { + h.keepGroup(g) + } + } +} + +// recordAct writes an act's outcome into the delivery record that holds its pull request at its head; none holds it +// before the Approve's merge is handed (switch.go), and then the outcome is on its delivery's transitions alone. +func (h *Holder) recordAct(a heldAct, e forgemsg.OwedEnded) error { + if h.Records == nil { + return nil + } + current, err := h.Records.Current() + if err != nil { + return err + } + prefix := strings.ToLower(a.repository) + "@" + for _, r := range current { + for member, head := range r.Heads { + if head == a.head && strings.HasPrefix(strings.ToLower(member), prefix) { + at := e.At.UTC() + return h.Records.Act(r, RecordAct{ID: e.ID, Act: a.effect.Kind, Repository: a.repository, Number: a.number, + Head: a.head, Outcome: e.Outcome, Words: e.Words, At: at}) + } + } + } + return nil +} + +// RecordAct is one act the gate owed the forge for a pull request the record holds, with the forge bridge's outcome. +type RecordAct struct { + ID string `json:"id"` + Act string `json:"act"` + Repository string `json:"repository"` + Number int `json:"number"` + Head string `json:"head"` + Outcome string `json:"outcome"` + Words string `json:"words,omitempty"` + At time.Time `json:"at"` +} + +// Act writes one act's outcome into the record, unless it holds that outcome already. +func (rs *Records) Act(r *Record, a RecordAct) error { + if had, ok := r.Acts[a.ID]; ok && had.Outcome == a.Outcome { + return nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpAct, a) + if err != nil { + return err + } + if r.Acts == nil { + r.Acts = map[string]RecordAct{} + } + r.Acts[a.ID], r.Sequence = a, seq + return nil +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 4bf027ce0..9cd40c3ce 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -53,13 +53,14 @@ const ( OpPin = "pin" // set-entry into pinned, by revision OpFree = "free" // remove from pinned, by revision OpEnd = "end" // set-fields: ended + OpAct = "act" // set-entry into acts, by id: an act owed the forge with its outcome OpRemove = "remove" // remove the instance, at its retention OpHold = "hold" // a month's list: set-entry into records, by instance OpDrop = "drop" // a month's list: remove from records, by instance ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpEnd, OpRemove} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpAct, OpEnd, OpRemove} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} @@ -94,7 +95,9 @@ type Record struct { Away map[string]NodeAway `json:"away,omitempty"` // Pinned are the graph revisions the record points at, by revision. Pinned map[string]PinnedRevision `json:"pinned,omitempty"` - Ended *RecordEnded `json:"ended,omitempty"` + // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go). + Acts map[string]RecordAct `json:"acts,omitempty"` + Ended *RecordEnded `json:"ended,omitempty"` } // RecordGroup is the group's state as the record keeps it: what `merges` held for the group, and its members. diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index d766a5f54..190578972 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -27,6 +27,7 @@ "git.forge.status", "git.repo.writable", "mesh-delivery-forge-bridge.forge.**", + "mesh-delivery-forge-bridge.owed.ended", "mesh-config.config-changed", "mesh-delivery-indexer.take-in.taken" ], @@ -109,6 +110,24 @@ }, "ended": { "type": "object" + }, + "id": { + "type": "string" + }, + "act": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "number": { + "type": "integer" + }, + "head": { + "type": "string" + }, + "words": { + "type": "string" } }, "operations": [ @@ -203,6 +222,23 @@ "into": "pinned", "key": "revision" }, + { + "name": "act", + "effect": "set-entry", + "fields": [ + "id", + "act", + "repository", + "number", + "head", + "outcome", + "words", + "at" + ], + "into": "acts", + "key": "id", + "sequence": true + }, { "name": "end", "effect": "set-fields", -- 2.54.0 From 32528aa58fb6a850773bea0c538b7a70be55fb3d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:33 +0100 Subject: [PATCH 047/243] An act owed to the forge with no outcome 15 minutes after it was said is listed by stalled (milestone 47/G, W11.17b) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 0fa400d5.) Builds: design 47 G §166, §169 (GATE6) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/owed_test.go | 22 +++++++++ .../cmd/mesh-delivery-gate/owedended.go | 45 +++++++++++++++++++ .../cmd/mesh-delivery-gate/verbs.go | 1 + 3 files changed, 68 insertions(+) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go index 6c1546f18..58b0b8858 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go @@ -3,6 +3,7 @@ package main import ( "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" ) @@ -45,6 +46,16 @@ func TestAnActOwedStaysUntilItsOutcomeIsRecorded(t *testing.T) { } } + // Unanswered, it is stalled only past 15 minutes, naming the oldest act. + if lines := stalledOf(w, id); len(lines) != 0 { + t.Fatalf("stalled before its bound: %+v", lines) + } + w.later(16 * time.Minute) + if lines := stalledOf(w, id); len(lines) != 1 || !strings.Contains(lines[0].Says, "has no outcome from the forge bridge") || + lines[0].Bound != "15m0s" { + t.Fatalf("an act unanswered past 15 minutes is not stalled: %+v", lines) + } + // The owner starts again over the same state: every unanswered act is said again, under the same id. w2 := newWorldOver(t, w.store, w.forge, w.now) w2.owed.silent = true @@ -96,6 +107,17 @@ func TestAnActOwedStaysUntilItsOutcomeIsRecorded(t *testing.T) { } } +// stalledOf are the stalled lines naming a delivery that say an act has no outcome. +func stalledOf(w *world, id string) []StalledLine { + var out []StalledLine + for _, l := range w.h.Stalled() { + if l.ID == id && strings.Contains(l.Says, "owed to the forge") { + out = append(out, l) + } + } + return out +} + // owedHolds is whether the delivery still holds the act unanswered. func owedHolds(w *world, id, act string) bool { for _, e := range w.delivery(id).Owed { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 2f76733ad..46039cc45 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -154,3 +154,48 @@ func (rs *Records) Act(r *Record, a RecordAct) error { r.Acts[a.ID], r.Sequence = a, seq return nil } + +// owedBound is how long an act said may wait for its owed.ended before it is stalled: many times a forge call's time, and +// short beside the check's hour (design 47 *What the gate owes the forge*). +const owedBound = 15 * time.Minute + +// owedLines are the stalled lines of the acts said and unanswered past owedBound: one per delivery or group, naming its +// oldest such act and how many more wait. Under the lock. +func (h *Holder) owedLines(now time.Time) []StalledLine { + var out []StalledLine + line := func(id string, state State, number int, owed []Effect) { + var oldest *Effect + more := 0 + for i, e := range owed { + if e.Said == nil || now.Sub(*e.Said) < owedBound { + continue + } + if oldest == nil || e.Said.Before(*oldest.Said) { + if oldest != nil { + more++ + } + oldest = &owed[i] + } else { + more++ + } + } + if oldest == nil { + return + } + says := fmt.Sprintf("the %s owed to the forge, said at %s as %s, has no outcome from the forge bridge", oldest.Kind, + oldest.Said.UTC().Format(time.RFC3339), oldest.ID) + if more > 0 { + says += fmt.Sprintf(", nor have %d more", more) + } + out = append(out, StalledLine{ID: id, Number: number, State: state, For: now.Sub(*oldest.Said).Round(time.Second).String(), + Bound: owedBound.String(), H2: "none: the forge bridge's bridge_owed reads what it performed — the operator's", + Says: says + "; it is said again at the gate's start"}) + } + for id, d := range h.deliveries { + line(id, d.State, d.Number, d.Owed) + } + for id, g := range h.groups { + line(id, "", 0, g.Owed) + } + return out +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 7abc79052..019a320c8 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -230,6 +230,7 @@ func (h *Holder) Stalled() []StalledLine { out = append(out, h.readyLines(now)...) out = append(out, h.turnLines(now)...) // an approved merge held past its bound by another delivery's check (ADR 0548 §4) out = append(out, h.groupLines(now)...) + out = append(out, h.owedLines(now)...) // an act owed the forge with no outcome past its bound (W11.17) sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) return out } -- 2.54.0 From c6cf8b64f1b476abac7ba9bf16c9bfe60ce861af Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:33 +0100 Subject: [PATCH 048/243] The gate serves the seat mesh-delivery's verb source: each repository's trunk branch and head and the revision, read from source.main (milestone 47/G, source verb) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node-engine reads the repositories here to call a reverted build's catch-up tool once per repository, never the gate's state. Rows in issue 888's test and the bus test, which call the verb over the runtime. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from b45fd0ab.) Builds: design 47 G §15 in part (GATE7) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bridge_test.go | 11 ++++++ .../cmd/mesh-delivery-gate/bus_test.go | 5 +++ .../cmd/mesh-delivery-gate/issue888_test.go | 7 ++++ .../cmd/mesh-delivery-gate/main.go | 6 +++ .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/merges.go | 37 +++++++++++++++++++ modules/mesh-delivery-gate/module.json | 1 + 7 files changed, 68 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go index b3e2c98b4..fb246c590 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go @@ -137,6 +137,17 @@ func (s *fakeSource) Move(update func(prev *Source) (Source, bool)) error { return nil } +// Read is source.main as held, and its revision: the number of writes. +func (s *fakeSource) Read() (*Source, uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.main == nil { + return nil, 0, nil + } + held := *s.main + return &held, uint64(len(s.moves)), nil +} + func sortStrings(s []string) { sort.Strings(s) } func itoa(n int) string { return strconv.Itoa(n) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 7eaeaa47d..a33421423 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -631,6 +631,11 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB src, err := mdc.DecodeSource(e.Value()) return err == nil && src.Group == "fix/x" && src.Heads["novox/app"] == strings.Repeat("b", 40) }) + // The seat's verb source answers it from the bucket, with its revision on the bus. + said := strings.ReplaceAll(rt.call(t, Seat+".source", map[string]any{}), `\"`, `"`) + if !strings.Contains(said, `{"head":"`+strings.Repeat("b", 40)+`","repository":"novox/app","trunk":"main"}`) || !strings.Contains(said, `"revision":`) { + t.Fatalf("source answers %s", said) + } // The delivery ends merged (ADR 0539 §17), and the gate asked the controller nothing (R6-C3). rt.until("the delivery to be merged", eventWithin, func() bool { e, err := b.kvs["deliveries"].Get(ctx, kvKey(IDOf("novox/app", head))) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go index 68cd6b853..ab6b42d94 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go @@ -5,6 +5,7 @@ import ( "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" ) // Issue 888: the checker judges on the trunk source.main names (mesh-delivery modules/mesh-delivery-checker main.go), and the @@ -38,6 +39,12 @@ func TestAMergeHeardOnlyFromTheForgeMovesSourceMain(t *testing.T) { if w.source.main == nil || w.source.main.Heads[i888App] != i888Merge1 || w.source.main.Heads[i888Other] != i888OtherH { t.Fatalf("source.main is %+v; wanted %s at %s and %s kept", w.source.main, i888App, i888Merge1, i888Other) } + // The seat's verb source answers each repository's trunk branch and head, and the revision, from the same key. + if a := w.h.SourceRevision(); a.Error != "" || a.Revision != 1 || len(a.Repositories) != 2 || + a.Repositories[0] != (seat.SourceHead{Repository: i888App, Trunk: MergeBranch, Head: i888Merge1}) || + a.Repositories[1].Repository != i888Other || a.Repositories[1].Head != i888OtherH { + t.Fatalf("source answers %+v", a) + } n := len(w.source.moves) w.settleAll() w.h.PullMerged(i888MergedAt(i888App, 3, i888Head1, i888Merge1, w.now.Add(-time.Minute))) // heard again diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 50a28b098..f6841d265 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -354,6 +354,12 @@ func tools(h *Holder, l *listening) []stdio.Tool { Input: map[string]any{"type": "object", "properties": map[string]any{ "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}}, Run: func(a map[string]any) (any, error) { return h.Groups(boolArg(a, "all")), nil }}, + {Name: seat + "source", + Description: "The source revision the gate holds: every repository it names, its trunk branch and that trunk's head, the " + + "group that moved it last and its revision. Read, never written: a node-engine reads the repositories here to call a " + + "reverted build's catch-up tool once per repository, never the gate's state.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { return h.SourceRevision(), nil }}, {Name: seat + "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each of the commit's " + "statuses (mesh/check, mesh/delivery, …) with its state, description and when it " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 28891258d..e61ec7458 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -75,7 +75,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans; and // settle, a person's ask to settle merges held without a passing check (ADR 0556 clause 5). - if want := []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close", "settle"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { + if want := []string{"deliveries", "show", "groups", "source", "checks", "table", "stalled", "recheck", "close", "settle"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want) } // The forge messages' layout is the bridge's published package forgemsg, taken by its series (ADR 0634 §3, design 32). diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index 2cbf862af..1fd3797f7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -11,6 +11,7 @@ import ( "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -32,6 +33,8 @@ type SourceStore interface { // Move reads source.main and writes what update makes of it, unless update says nothing is to be written; a write // another writer won is read again and tried again. Move(update func(prev *Source) (Source, bool)) error + // Read is source.main as held and its revision on the bus; nil and 0 before it is first written. + Read() (*Source, uint64, error) } // kvMerges is merges over the runtime. @@ -107,6 +110,40 @@ func (kvSource) Move(update func(prev *Source) (Source, bool)) error { return fmt.Errorf("source.main was written by another writer first %d times: %w", changeAttempts, stdio.ErrChanged) } +func (kvSource) Read() (*Source, uint64, error) { + e, err := stdio.State("source").Get(mdc.SourceMainKey) + if err != nil || e == nil { + return nil, 0, err + } + held, err := mdc.DecodeSource(e.Value) + if err != nil { + return nil, 0, fmt.Errorf("source.main is not readable: %w", err) + } + return &held, e.Revision, nil +} + +// SourceRevision is the seat's verb source: every repository source.main names, its trunk branch and head, the group that +// moved it last and its revision, or why it cannot be read. source.main names each repository's main only (MergeBranch). +// Outside the lock. +func (h *Holder) SourceRevision() seat.SourceAnswer { + if h.Source == nil { + return seat.SourceAnswer{Error: "no source bucket"} + } + held, revision, err := h.Source.Read() + if err != nil { + return seat.SourceAnswer{Error: err.Error()} + } + if held == nil { + return seat.SourceAnswer{} + } + out := seat.SourceAnswer{Group: held.Group, Revision: int64(revision)} + for repository, head := range held.Heads { + out.Repositories = append(out.Repositories, seat.SourceHead{Repository: repository, Trunk: MergeBranch, Head: head}) + } + sort.Slice(out.Repositories, func(i, j int) bool { return out.Repositories[i].Repository < out.Repositories[j].Repository }) + return out +} + // SourceMain is source.main as the bucket source holds it, read and never written: its group and each repository's trunk // head, or why it cannot be read. Outside the lock. func (h *Holder) SourceMain() map[string]any { diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 190578972..cf20565ca 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -10,6 +10,7 @@ "deliveries", "show", "groups", + "source", "checks", "table", "stalled", -- 2.54.0 From 64ed69db9b09ad61e94c576e74906aebeb2adc52 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:37:37 +0100 Subject: [PATCH 049/243] The gate's groups answers the seat's groups: members in merge order, the acts said with their ids and the acts recorded with their outcomes, one group alone, and none for a group it does not know (milestone 47/G, W11.10; F-forge-7, F-forge-8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The forge bridge reads a group's members and what it owes from here, never from the gate's state. The text form of the members and composed_gate are gone with the answer of before. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from de368344.) Builds: design 47 G §157, §304 (GATE5) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/group_test.go | 53 ++++++++--- .../cmd/mesh-delivery-gate/holder_test.go | 22 ++--- .../cmd/mesh-delivery-gate/main.go | 13 +-- .../cmd/mesh-delivery-gate/owed.go | 19 ++++ .../cmd/mesh-delivery-gate/verbs.go | 94 +++++++++++-------- 5 files changed, 133 insertions(+), 68 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_test.go index 9e5ac63ff..635035d4c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_test.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "errors" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "strings" "testing" "time" @@ -82,20 +83,48 @@ func TestAFeaturesSinglePullRequestIsALoneDeliveryAskedAlone(t *testing.T) { } } -// A group answer says its members' gate is the composed check's (agreed with the controller, issue 977). -func TestTheGroupsAnswerSaysItsGateIsTheComposedChecks(t *testing.T) { - w := approvedGroup(t) - lines := w.h.Groups(false) - if len(lines) != 1 || !lines[0].ComposedGate { - t.Fatalf("groups: %+v", lines) +// W11.10 (design 47; F-forge-7, F-forge-8): groups answers the seat's groups: the members in merge order (the SDK first, +// whatever its id), the acts said and not answered with their ids, the acts a record holds with their outcomes, and for a +// group it does not know no groups and no error. +func TestTheGroupsAnswerIsTheSeatsGroupInMergeOrderWithItsActs(t *testing.T) { + w := approvedGroupOf(t, member{"novox/app", 1, gHc, "go.mod"}, member{SDKRepository, 2, gHk, "go/x.go"}) + app := w.h.deliveries[IDOf("novox/app", gHc)] + said := w.now + app.Owed = append(app.Owed, Effect{Kind: EffectStatus, Context: "mesh/merge-gate", State: "pending", Line: "waits", + Commit: gHc, Since: said, ID: "status.x", Said: &said}) + a := w.h.Groups(false, "") + if a.Error != "" || len(a.Groups) != 1 || len(a.Groups[0].Members) != 2 { + t.Fatalf("groups answers %+v", a) } - raw, _ := json.Marshal(lines[0]) - var got map[string]any - if err := json.Unmarshal(raw, &got); err != nil || got["composed_gate"] != true { - t.Fatalf("the group line says %s", raw) + g := a.Groups[0] + if m := g.Members[0]; m.Repository != SDKRepository || m.Number != 2 || m.Head != gHk || m.Base != "main" || !m.Open { + t.Fatalf("the first member in merge order is %+v, not the SDK", m) } - if _, ordered := got["order"]; ordered { - t.Fatalf("a group line still says an order: %s", raw) + var act *seat.Act + for i := range g.Acts { + if g.Acts[i].Owed.ID == "status.x" { + act = &g.Acts[i] + } + } + if act == nil || act.Outcome != "" || act.Owed.Act != seat.OwedActStatus || act.Owed.Repository != "novox/app" || + act.Owed.Number != 1 || act.Owed.Head != gHc || act.Owed.Context != "mesh/merge-gate" || act.Owed.Said == nil { + t.Fatalf("the act said and not answered is %+v in %+v", act, g.Acts) + } + if one := w.h.Groups(false, "feat/g"); len(one.Groups) != 1 || one.Groups[0].ID != "feat/g" { + t.Fatalf("groups of feat/g answers %+v", one) + } + none := w.h.Groups(true, "feat/nothing") + if raw, _ := json.Marshal(none); none.Error != "" || len(none.Groups) != 0 || strings.Contains(string(raw), "error") { + t.Fatalf("a group it does not know is answered %s", raw) + } + at := w.now + recorded := recordedActs([]*Record{{Acts: map[string]RecordAct{ + "note.y": {ID: "note.y", Act: EffectNote, Repository: "novox/App", Number: 1, Head: gHc, Outcome: seat.ActOutcomeRefused, Words: "no", At: at}, + "other": {ID: "other", Act: EffectNote, Repository: "novox/other", Number: 9, Head: gHc, Outcome: seat.ActOutcomePerformed, At: at}, + }}}, map[string]bool{"novox/app@" + gHc: true}) + if len(recorded) != 1 || recorded[0].Owed.ID != "note.y" || recorded[0].Outcome != seat.ActOutcomeRefused || + recorded[0].Words != "no" || recorded[0].Owed.Act != seat.OwedActNote || recorded[0].At == nil { + t.Fatalf("the recorded acts are %+v", recorded) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go index e62fe9aa4..73ec828ae 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "strings" "testing" "time" @@ -336,8 +337,8 @@ func TestStalledAndCloseWorkFromTheTable(t *testing.T) { } } -// novox/hq issue 867, ADR 0382 §1 (contract A): each member of a group in `groups` also says its pull -// request number, full head, base branch and whether it is open; `members` keeps its text form. +// novox/hq issue 867, ADR 0382 §1 (contract A): each member of a group in `groups` says its pull request number, full +// head, base branch and whether it is open, under the seat's keys. func TestGroupsNamesEachMembersPullRequestHeadBaseAndOpen(t *testing.T) { w := newWorld(t) headA := "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" @@ -346,28 +347,25 @@ func TestGroupsNamesEachMembersPullRequestHeadBaseAndOpen(t *testing.T) { pb.Base = "release/1" w.h.PullUpdated(pr("novox/a", 1, headA, "feat/z")) w.h.PullUpdated(pb) - lines := w.h.Groups(false) - if len(lines) != 1 || len(lines[0].Details) != 2 || len(lines[0].Members) != 2 { + lines := w.h.Groups(false, "").Groups + if len(lines) != 1 || len(lines[0].Members) != 2 { t.Fatalf("groups: %+v", lines) } - got := map[string]MemberLine{} - for _, m := range lines[0].Details { + got := map[string]seat.Member{} + for _, m := range lines[0].Members { got[m.Repository] = m } a, b := got["novox/a"], got["novox/b"] - if a.Number != 1 || a.Head != headA || a.Base != "main" || !a.Open || a.ID == "" { + if a.Number != 1 || a.Head != headA || a.Base != "main" || !a.Open || a.ID == "" || a.State == "" { t.Fatalf("member a: %+v", a) } if b.Number != 2 || b.Head != headB || b.Base != "release/1" || !b.Open { t.Fatalf("member b: %+v", b) } - if !strings.Contains(lines[0].Members[0], "(") { - t.Fatalf("members lost its text form: %v", lines[0].Members) - } // Closed unmerged: still listed while the group is, but not open. w.h.PullClosed(pr("novox/b", 2, headB, "feat/z")) - for _, l := range w.h.Groups(true) { - for _, m := range l.Details { + for _, l := range w.h.Groups(true, "").Groups { + for _, m := range l.Members { if m.Repository == "novox/b" && m.Open { t.Fatalf("a closed pull request is open: %+v", m) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index f6841d265..e5af7a18c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -347,13 +347,14 @@ func tools(h *Holder, l *listening) []stdio.Tool { "required": []string{"id"}}, Run: func(a map[string]any) (any, error) { return h.Show(strArg(a, "id")) }}, {Name: seat + "groups", - Description: "Every delivery group: its members, its composed check and its state, derived from its members. " + - "A group has no order: it is merged in one pass, and moves source.main once (novox/hq ADR 0418, ADR 0539 clause 17); what " + - "reached the machines is the delivery view's. On mesh/check its check is the checker's one verdict at every member's head " + - "(ADR 0447); composed_gate says its members' gate is the controller's composed check's (ADR 0403), for a base still on it.", + Description: "Every delivery group: its members in merge order, its check and its state, derived from its members, " + + "and the acts the gate said it owes the forge for it, each with its id and its recorded outcome (design 47 *What the " + + "gate owes the forge*). With a group, that group alone, ended or not, and no groups for a group it does not know. " + + "The group moves the source revision once (novox/hq ADR 0418); what reached the machines is the delivery view's.", Input: map[string]any{"type": "object", "properties": map[string]any{ - "all": map[string]any{"type": "string", "enum": []string{"true", "false"}}}}, - Run: func(a map[string]any) (any, error) { return h.Groups(boolArg(a, "all")), nil }}, + "all": map[string]any{"type": "string", "enum": []string{"true", "false"}, "description": "\"true\": the groups that ended too"}, + "group": str("one group's id: that group alone, ended or not")}}, + Run: func(a map[string]any) (any, error) { return h.Groups(boolArg(a, "all"), strArg(a, "group")), nil }}, {Name: seat + "source", Description: "The source revision the gate holds: every repository it names, its trunk branch and that trunk's head, the " + "group that moved it last and its revision. Read, never written: a node-engine reads the repositories here to call a " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go index 9ee4587e9..6b0eaa6f7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed.go @@ -90,6 +90,25 @@ func seatOwedOf(o Owed, id, repository string, number int, head string, said *ti return out } +// owedSaid is an effect said as owed, as do says it: the act its id names, with its repository, number and head (for a +// group's status, the member's its Last names). False for an effect that is not said as owed, or has no id yet. +func owedSaid(e Effect, repository string, number int, head, body, url string, group bool) (seat.Owed, bool) { + if e.ID == "" || e.Kind == EffectEmit || e.Kind == EffectView && number == 0 { + return seat.Owed{}, false + } + page := url + if group && e.Kind == EffectStatus { + repository, number, page = memberOfLast(e.Last) + head = e.Commit + } + act, value, num, hd := owedOf(e, repository, number, head, body, page) + if act == "" { + return seat.Owed{}, false + } + value.Act = actKind(act) + return seatOwedOf(value, e.ID, repository, num, hd, e.Said), true +} + // actKind is the kind of an owed act: `status=` is a status, `comment.` a comment. func actKind(act string) string { kind, _, _ := mdc.ParseOwedAct(act) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 019a320c8..ff854fe20 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -5,6 +5,8 @@ import ( "sort" "strings" "time" + + seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" ) // The seat's verbs (novox/hq ADR 0239, the mesh-delivery seat in the controller's set): six that read, and @@ -100,64 +102,80 @@ func (h *Holder) Show(id string) (any, error) { return nil, fmt.Errorf("no delivery or group %q: `deliveries` lists them", id) } -// GroupLine is one group as `groups` lists it. -type GroupLine struct { - ID string `json:"id"` - State string `json:"state"` - Why string `json:"why"` - // ComposedGate says the group's gate is the controller's composed check's: each member's own pull request check runs - // the repository's check alone, and the composed verdict sets its mesh/merge-gate (novox/hq ADR 0403 §1, issue 978). - // On mesh/check the group's check is the checker's one verdict at every member's head (ADR 0447 §2, issue 1302). - ComposedGate bool `json:"composed_gate"` - Check string `json:"composed,omitempty"` - Members []string `json:"members"` - // Details are the members again, one object each, with what a reader needs to find the pull request's - // siblings: its number, full head, base branch and whether it is open (novox/hq ADR 0382 §1). Additive: - // `members` keeps its text form for older readers. - Details []MemberLine `json:"member_details,omitempty"` -} - -// MemberLine is one member of a group, as `groups` lists it beside its text line. -type MemberLine struct { - ID string `json:"id"` - Repository string `json:"repository"` - State State `json:"state"` - Number int `json:"number"` - Head string `json:"head"` - Base string `json:"base"` - Open bool `json:"open"` -} - -// Groups is the `groups` verb. -func (h *Holder) Groups(all bool) []GroupLine { +// Groups is the seat's verb groups: every delivery group not ended (all: those that ended too), or with only that group +// alone, ended or not, and none for a group it does not know. Each with its members in merge order (membersOf), its state +// and check, and the acts the gate said for it and its members: each unanswered act as it was said (the forge bridge's to +// perform), and each act a delivery record holds with the outcome recorded (owedended.go). Outside the lock: the records +// are read first. +func (h *Holder) Groups(all bool, only string) seat.GroupsAnswer { + var kept []*Record + if h.Records != nil { + var err error + if kept, err = h.Records.Kept(); err != nil { + return seat.GroupsAnswer{Error: "the delivery records cannot be read: " + err.Error()} + } + } h.mu.Lock() defer h.mu.Unlock() - var out []GroupLine + out := seat.GroupsAnswer{Groups: []seat.Group{}} for _, g := range h.groups { + if only != "" && g.ID != only { + continue + } members := h.membersOf(g) if len(members) == 0 { continue } state, why := GroupState(g, members) - if !all && (state == GroupMerged || state == GroupClosed) { + if only == "" && !all && (state == GroupMerged || state == GroupClosed) { continue } - l := GroupLine{ID: g.ID, State: state, Why: why, ComposedGate: true} + l := seat.Group{ID: g.ID, State: state, Why: why, Members: []seat.Member{}, Acts: []seat.Act{}} + heads := map[string]bool{} for _, d := range members { - l.Members = append(l.Members, fmt.Sprintf("%s (%s)", d.ID, d.State)) - l.Details = append(l.Details, MemberLine{ID: d.ID, Repository: d.Repository, State: d.State, - Number: d.Number, Head: d.Commit, Base: d.Base, + l.Members = append(l.Members, seat.Member{ID: d.ID, Repository: d.Repository, State: string(d.State), + Number: int64(d.Number), Head: d.Commit, Base: d.Base, Open: !d.State.Final() && d.MergedAs == "" && !d.ClosedUnmerged}) + heads[strings.ToLower(d.Repository)+"@"+d.Commit] = true + for _, e := range d.Owed { + if o, ok := owedSaid(e, d.Repository, d.Number, d.Commit, ViewBody(d, g), d.HTMLURL, false); ok { + l.Acts = append(l.Acts, seat.Act{Owed: o}) + } + } } + for _, e := range g.Owed { + if o, ok := owedSaid(e, "", 0, "", "", "", true); ok { + l.Acts = append(l.Acts, seat.Act{Owed: o}) + } + } + l.Acts = append(l.Acts, recordedActs(kept, heads)...) if g.Check != nil { l.Check = g.Check.Verdict if l.Check == "" { l.Check = "asked as " + g.Check.Asked } } - out = append(out, l) + out.Groups = append(out.Groups, l) } - sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + sort.Slice(out.Groups, func(i, j int) bool { return out.Groups[i].ID < out.Groups[j].ID }) + return out +} + +// recordedActs are the acts the kept records hold for pull requests at these heads (`@`, lower case), +// each with its outcome, ordered by id. +func recordedActs(kept []*Record, heads map[string]bool) []seat.Act { + var out []seat.Act + for _, r := range kept { + for _, a := range r.Acts { + if !heads[strings.ToLower(a.Repository)+"@"+a.Head] { + continue + } + at := a.At + out = append(out, seat.Act{Outcome: a.Outcome, Words: a.Words, At: &at, Owed: seat.Owed{ID: a.ID, + Act: a.Act, Repository: a.Repository, Number: int64(a.Number), Head: a.Head}}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Owed.ID < out[j].Owed.ID }) return out } -- 2.54.0 From badfe0d16617be3787dfc28a6d224a4215ab58ec Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:40:34 +0100 Subject: [PATCH 050/243] The switch says every act the old states owed and merges hold under its switched id, so the forge bridge performs only what the old bridge did not (milestone 47/G, W11.17d carry) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every key of owed, and the member merging in every key of merges, is said as owed under meshdeliveryclient.SwitchedActID(state, key, revision), at every switch, the same act each time. The switch reads the old states through one reader (OldState). Rows in the switch's test. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from dd0c4294.) Builds: design 47 G §169 (GATE5, GATE6); only the owed carry: the member merging in the old state merges is not carried as a merge act (G §167, §542) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bridge_test.go | 47 +++++++++++++++ .../cmd/mesh-delivery-gate/fakes_test.go | 4 +- .../cmd/mesh-delivery-gate/holder.go | 1 + .../cmd/mesh-delivery-gate/main.go | 1 + .../cmd/mesh-delivery-gate/merges.go | 20 +++++++ .../cmd/mesh-delivery-gate/record_test.go | 21 +++++++ .../cmd/mesh-delivery-gate/switch.go | 60 +++++++++++++++++++ 7 files changed, 153 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go index fb246c590..13770632f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go @@ -115,6 +115,47 @@ func (m *fakeMerges) last(group string) Merge { return m.values[group] } +// fakeOld is the gate's old states merges and owed, which only the switch reads. +type fakeOld struct { + mu sync.Mutex + states map[string]map[string]oldEntry +} + +type oldEntry struct { + raw []byte + revision uint64 +} + +func (o *fakeOld) put(state, key string, raw []byte, revision uint64) { + o.mu.Lock() + defer o.mu.Unlock() + if o.states[state] == nil { + o.states[state] = map[string]oldEntry{} + } + o.states[state][key] = oldEntry{raw: raw, revision: revision} +} + +func (o *fakeOld) Keys(state string) ([]string, error) { + o.mu.Lock() + defer o.mu.Unlock() + var out []string + for k := range o.states[state] { + out = append(out, k) + } + sort.Strings(out) + return out, nil +} + +func (o *fakeOld) Get(state, key string) ([]byte, uint64, error) { + o.mu.Lock() + defer o.mu.Unlock() + e, ok := o.states[state][key] + if !ok { + return nil, 0, nil + } + return e.raw, e.revision, nil +} + // fakeSource is the gate's bucket source with its one key. type fakeSource struct { mu sync.Mutex @@ -174,6 +215,12 @@ func newFakeOwed(f *fakeForge) *fakeOwed { } func (o *fakeOwed) Say(v seat.Owed) error { + if _, _, _, switched := mdc.ParseSwitchedActID(v.ID); switched { + o.mu.Lock() + defer o.mu.Unlock() + o.said = append(o.said, v) // the bridge counts it against what the old bridge performed + return nil + } key, _, found := strings.Cut(v.ID, "@") if !found { return fmt.Errorf("the act's id %q holds no @", v.ID) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go index d966a7b8d..4271e3165 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go @@ -287,6 +287,7 @@ type world struct { forge *fakeForge op *fakeOperator merges *fakeMerges + old *fakeOld source *fakeSource owed *fakeOwed // asked is the key of the ask a test answered. @@ -320,7 +321,8 @@ func newWorldOver(t *testing.T, store *memStore, forge *fakeForge, now time.Time w.op.auto = true w.h.Operator = w.op w.merges, w.source = newFakeMerges(forge), &fakeSource{} - w.h.Merges, w.h.Source = w.merges, w.source + w.old = &fakeOld{states: map[string]map[string]oldEntry{}} + w.h.Merges, w.h.Old, w.h.Source = w.merges, w.old, w.source w.merges.h = w.h w.owed = newFakeOwed(forge) w.owed.h = w.h diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 14a0cf246..9ba599e69 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -35,6 +35,7 @@ type Holder struct { // (approval.go); Merges and Source are the gate's merges and source (merges.go). Operator Operator Merges MergeStore + Old OldState Source SourceStore // Owed says what a transition owes the forge as the seat's event owed (owed.go). Owed OwedSayer diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index e5af7a18c..8f1721dfd 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -77,6 +77,7 @@ func main() { h.SaySeat = stdio.SeatPublish h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish} h.Merges = recordedMerges{MergeStore: kvMerges{}, records: h.Records, members: h.groupMembers} + h.Old = kvOld{} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf} l := &listening{now: "not yet: starting"} go run(h, l) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index 1fd3797f7..1404f6237 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -28,6 +28,26 @@ type MergeStore interface { Put(group string, m Merge) error } +// OldState reads the gate's old state owed, which the switch says again as switched acts (switch.go): each key, and one +// key's value with its revision. +type OldState interface { + Keys(state string) ([]string, error) + Get(state, key string) (value []byte, revision uint64, err error) +} + +// kvOld is the old state over the runtime. +type kvOld struct{} + +func (kvOld) Keys(state string) ([]string, error) { return stdio.State(state).Keys() } + +func (kvOld) Get(state, key string) ([]byte, uint64, error) { + e, err := stdio.State(state).Get(key) + if err != nil || e == nil { + return nil, 0, err + } + return e.Value, e.Revision, nil +} + // SourceStore is the gate's bucket source, one key, moved by compare-and-set. type SourceStore interface { // Move reads source.main and writes what update makes of it, unless update says nothing is to be written; a write diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index e9fb985e6..2a04a3214 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -541,15 +541,36 @@ func TestTheSwitchCarriesTheGroupsInFlightIntoRecordsOnce(t *testing.T) { l := newLedgerFake(t) w.h.Records = &Records{Ledger: l, Now: w.h.Now} w.h.Merges = recordedMerges{MergeStore: w.merges, records: w.h.Records, members: w.h.groupMembers} + // W11.17d: what the old state owed is said as switched acts: every key of owed (G §169). + statusKey, _ := mdc.OwedKey("novox/app", 7, readyHead, statusAct("mesh/delivery")) + w.old.put(mdc.SwitchedOwed, statusKey, []byte(`{"act":"status","context":"mesh/delivery","state":"success","text":"ready","target":"none"}`), 4) + switched := func() []meshdelivery.Owed { + var out []meshdelivery.Owed + for _, o := range w.owed.said { + if _, _, _, ok := mdc.ParseSwitchedActID(o.ID); ok { + out = append(out, o) + } + } + return out + } opened, found, err := w.h.Switch() if err != nil || opened != 1 || found != 0 { t.Fatalf("the first switch: %d opened, %d found, %v", opened, found, err) } + carried := switched() + wantStatus, _ := mdc.SwitchedActID(mdc.SwitchedOwed, statusKey, 4) + if len(carried) != 1 || carried[0].ID != wantStatus || carried[0].Act != meshdelivery.OwedActStatus || carried[0].Context != "mesh/delivery" || + carried[0].Commit != readyHead { + t.Fatalf("the switch carried %+v", carried) + } asked := w.op.count() opened, found, err = w.h.Switch() if err != nil || opened != 0 || found != 1 || w.op.count() != asked { t.Fatalf("the switch again: %d opened, %d found, %v, %d asks (was %d)", opened, found, err, w.op.count(), asked) } + if again := switched(); len(again) != 2 || again[1].ID != wantStatus { + t.Fatalf("the switch again said %+v, wanted the same act under the same id", again) + } w.op.answer(w.op.open()[0], OptionApprove) w.settleAll() w.settleAll() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go index 02f9893bc..8b4c14264 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go @@ -8,6 +8,8 @@ package main // second record and asks nothing again. import ( + "encoding/json" + "errors" "fmt" "sort" @@ -123,9 +125,67 @@ func (h *Holder) Switch() (opened, found int, err error) { } } } + if err := h.carryOwed(); err != nil { + return opened, found, err + } return opened, found, nil } +// oldMerge is a group's merge as the old state merges holds it, with its key and revision; nil when it holds none. +func (h *Holder) oldMerge(group string) (*Merge, uint64, error) { + if h.Old == nil { + return nil, 0, nil + } + key, err := mdc.MergesKey(group) + if err != nil { + return nil, 0, err + } + raw, rev, err := h.Old.Get(mdc.SwitchedMerges, key) + if err != nil || raw == nil { + return nil, 0, err + } + m, err := DecodeMerge(raw) + if err != nil { + return nil, 0, err + } + return &m, rev, nil +} + +// carryOwed says every act the old state owed holds as owed, under its switched id (the state, the key and its revision: +// meshdeliveryclient.SwitchedActID), so the forge bridge performs what the old bridge did not and counts as performed what +// it did (design 47 G §169). No merge act is made from the old state merges (G §167, §542). Said at every switch, it is the same act each time. A key that cannot be read as an act is said +// as refused and skipped. +func (h *Holder) carryOwed() error { + if h.Old == nil || h.Owed == nil { + return nil + } + now := h.Now().UTC() + keys, err := h.Old.Keys(mdc.SwitchedOwed) + if err != nil { + return fmt.Errorf("the state owed cannot be read for the switch: %w", err) + } + for _, key := range keys { + raw, rev, err := h.Old.Get(mdc.SwitchedOwed, key) + if err != nil { + return fmt.Errorf("the owed act %s cannot be read for the switch: %w", key, err) + } + if raw == nil { + continue + } + repository, number, head, _, kerr := mdc.ParseOwedKey(key) + var o Owed + id, ierr := mdc.SwitchedActID(mdc.SwitchedOwed, key, rev) + if derr := json.Unmarshal(raw, &o); kerr != nil || derr != nil || ierr != nil { + h.Refuse(fmt.Errorf("the owed act %s is not carried by the switch: %v", key, errors.Join(kerr, derr, ierr))) + continue + } + if err := h.Owed.Say(seatOwedOf(o, id, repository, number, head, &now)); err != nil { + return fmt.Errorf("the owed act %s cannot be said for the switch: %w", key, err) + } + } + return nil +} + // groupMembers are the members' ids of the milestone a merge is handed under, under the lock. func (h *Holder) groupMembers(name string) []string { h.mu.Lock() -- 2.54.0 From 59f9a00c64dd2132f5e7af199f27a75bf0c626b6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:40:50 +0100 Subject: [PATCH 051/243] A merge the forge made with no record's merge-pull opens a forge-made delivery record, said once as its transition with why forge-made (milestone 47/G, W11.17e) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The record holds the pull request at its head and has no Approve; the delivery keeps that it was recorded (forge_recorded). Rows in issue 888's test. The test bridge never performs a switched act: the forge bridge decides. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 674f49ef.) Builds: design 47 G §170 (GATE6) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/issue888_test.go | 16 ++++++++ .../cmd/mesh-delivery-gate/merges.go | 38 +++++++++++++++++++ .../cmd/mesh-delivery-gate/model.go | 3 ++ .../cmd/mesh-delivery-gate/ready.go | 1 + 4 files changed, 58 insertions(+) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go index ab6b42d94..d28d6d1a1 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go @@ -31,6 +31,7 @@ func i888MergedAt(repo string, number int, head, merge string, at time.Time) Pul // 888/AC1: a merge into main the gate did not hand is heard from the forge: source.main holds it, and keeps the others. func TestAMergeHeardOnlyFromTheForgeMovesSourceMain(t *testing.T) { w := newWorld(t) + w.h.Records = &Records{Ledger: newLedgerFake(t), Now: w.h.Now} w.source.main = &Source{V: mdc.Version, Group: "earlier", Heads: map[string]string{i888Other: i888OtherH}} w.h.PullUpdated(pr(i888App, 3, i888Head1, "fix/x", "internal/a.go")) w.settleAll() @@ -52,6 +53,21 @@ func TestAMergeHeardOnlyFromTheForgeMovesSourceMain(t *testing.T) { if len(w.source.moves) != n || n != 1 { t.Fatalf("source.main moved %d then %d times; wanted once", n, len(w.source.moves)) } + // W11.17e: a merge with no record's merge-pull opens a forge-made record at its head, said once as a transition + // with why forge-made. + records, err := w.h.Records.Current() + if err != nil || len(records) != 1 || !records[0].ForgeMade || records[0].Heads[mdc.ApprovedKey(i888App, 3)] != i888Head1 { + t.Fatalf("the forge-made record is %+v (%v)", records, err) + } + var said []map[string]any + for _, e := range w.said { + if e["event-kind"] == "transition" && e["why"] == "forge-made" { + said = append(said, e) + } + } + if len(said) != 1 || said[0]["record"] != records[0].Instance { + t.Fatalf("the forge-made record is said %d times: %v", len(said), said) + } } // 888/AC2: a group the gate merged through the bridge moves source.main once, in finish, though the forge's word of each diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index 1404f6237..3f32bfb77 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -652,6 +652,44 @@ func (h *Holder) settleForgeMerges() []func() { return out } +// recordForgeMade opens a forge-made delivery record for every merge into the trunk the forge made with no record's +// merge-pull (design 47 *What the gate owes the forge*): one the gate did not say the merge of, heard as it happened. It +// holds the pull request at its head, has no Approve, and is said as the delivery's transition with why forge-made, once. +// Under the lock; the record is opened outside it. +func (h *Holder) recordForgeMade() []func() { + if h.Records == nil { + return nil + } + var out []func() + for _, d := range h.deliveries { + if d.MergedAs == "" || d.ForgeRecorded || d.LearntLate || gateMerges(d) || d.Number == 0 || d.Base != MergeBranch { + continue + } + id, milestone, heads := d.ID, milestoneOf(d), map[string]string{mdc.ApprovedKey(d.Repository, d.Number): d.Commit} + out = append(out, func() { + r, _, err := h.Records.Open(milestone, heads, true) + h.mu.Lock() + defer h.mu.Unlock() + m := h.deliveries[id] + if err != nil || m == nil || r == nil { + if err != nil { + h.Logf("[mesh-delivery] %s merged at the forge with no record's merge-pull, and its forge-made record cannot be opened yet: %v; asked again in %s", id, err, askAgainAfter) + h.again("a forge-made record, opened for a merge the forge made") + } + return + } + now := h.Now() + body := transitionEvent(m, Transition{At: now, From: m.State, To: m.State, Event: "forge-made", Why: "forge-made"}, time.Time{}) + body["record"] = r.Instance + m.Owed = append(m.Owed, Effect{Kind: EffectEmit, Event: "transition", Line: string(mustJSON(body)), Since: now}) + m.ForgeRecorded = true + h.keep(m) + h.Logf("[mesh-delivery] %s merged at the forge with no record's merge-pull: its forge-made record is %s", id, r.Instance) + }) + } + return out +} + // trunkHead is a full commit id as the forge names a branch's head. var trunkHead = regexp.MustCompile(`^[0-9a-f]{40}$`) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 37e27c0bd..8f69a2209 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -86,6 +86,9 @@ type Delivery struct { // InSource says the merge was looked at for source.main: written there, or left as older than the trunk's head (issue 888). // A merge the gate handed to the bridge on an Approve is moved there by finish instead (merges.go). InSource bool `json:"in_source,omitempty"` + // ForgeRecorded is a merge the forge made with no record's merge-pull whose forge-made record is opened and said + // (merges.go recordForgeMade). + ForgeRecorded bool `json:"forge_recorded,omitempty"` // LearntLate says its merge was learnt after the fact, from its absence among the forge's open pull requests // (settleGone, issue 1275): it moved no trunk here, and it never moves source.main, not even at a catch-up. LearntLate bool `json:"learnt_late,omitempty"` diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 442593248..37186e644 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -313,6 +313,7 @@ func (h *Holder) readyAsks(now time.Time) []func() { asks = append(asks, h.turnWrites()...) // the merge turns, after the holds of this look (ADR 0548 §3) asks = append(asks, h.settleMerges()...) asks = append(asks, h.settleForgeMerges()...) + asks = append(asks, h.recordForgeMade()...) asks = append(asks, h.seedTrunks()...) asks = append(asks, h.withdrawnKeys()...) asks = append(h.cancels, asks...) -- 2.54.0 From ee9c389888283f6051f6d32338e88c189f935ce6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:41:20 +0100 Subject: [PATCH 052/243] The gate takes owed.ended and take-in.taken from its manifest's consumes and names neither the forge bridge nor the indexer in code (milestone 47/G, the names move) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module's own manifest is embedded (package at the module's root); an event the gate hears from another module is the consumes entry with that event's name after its owner. No new seat, no second publisher on the seat git (the lead's ruling). Rows in the manifest test, which also reads the gate's sources for those names. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 2de87fd8.) Builds: design 47 G §7, §146 (GATE16); the bridge's name stays in code for the old merge follower (eb131beb left out, G §167, §542) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bus_test.go | 2 +- .../cmd/mesh-delivery-gate/consumer.go | 34 ++++++++++++++++--- .../cmd/mesh-delivery-gate/manifest_test.go | 21 ++++++++++++ .../cmd/mesh-delivery-gate/owedended.go | 4 +-- modules/mesh-delivery-gate/manifest.go | 23 +++++++++++++ 5 files changed, 77 insertions(+), 7 deletions(-) create mode 100644 modules/mesh-delivery-gate/manifest.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index a33421423..d52c1e910 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -100,7 +100,7 @@ func newTestBus(t *testing.T) *testBus { // The streams the events are on (mesh.mod..event., mesh.seat..event.). if _, err := js.CreateStream(context.Background(), jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.seat.git.event.>", "mesh.seat." + oc.Seat + ".event.>", - "mesh.mod." + forgeBridgeModule + ".event.>"}}); err != nil { + "mesh.mod.mesh-delivery-forge-bridge.event.>"}}); err != nil { t.Fatal(err) } return b diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go index 33f55c13d..ae74e5266 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/consumer.go @@ -12,6 +12,7 @@ package main import ( "encoding/json" "fmt" + ownmanifest "mesh-delivery-gate" "strings" "sync" "time" @@ -48,12 +49,37 @@ type stepFamily struct { // The step events on main the record is built from. A family is added here, with its `consumes` line, when its owner // publishes it (design 47 *What each step says*); each other step's event is its owner's to add in the same milestone. var stepFamilies = []stepFamily{ - // An entry taken in (the indexer's own `take-in.taken`, ADR 0633 §1): the merges a source revision took in. - {Pattern: indexerModule + "." + takein.TakeInTakenEvent, Owner: indexerModule, Event: takein.TakeInTakenEvent, Decode: decodeTakenIn}, + // An entry taken in (the holder of the seat mesh-catalog's own `take-in.taken`, ADR 0633 §1): the merges a source + // revision took in. Its owner is the one the manifest consumes it from. + family(takein.TakeInTakenEvent, decodeTakenIn), } -// indexerModule is the holder of the seat mesh-catalog whose own events the record hears. -const indexerModule = "mesh-delivery-indexer" +// family is a step family of an event as the manifest consumes it: `.`. +func family(event string, decode func(json.RawMessage, time.Time) ([]StepOutcome, error)) stepFamily { + pattern := declared(event) + owner, _, _ := strings.Cut(pattern, ".") + return stepFamily{Pattern: pattern, Owner: owner, Event: event, Decode: decode} +} + +// consumed is what the gate's manifest consumes (its own manifest): every event it follows is one of them. +var consumed = func() []string { + c, err := ownmanifest.Consumes() + if err != nil { + panic(err) + } + return c +}() + +// declared is the event the manifest consumes under this name after its owner, `.`: the gate names no +// other module in code. Empty, and so never followed or matched, when the manifest consumes none. +func declared(event string) string { + for _, c := range consumed { + if _, name, ok := strings.Cut(c, "."); ok && name == event { + return c + } + } + return "" +} // stepPatterns are the patterns of every family, in the order the manifest consumes them. func stepPatterns() []string { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index e61ec7458..16147a69d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -93,6 +93,27 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { t.Errorf("the manifest still consumes %s (ADR 0539 §17)", c) } } + // The names move (milestone 47/G): the families heard from another module are its manifest's consumes, and no Go + // source of the gate names those modules. + if !slices.Contains(m.Consumes, owedEndedEvent) || owedEndedEvent == "" || len(stepFamilies) != 1 || + !slices.Contains(m.Consumes, stepFamilies[0].Pattern) || stepFamilies[0].Owner+"."+stepFamilies[0].Event != stepFamilies[0].Pattern { + t.Fatalf("owed.ended is %q and the step families %+v, not the manifest's consumes %v", owedEndedEvent, stepFamilies, m.Consumes) + } + sources, _ := filepath.Glob("*.go") + for _, f := range sources { + if strings.HasSuffix(f, "_test.go") { + continue + } + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for _, name := range []string{`"mesh-delivery-indexer`} { + if strings.Contains(string(raw), name) { + t.Errorf("%s names %s in code: it is the manifest's to say", f, name) + } + } + } if !reflect.DeepEqual(m.Consumes, followed) || !slices.Contains(m.Consumes, "mesh-delivery-forge-bridge.forge.**") { t.Fatalf("consumes %v, follows %v (the bridge's forge messages among them)", m.Consumes, followed) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go index 46039cc45..5efc1c744 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owedended.go @@ -10,8 +10,8 @@ import ( ) // owedEndedEvent is the forge bridge's word on an act owed, owed.ended on its stream forge (design 47 *What the gate owes -// the forge*): the outcome of one id, the gate's own or the checker's. -const owedEndedEvent = forgeBridgeModule + "." + forgemsg.OwedEndedEvent +// the forge*): the outcome of one id, the gate's own or the checker's. Its owner is the one the manifest consumes it from. +var owedEndedEvent = declared(forgemsg.OwedEndedEvent) // OwedEnded takes the forge bridge's outcome of one act this owner said as owed. The outcome is written into the delivery // record that holds the pull request at that head, and only then is the act answered: taken off its delivery's or group's diff --git a/modules/mesh-delivery-gate/manifest.go b/modules/mesh-delivery-gate/manifest.go new file mode 100644 index 000000000..a6214ed30 --- /dev/null +++ b/modules/mesh-delivery-gate/manifest.go @@ -0,0 +1,23 @@ +// Package gate holds the delivery gate's own manifest, so its code takes from the manifest's declarations what it hears, +// never another module's name written in code. +package gate + +import ( + _ "embed" + "encoding/json" + "fmt" +) + +//go:embed module.json +var manifest []byte + +// Consumes is what the gate's manifest consumes, in its order. +func Consumes() ([]string, error) { + var m struct { + Consumes []string `json:"consumes"` + } + if err := json.Unmarshal(manifest, &m); err != nil { + return nil, fmt.Errorf("the gate's own manifest is not readable: %w", err) + } + return m.Consumes, nil +} -- 2.54.0 From 6ac5c934d2744fb87cb8ca52129dc566054ac641 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:41:20 +0100 Subject: [PATCH 053/243] The gate holds no merge for a check: the hold, the turns, their stalled line and tools, and check-gates-merge go; the merge judges the operator's Approve alone (milestone 47/G, W11.14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0661: the check gates the ask, in one place (askWaits); once the Approve covers the heads (approvedWaits) no verdict, carry, composed check or required status is judged, an Approve given before a trunk move merges onto the new trunk, and a failure after the Approve is said once on the pull request and as a stalled line naming the merge it follows. A check failing before the answer withdraws the ask. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 43696644.) Builds: design 47 G §278, test 52 (GATE10) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/adr505_test.go | 25 +- .../cmd/mesh-delivery-gate/adr548_test.go | 244 --------- .../cmd/mesh-delivery-gate/adr661_test.go | 136 ++++++ .../cmd/mesh-delivery-gate/approval.go | 6 +- .../cmd/mesh-delivery-gate/conditions.go | 3 - .../cmd/mesh-delivery-gate/events.go | 4 +- .../cmd/mesh-delivery-gate/events_test.go | 2 +- .../cmd/mesh-delivery-gate/group_merge.go | 20 +- .../cmd/mesh-delivery-gate/helpers_test.go | 6 - .../cmd/mesh-delivery-gate/holder.go | 9 +- .../cmd/mesh-delivery-gate/issue1275_test.go | 62 +-- .../issue1275_whole_test.go | 95 ++-- .../cmd/mesh-delivery-gate/issue1348_test.go | 13 +- .../cmd/mesh-delivery-gate/keeps.go | 2 +- .../cmd/mesh-delivery-gate/main.go | 19 - .../cmd/mesh-delivery-gate/manifest_test.go | 14 +- .../cmd/mesh-delivery-gate/model.go | 7 + .../cmd/mesh-delivery-gate/onestatus.go | 10 +- .../cmd/mesh-delivery-gate/ready.go | 121 ++--- .../cmd/mesh-delivery-gate/ready_test.go | 106 ---- .../cmd/mesh-delivery-gate/settings.go | 25 +- .../cmd/mesh-delivery-gate/table.go | 23 +- .../cmd/mesh-delivery-gate/table_test.go | 35 +- .../cmd/mesh-delivery-gate/turns.go | 461 ------------------ .../cmd/mesh-delivery-gate/values.go | 61 --- .../cmd/mesh-delivery-gate/values_test.go | 13 - .../cmd/mesh-delivery-gate/verbs.go | 7 +- .../mesh-delivery-gate/files/settings.json | 1 - modules/mesh-delivery-gate/module.json | 15 +- 29 files changed, 362 insertions(+), 1183 deletions(-) delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr548_test.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go index 520dbb8bd..3848448a7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go @@ -36,15 +36,6 @@ func TestTheBoundsAreSettingsWithTheirDefaults(t *testing.T) { if h.refusalFloor() != 10*time.Minute || h.authorBound() != 2*time.Hour { t.Fatalf("empty file: floor %s, author %s", h.refusalFloor(), h.authorBound()) } - if !h.checkGates() { - t.Fatalf("check-gates-merge unset reads false; the default is true") - } - if s, err = read(`{"check-gates-merge": false}`); err != nil || s.CheckGatesMerge == nil || *s.CheckGatesMerge || h.checkGates() { - t.Fatalf("check-gates-merge false: %+v %v, gates %v", s, err, h.checkGates()) - } - if _, err := read(`{"check-gates-merge": "no"}`); err == nil { - t.Fatalf("a check-gates-merge that is not true or false was taken") - } if _, err := read(`{"author-new-head-minutes": 0}`); err == nil { t.Fatalf("a bound of zero minutes was taken") } @@ -139,9 +130,9 @@ func TestAnyOtherRefusalIsStalledAtOnce(t *testing.T) { } } -// rejectedAfterItsApprove is a delivery approved and then rejected: its check failed after the ask (as in events_test.go), +// rejectedBeforeItsAnswer is a delivery approved and then rejected: its check failed after the ask (as in events_test.go), // its pull request's description naming who pushed it when pushedBy is given. -func rejectedAfterItsApprove(t *testing.T, pushedBy string) (*world, string) { +func rejectedBeforeItsAnswer(t *testing.T, pushedBy string) (*world, string) { t.Helper() w := newWorld(t) w.op.auto = false @@ -163,7 +154,11 @@ func rejectedAfterItsApprove(t *testing.T, pushedBy string) (*world, string) { d.Check = nil w.h.mu.Unlock() hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1")) - approve(w) + w.settleAll() + // A check that fails before the operator answers withdraws the ask (novox/hq ADR 0661 §5, ADR 0441 §1). + if open := w.op.open(); len(open) != 0 { + t.Fatalf("the ask stands after its check failed: %v", open) + } w.h.React() if w.state(id) != Rejected { t.Fatalf("not rejected: %s", w.state(id)) @@ -175,7 +170,7 @@ func rejectedAfterItsApprove(t *testing.T, pushedBy string) (*world, string) { // with no new head, in words naming its author (its branch, and the session that pushed it) and needing a new head — // never "stop it", never "the operator's". func TestARejectedDeliveryIsItsAuthorsForTwoHours(t *testing.T) { - w, id := rejectedAfterItsApprove(t, "quiet-deliveries@shanks") + w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries@shanks") said := false for _, c := range w.forge.comments { said = said || strings.Contains(c, "not merged") @@ -208,7 +203,7 @@ func TestARejectedDeliveryIsItsAuthorsForTwoHours(t *testing.T) { // ADR 0505 §5: with no Pushed-by line, the branch alone names its author. func TestARejectedDeliveryWithNoPushedByLineIsNamedByItsBranch(t *testing.T) { - w, id := rejectedAfterItsApprove(t, "") + w, id := rejectedBeforeItsAnswer(t, "") w.later(2*time.Hour + time.Minute) if got := linesFor(w, id); len(got) != 1 || got[0].Author != "the author of feat/x" { t.Fatalf("not named by its branch: %+v", got) @@ -217,7 +212,7 @@ func TestARejectedDeliveryWithNoPushedByLineIsNamedByItsBranch(t *testing.T) { // ADR 0505 §5: a new head within the two hours ends it: the rejected head is never the operator's. func TestANewHeadWithinTwoHoursKeepsTheRejectedHeadFromTheOperator(t *testing.T) { - w, id := rejectedAfterItsApprove(t, "quiet-deliveries@shanks") + w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries@shanks") w.later(time.Hour) w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go")) w.settleAll() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr548_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr548_test.go deleted file mode 100644 index fdcaacc0a..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr548_test.go +++ /dev/null @@ -1,244 +0,0 @@ -package main - -import ( - "fmt" - "strings" - "sync" - "testing" - "time" -) - -// One check per delivery (novox/hq ADR 0548), the gate's part: nothing merges into a repository while a check of other heads -// judges it (§2); an approved merge holds a turn on its repositories (§3); a held merge past its bound is a stalled line, read -// and handed by two tools (§4); an approved delivery whose trunk moved merges on its carried verdict with no second ask (§8). - -const otherHead = "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3" - -// approvedLone is a lone delivery into oneRepo whose check passed, approved by the operator (the world approves at once). -func approvedLone(w *world) string { - w.t.Helper() - p := pr(oneRepo, 1, oneHead, "feat/one") - requiresCheck(w, p) - w.h.PullUpdated(p) - putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead)) - w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk)) - return IDOf(oneRepo, oneHead) -} - -// memTurns is the bucket turns in memory; it logs each write with the merges the fake bridge had made by then. -type memTurns struct { - mu sync.Mutex - m map[string]Turn - log []string - merges func() int -} - -func newMemTurns(w *world) *memTurns { - return &memTurns{m: map[string]Turn{}, merges: func() int { return len(w.forge.merges) }} -} - -func (f *memTurns) All() (map[string]Turn, error) { - f.mu.Lock() - defer f.mu.Unlock() - out := map[string]Turn{} - for k, v := range f.m { - out[k] = v - } - return out, nil -} - -func (f *memTurns) Put(t Turn) error { - if _, err := EncodeTurn(t); err != nil { - return err - } - f.mu.Lock() - defer f.mu.Unlock() - f.m[t.Repository] = t - f.log = append(f.log, fmt.Sprintf("put %s after %d merges", t.Repository, f.merges())) - return nil -} - -func (f *memTurns) Delete(repository string) error { - f.mu.Lock() - defer f.mu.Unlock() - delete(f.m, repository) - f.log = append(f.log, fmt.Sprintf("delete %s after %d merges", repository, f.merges())) - return nil -} - -func (f *memTurns) said() string { f.mu.Lock(); defer f.mu.Unlock(); return strings.Join(f.log, "; ") } - -func TestOneCheckPerDeliveryAtTheGate(t *testing.T) { - for name, run := range map[string]func(t *testing.T){ - // §2: a merge into a repository a running verdict names is not written to merges, and is written on that verdict's pass. - "an approved merge waits for a running check on its repository and merges on its end": func(t *testing.T) { - w := newWorld(t) - putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead)) - id := approvedLone(w) - w.settleAll() - if w.op.count() != 1 || len(w.forge.merges) != 0 { - t.Fatalf("asked %d times, merges %v: want the Approve asked and the merge held", w.op.count(), w.forge.merges) - } - d := w.delivery(id) - if d.Ready == nil || !strings.Contains(d.Ready.Waits, "waits for the check of g") || !strings.Contains(d.Ready.Waits, oneRepo) || - d.Ready.HeldSince == nil { - t.Fatalf("the merge does not say what it waits for: %+v", d.Ready) - } - // §9: deliveries says what it waits for. - for _, l := range w.h.Deliveries("", "", "", false) { - if l.ID == id && !strings.Contains(l.Waits, "approved; waits for the check of g") { - t.Errorf("deliveries says it waits for %q", l.Waits) - } - } - // §4: past 30 minutes the held merge is a stalled line naming the check it waits for and since when. - for _, l := range w.h.Stalled() { - if l.ID == id { - t.Fatalf("a stalled line within the bound: %+v", l) - } - } - w.later(turnBound + time.Minute) - w.settleAll() - var line *StalledLine - for _, l := range w.h.Stalled() { - if l.ID == id { - line = &l - } - } - if line == nil || line.Bound != "30m0s" || !strings.Contains(line.Says, "waits for the check of g") || !strings.Contains(line.Says, "held since") || - !strings.Contains(line.Says, "checker_status") || !strings.Contains(line.H2, "delivery_hand_now") { - t.Fatalf("the stalled line of the held merge = %+v", line) - } - putCheckerVerdict(w, "pass", headOf(oneRepo, 9, otherHead)) - w.settleAll() - if len(w.forge.merges) != 1 || w.op.count() != 1 { - t.Fatalf("after the check's end: merges %v, asked %d times; want merged with no second ask", w.forge.merges, w.op.count()) - } - }, - "a running check on another repository holds nothing": func(t *testing.T) { - w := newWorld(t) - putCheckerVerdict(w, "running", headOf(oneRepo2, 9, otherHead)) - approvedLone(w) - w.settleAll() - if len(w.forge.merges) != 1 { - t.Fatalf("merges %v: a check of another repository held the merge (%s)", w.forge.merges, w.logs()) - } - }, - "the delivery's own running verdict does not hold it": func(t *testing.T) { - w := newWorld(t) - w.op.auto = false - id := approvedLone(w) - w.settleAll() - putCheckerVerdict(w, "running", headOf(oneRepo, 1, oneHead)) // its own heads judged again (a carry) - for _, k := range w.op.open() { - w.op.answer(k, OptionApprove) - } - w.settleAll() - if d := w.delivery(id); len(w.forge.merges) != 1 || d.Ready == nil || d.Ready.HeldSince != nil { - t.Fatalf("merges %v, run %+v: its own check held it", w.forge.merges, d.Ready) - } - }, - // §3: the turn is written before the verdicts are read for the merge, and deleted after the last merge. - "a lone merge's turn is written before it is handed and deleted once it merged": func(t *testing.T) { - w := newWorld(t) - turns := newMemTurns(w) - w.h.Turns = turns - approvedLone(w) - w.settleAll() - if len(w.forge.merges) != 1 || turns.said() != "put "+oneRepo+" after 0 merges; delete "+oneRepo+" after 1 merges" { - t.Fatalf("merges %v; the turns were %q", w.forge.merges, turns.said()) - } - }, - "a group's turns stand on every member's repository until its last member merged": func(t *testing.T) { - w := newWorld(t) - turns := newMemTurns(w) - w.h.Turns = turns - a, b := pr(oneRepo, 1, oneHead, "feat/two"), pr(oneRepo2, 2, oneHead2, "feat/two") - requiresCheck(w, a, b) - w.h.PullUpdated(a) - w.h.PullUpdated(b) - w.h.wantForge() - putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead), headOf(oneRepo2, 2, oneHead2)) - w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk)) - w.h.CommitStatus(checkStatus(oneRepo2, oneHead2, "success", oneTrunk)) - for range 3 { - w.settleAll() - } - want := "put " + oneRepo2 + " after 0 merges; put " + oneRepo + " after 0 merges; delete " + oneRepo2 + " after 2 merges; delete " + oneRepo + " after 2 merges" - if len(w.forge.merges) != 2 || turns.said() != want { - t.Fatalf("merges %v; the turns were %q, want %q", w.forge.merges, turns.said(), want) - } - }, - // §8: an approved delivery whose trunk moved goes back to proposed, waits for its carried verdict rather than being - // handed on the old one, and merges on it with no second ask. - "an approved merge whose trunk moved merges on its carried verdict with no second ask": func(t *testing.T) { - w := newWorld(t) - putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead)) // holds the approved merge (§2) - id := approvedLone(w) - w.settleAll() - if w.op.count() != 1 || len(w.forge.merges) != 0 { - t.Fatalf("asked %d times, merges %v; want approved and held", w.op.count(), w.forge.merges) - } - w.h.PullMerged(merged(oneRepo, 5, otherHead, oneTrunk2)) // a merge at the forge moves the trunk - putCheckerVerdict(w, "pass", headOf(oneRepo, 9, otherHead)) - w.settleAll() - if len(w.forge.merges) != 0 { - t.Fatalf("merges %v: handed on a verdict judged on the old trunk", w.forge.merges) - } - d := w.delivery(id) - if d.Ready == nil || !strings.Contains(d.Ready.Waits, "being carried") { - t.Fatalf("after the trunk moved it waits for %q, want its verdict carried", d.Ready.Waits) - } - // The checker's carried line: the head on the new trunk, carried from the old one. - carried := checkStatus(oneRepo, oneHead, "success", oneTrunk2) - carried.Description = "pass " + oneHead[:12] + " on " + oneTrunk2[:12] + " | carried from " + oneTrunk[:12] + " | unit 0 run, 4 kept" - if head, trunk := checkLine(carried.Description); head != oneHead[:12] || trunk != oneTrunk2[:12] { - t.Fatalf("the carried line reads as %s on %s", head, trunk) - } - if head, trunk := checkLine("failed: a test"); head != "" || trunk != "" { - t.Fatalf("a line that names no commits gave %q %q", head, trunk) - } - putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead)) - w.h.CommitStatus(carried) - w.settleAll() - if w.op.count() != 1 { - t.Errorf("the Approve was asked %d times, want once: it stands through the carry", w.op.count()) - } - if d := w.delivery(id); d.Check == nil || d.Check.Trunk != oneTrunk2[:12] || len(w.forge.merges) != 1 { - t.Errorf("after the carry: verdict %+v, merges %v; want merged on the carried verdict", d.Check, w.forge.merges) - } - }, - // §4: delivery_turns reads every turn with the check it waits for; delivery_hand_now hands a held merge at once on the - // operator's word, and refuses one no check holds. - "the turns are read, and a held merge is handed now on the operator's word": func(t *testing.T) { - w := newWorld(t) - w.h.Turns = newMemTurns(w) - putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead)) - id := approvedLone(w) - w.settleAll() - read := w.h.TurnsRead() - lines := read["turns"].([]TurnLine) - if len(lines) != 1 || lines[0].Repository != oneRepo || len(lines[0].Turn.Waiting) != 1 || lines[0].Turn.Waiting[0].WaitsFor == "" || - !strings.Contains(lines[0].Waits["feat/one"].(string), "the check g.") || read["written"] != true { - t.Fatalf("delivery_turns = %+v", read) - } - if _, err := w.h.HandNow(id, ""); err == nil || !strings.Contains(err.Error(), "why") { - t.Errorf("a hand without the operator's why: %v", err) - } - if _, err := w.h.HandNow(IDOf(oneRepo2, oneHead2), "x"); err == nil { - t.Error("an unknown delivery was handed") - } - if _, err := w.h.HandNow(id, "the operator, on telegram: merge it now"); err != nil { - t.Fatal(err) - } - w.settleAll() - if len(w.forge.merges) != 1 || w.delivery(id).Ready.HandNow == "" { - t.Fatalf("merges %v after the hand", w.forge.merges) - } - if _, err := w.h.HandNow(id, "again"); err == nil { - t.Error("a merged delivery was handed again") - } - }, - } { - t.Run(name, run) - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go new file mode 100644 index 000000000..164d3a1f7 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// No check gates a merge (novox/hq ADR 0661), the gate's part: no approved merge waits for another delivery's running check, +// and no turn or stalled line holds it (§1); the merge judges the operator's Approve alone, so an Approve given before a trunk +// move merges onto the new trunk with no wait for its carry (§3); a failure after the Approve is said once on the pull +// request and as a stalled line naming the merge it follows, and refuses nothing (§5). + +const otherHead = "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3" + +// approvedLone is a lone delivery into oneRepo whose check passed, approved by the operator (the world approves at once). +func approvedLone(w *world) string { + w.t.Helper() + p := pr(oneRepo, 1, oneHead, "feat/one") + requiresCheck(w, p) + w.h.PullUpdated(p) + putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead)) + w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk)) + return IDOf(oneRepo, oneHead) +} + +// sayingFailed is the forge's word that oneHead's check failed on the trunk commit given (a carry's line). +func sayingFailed(trunk string) ForgeStatusEvent { + e := checkStatus(oneRepo, oneHead, "failure", trunk) + e.Description = "failed " + oneHead[:12] + " on " + trunk[:12] + " | a test failed" + return e +} + +func lineOf(w *world, id string) *StalledLine { + for _, l := range w.h.Stalled() { + if l.ID == id { + return &l + } + } + return nil +} + +func TestNoCheckGatesAMergeAtTheGate(t *testing.T) { + for name, run := range map[string]func(t *testing.T){ + // §1: a running check of other heads on the same repository holds nothing, and no stalled line is raised for it. + "an approved merge into a repository under a running check merges at once": func(t *testing.T) { + w := newWorld(t) + putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead)) + id := approvedLone(w) + w.settleAll() + if w.op.count() != 1 || len(w.forge.merges) != 1 { + t.Fatalf("asked %d times, merges %v: want asked once and merged at once (%s)", w.op.count(), w.forge.merges, w.logs()) + } + w.h.PullMerged(merged(oneRepo, 1, oneHead, strings.Repeat("c", 40))) // the seat git says the merge too + w.later(time.Hour) + w.settleAll() + if l := lineOf(w, id); l != nil { + t.Fatalf("a stalled line for a merge a running check once held: %+v", l) + } + }, + "the delivery's own running verdict does not hold it": func(t *testing.T) { + w := newWorld(t) + w.op.auto = false + id := approvedLone(w) + w.settleAll() + putCheckerVerdict(w, "running", headOf(oneRepo, 1, oneHead)) // its own heads judged again (a carry) + for _, k := range w.op.open() { + w.op.answer(k, OptionApprove) + } + w.settleAll() + if d := w.delivery(id); len(w.forge.merges) != 1 || d.Ready == nil { + t.Fatalf("merges %v, run %+v: its own check held it", w.forge.merges, d.Ready) + } + }, + // §3 (and ADR 0548 §8): the ask stands through a trunk move; the Approve given then merges onto the new trunk on the + // verdict judged against the old one, with no wait for its carry and no second ask. + "an Approve given after a trunk move merges at once, its carry after": func(t *testing.T) { + w := newWorld(t) + w.op.auto = false + id := approvedLone(w) + w.settleAll() + if w.op.count() != 1 || len(w.forge.merges) != 0 { + t.Fatalf("asked %d times, merges %v: want asked and not merged yet", w.op.count(), w.forge.merges) + } + w.h.PullMerged(merged(oneRepo, 5, otherHead, oneTrunk2)) // a merge at the forge moves the trunk + w.settleAll() + for _, k := range w.op.open() { + w.op.answer(k, OptionApprove) + } + w.settleAll() + d := w.delivery(id) + if len(w.forge.merges) != 1 || w.op.count() != 1 || d.State != Ready && d.State != Merged { + t.Fatalf("merges %v, asked %d times, %s (waits %q): want merged at once on the old verdict (%s)", w.forge.merges, + w.op.count(), d.State, d.Ready.Waits, w.logs()) + } + }, + // §5: a failure after the Approve, a carry's, is said once on the pull request and as one stalled line naming the merge; + // it refuses nothing, and a later pass clears the line. + "a failure after the Approve is said once and refuses nothing": func(t *testing.T) { + w := newWorld(t) + id := approvedLone(w) + w.settleAll() + if len(w.forge.merges) != 1 { + t.Fatalf("merges %v: want merged at once", w.forge.merges) + } + before := len(w.forge.comments) + w.h.CommitStatus(sayingFailed(oneTrunk2)) + w.settleAll() + w.h.CommitStatus(sayingFailed(oneTrunk2)) + w.settleAll() + said := 0 + for _, c := range w.forge.comments[before:] { + if strings.Contains(c, "failed after the Approve") { + said++ + } + } + if said != 1 { + t.Fatalf("said %d times on the pull request, want once: %v", said, w.forge.comments[before:]) + } + l := lineOf(w, id) + if l == nil || !strings.Contains(l.Says, "after the operator's Approve") || !strings.Contains(l.Says, oneHead[:12]) { + t.Fatalf("the stalled line = %+v", l) + } + if d := w.delivery(id); d.State == Rejected || len(w.forge.merges) != 1 { + t.Fatalf("%s, merges %v: the failure refused the approved merge", d.State, w.forge.merges) + } + w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk2)) + w.settleAll() + if l := lineOf(w, id); l != nil { + t.Fatalf("a later pass left the stalled line: %+v", l) + } + }, + } { + t.Run(name, run) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 00f2b66e8..f23058fa5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -338,9 +338,9 @@ func (h *Holder) endStaleAsks() { h.cancels = append(h.cancels, h.dropAsk(r.AskID, "its head "+shortOf(d.Commit, 12)+" is no longer the pull request's head, or the pull request closed")) case !h.headsHold(d, r): h.cancels = append(h.cancels, h.dropAsk(r.AskID, "a head it covers moved on: the heads it covers are no longer the group's heads")) - case r.Approved == nil && r.AskID != "" && r.AskCheck != "" && !h.checkGates() && h.checkMoved(d, r.AskCheck): - // The check gates no merge, so the ask was made before its verdict: once the check passes or fails (or a pass it - // named is gone), the ask is asked again in words that say so (issue 1733). Running and not started say the same. + case r.Approved == nil && r.AskID != "" && r.AskCheck != "" && h.checkMoved(d, r.AskCheck): + // The check gates the ask, so it was made on a pass: once that pass is gone (a failure, or a check of these heads + // running again), the ask not yet answered is withdrawn in words that say so (novox/hq ADR 0661 §5, ADR 0441 §1). h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, "the check at its heads is now: "+h.checkAt(h.askMembers(d)).Words)) case r.Approved == nil && r.AskID != "" && !r.AskedAt.IsZero() && !h.Now().Before(r.AskedAt.Add(AskLasts)): // The ask lapsed at the router with nobody's answer, and nothing in the gate's inputs says so: it is taken away and diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go index 05cceed05..70da4ca6b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/conditions.go @@ -225,9 +225,6 @@ func (h *Holder) nextStall(now time.Time) time.Time { consider(r.RefusedSince.Add(h.answeredRefusalBound(d.Repository))) consider(r.RefusedSince.Add(outOfDateFor)) } - if r.HeldSince != nil { - consider(r.HeldSince.Add(turnBound)) - } } return next } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go index 5b3e5b39e..1cd053de7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go @@ -67,9 +67,7 @@ func (h *Holder) CommitStatus(e ForgeStatusEvent) { h.takeCheckStatus(e.Owner+"/"+e.Repo, e.SHA, e.State, e.Description) } if e.State != "success" { - if h.checkGates() { - h.withdrawForStatus(e) // while the check gates no merge, its status is shown and takes back no ask - } + h.withdrawForStatus(e) // a status not a success takes back an ask not yet answered (novox/hq ADR 0661 §5) return } repository := strings.ToLower(e.Owner + "/" + e.Repo) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go index 63609fe7d..c71ad614a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go @@ -196,7 +196,7 @@ func TestNoWaitReadsADescription(t *testing.T) { w.h.mu.Lock() got := []string{state, why} for _, d := range ds { - got = append(got, w.h.readyWaits(d), w.h.ungatedWaits(d)) + got = append(got, w.h.askWaits(d), w.h.approvedWaits(d)) } w.h.mu.Unlock() if i == 0 { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go index 624a2271a..3452f4c5e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go @@ -43,14 +43,24 @@ const ( // conflict); empty when all hold. forge says the reason is the forge's merge answer rather than a status. An error is a // read that could not be made: the member then counts as not mergeable. func (h *Holder) notASuccess(m memberRead) (says string, forge bool, err error) { + return h.memberRead(m, true) +} + +// notMergeableNow is notASuccess at the merge: the head the Approve covers and the forge's merge answer, and no required +// status read again, for the merge judges the operator's Approve alone (novox/hq ADR 0661 §4). +func (h *Holder) notMergeableNow(m memberRead) (says string, forge bool, err error) { + return h.memberRead(m, false) +} + +func (h *Holder) memberRead(m memberRead, statuses bool) (says string, forge bool, err error) { fc, err := h.Forge.Statuses(m.owner, m.repo, m.number, m.head, m.base) if err != nil { return "", false, err } - if fc.Merge.Mergeable == nil { + if statuses && fc.Merge.Mergeable == nil { return "", false, fmt.Errorf("the forge could not say whether %s may merge: %s", m.id, fc.Merge.Says) } - if !*fc.Merge.Mergeable { + if statuses && !*fc.Merge.Mergeable { var parts []string for _, b := range fc.Merge.Blocking { parts = append(parts, fmt.Sprintf("%s (%s)", b.Context, b.State)) @@ -82,8 +92,8 @@ func (h *Holder) unmergedLive(g *Group) []*Delivery { return out } -// groupMergeable reads every member's required statuses, its head and the forge's merge answer at the head the Approve -// covers, and says whether all hold: no member is merged unless every member merges now (issue 1275, 2026-10-26: two of +// groupMergeable reads every member's head and the forge's merge answer at the head the Approve covers, no required status +// (novox/hq ADR 0661 §4), and says whether all hold: no member is merged unless every member merges now (issue 1275, 2026-10-26: two of // three merged and the third refused). When one does not, or cannot be read, nothing is handed to the bridge: the group is // held, said urgent (ADR 0441 §2, §4), and the target's turn is given back so the next event that decides whether it may // merge asks it again. A read that could not be made is a failed ask, tried again after askAgainAfter; the forge saying a @@ -97,7 +107,7 @@ func (h *Holder) groupMergeable(gid, target string, reads []memberRead) bool { } } for _, m := range reads { - says, forge, err := h.notASuccess(m) + says, forge, err := h.notMergeableNow(m) if err == nil && says == "" { continue } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/helpers_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/helpers_test.go index e9a896a8d..d6071043b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/helpers_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/helpers_test.go @@ -21,12 +21,6 @@ func verdictMoving(repo string, number int, sha string, _ ...string) CheckedEven return verdict(repo, number, sha, "pass") } -// ungated is the setting check-gates-merge false: mesh/check gates no merge (the operator, 2026-10-25 22:56). -func ungated(w *world) { - off := false - w.h.Settings = func() Settings { return Settings{CheckGatesMerge: &off} } -} - func statusOn(w *world, repo, commit string) string { w.forge.mu.Lock() defer w.forge.mu.Unlock() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index 9ba599e69..a427ebdad 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -85,7 +85,7 @@ type Holder struct { // unchecked are the open pull requests' heads seen without a merge check (unchecked.go). unchecked map[string]unchecked // checker are the checker's verdicts by the hash of the heads they judged, as its verdicts bucket holds them - // (onestatus.go, issue 1302); runningSince when this owner first saw each running (ADR 0548 §2, turns.go). + // (onestatus.go, issue 1302); runningSince when this owner first saw each running, said in the check's words. checker map[string]checkerVerdict runningSince map[string]time.Time // unmergeableSince is when the forge first said a group's member does not merge now, by group and member (issue 1275, @@ -93,11 +93,6 @@ type Holder struct { unmergeableSince map[string]time.Time // writeReads is whether the forge's merging account writes each repository, as last read (writes.go, issue 1945). writeReads map[string]*writeRead - // Turns is the gate's bucket turns (turns.go, ADR 0548 §3); nil keeps none. turnsHeld are the turns as written there, - // by repository, read once at the first look (turnsLoaded). - Turns TurnStore - turnsHeld map[string]Turn - turnsLoaded bool // reach is the delivery view's reach of each repository, by its key (reached.go, novox/hq ADR 0556): what `deliveries` // and `show` derive whether a merge is on the machines from. Nothing here records it. reach map[string]viewstate.Reach @@ -152,7 +147,7 @@ func (h *Holder) init() { h.lostRead = map[string]time.Time{} h.unchecked = map[string]unchecked{} h.checker = map[string]checkerVerdict{} - h.runningSince, h.turnsHeld = map[string]time.Time{}, map[string]Turn{} + h.runningSince = map[string]time.Time{} h.reach = map[string]viewstate.Reach{} h.started = h.Now() } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go index d7bdda30b..ea092f7fa 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go @@ -56,29 +56,16 @@ func urgentLines(w *world) []StalledLine { return out } -// 1275/AC2: an approved group whose member has a required status that is not a success at its head when the merge starts -// is merged not at all, though the first member in order is fine; the group says which member and which status. -func TestAnApprovedGroupWithAMemberWhoseRequiredStatusIsNotASuccessMergesNothing(t *testing.T) { +// novox/hq ADR 0661 §4 (changing 1275/AC2 and ADR 0441 §2): an approved group whose member's required status turned +// failed after the Approve merges whole, with no read of its statuses at the merge, and raises no line. +func TestAnApprovedGroupWhoseMemberStatusTurnedAfterTheApproveMergesWhole(t *testing.T) { w := approvedGroup(t) allGreen(w) w.op.answer(w.op.open()[0], OptionApprove) setChecks(w, gCtl, gHc, "success", "") // the second member's repository check went missing after the ask w.settleAll() - if len(w.forge.merges) != 0 { - t.Fatalf("merged with a required status missing: %v", w.forge.merges) - } - lines := urgentLines(w) - if len(lines) != 1 || !strings.Contains(lines[0].Says, "feat/g") || !strings.Contains(lines[0].Says, IDOf(gCtl, gHc)) || - !strings.Contains(lines[0].Says, "mesh/repo-check (missing)") || !strings.Contains(lines[0].Says, "Needed") { - t.Fatalf("the held group is not said as one urgent line naming group, member, status and need: %+v", lines) - } - // The status turns success: the whole group merges. - setChecks(w, gCtl, gHc, "success", "success") - w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "mesh-controller", SHA: gHc, Context: repoCheck, State: "success"}) - w.later(time30s) - w.settleAll() if len(w.forge.merges) != 2 || len(urgentLines(w)) != 0 { - t.Fatalf("merges %v, lines %+v; wanted both merged and no line", w.forge.merges, urgentLines(w)) + t.Fatalf("merges %v, lines %+v; wanted both merged on the Approve and no line", w.forge.merges, urgentLines(w)) } } @@ -142,33 +129,6 @@ func TestAStatusThatTurnsWhileTheAskWaitsWithdrawsItAndItsAnswerMergesNothing(t } } -// 1275/AC3 (review round 1): a member that is both refused and held for a required status is said by one stalled line, -// not two with the same id. -func TestAMemberBothRefusedAndHeldForAStatusIsOneStalledLine(t *testing.T) { - w := approvedGroup(t) - allGreen(w) - w.forge.refuseRepo = map[string]error{gCat: refusedWords("novox/mesh-catalog#2")} - w.op.answer(w.op.open()[0], OptionApprove) - w.settleAll() - setChecks(w, gCat, gHk, "success", "") // now its status is missing too; the next pass is held - w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "mesh-catalog", SHA: gHk, Context: repoCheck, State: "success"}) - w.h.mergeAgainNow(w.delivery(IDOf(gCat, gHk))) - w.later(time30s) - w.settleAll() - if g := w.h.groups["feat/g"]; g.Hold == nil { - t.Fatalf("the group was not held for the missing status") - } - n := 0 - for _, l := range w.h.Stalled() { - if l.ID == IDOf(gCat, gHk) { - n++ - } - } - if n != 1 { - t.Fatalf("%d stalled lines for one member: %+v", n, w.h.Stalled()) - } -} - // 1275/AC6, as novox/hq ADR 0505 §4 changed it: a member refused because its verdict is out of date (the trunk moved; its // fresh check runs) is no stalled line while the gate answers it, and urgent once ten minutes passed with the group not // merged; the pass stops at the refusal all the same. @@ -505,7 +465,7 @@ func TestAnApprovedGroupIsHandedWholeInItsOrderAndAStopIsSaid(t *testing.T) { w.settleAll() w.h.PullMerged(merged(gCat, 2, gHk, gT1)) bridgeMerged(w, gCat, 2, gHk, gT1) - setChecks(w, gCtl, gHc, "success", "") // the second member's status went missing: the pass stops at it + moveForgeHead(w, gCtl, 1, otherHead) // the second member's head moved at the forge: the pass stops at it w.settleAll() for range 3 { w.later(time30s) @@ -522,7 +482,7 @@ func TestAnApprovedGroupIsHandedWholeInItsOrderAndAStopIsSaid(t *testing.T) { !strings.Contains(c.Summary, "half merged (merged 1 of 2)") { t.Fatalf("raised %d times, last %+v; wanted once, urgent, naming the group half merged", raised, c) } - setChecks(w, gCtl, gHc, "success", "success") + moveForgeHead(w, gCtl, 1, gHc) w.h.CommitStatus(ForgeStatusEvent{Owner: "novox", Repo: "mesh-controller", SHA: gHc, Context: repoCheck, State: "success"}) w.settleAll() // the status is an event: the pass goes on with no timer handed := false @@ -770,3 +730,13 @@ func bridgeMerged(w *world, repository string, number int, head, merge string) { w.merges.say(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: repository, Number: number, Head: head, Merge: merge, Root: "root", Trees: map[string]string{"app": "tree"}}) } + +// moveForgeHead sets the head the forge says a pull request has, as a push the gate has not heard yet does. +func moveForgeHead(w *world, repo string, number int, head string) { + w.forge.mu.Lock() + defer w.forge.mu.Unlock() + key := fmt.Sprintf("%s#%d", repo, number) + p := w.forge.pulls[key] + p.HeadSHA = head + w.forge.pulls[key] = p +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_whole_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_whole_test.go index 0000692ac..eedf02500 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_whole_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_whole_test.go @@ -135,62 +135,47 @@ func handedWords(ps []MergePull) string { // Two approved merges into one repository: the one whose pass started has the repository until every member merged, and // the other is handed only then (2026-10-26: mesh-delivery#72 of feat/890-assignments merged between fix/734-lost-ask's -// members). This holds while mesh/check gates no merge too (ADR 0551): it waits for a merge, never for a check. +// members). It waits for a merge, never for a check: once the group's pass ended #72 is handed on its Approve, its verdict +// gone out of date with #73's merge or not (novox/hq ADR 0661 §3). func TestTwoApprovedMergesIntoOneRepositoryDoNotInterleave(t *testing.T) { - for _, gates := range []bool{true, false} { - t.Run(fmt.Sprintf("check-gates-merge %v", gates), func(t *testing.T) { - w := approvedGroupOf(t, lostAsk...) - if !gates { - ungated(w) - } - w.merges.manual = true - allGreenFor(w, lostAsk...) - w.op.answer(w.op.open()[0], OptionApprove) - w.settleAll() - if got := handedWords(handing(w)); got != "novox/mesh-sdk#92" { - t.Fatalf("handed %q, wanted the group's first member", got) - } - // feat/890-assignments: a lone pull request into mesh-delivery, approved while the group's pass is under way. - w.h.PullUpdated(pr(wDel, 72, wH72, "feat/890-assignments", "modules/mesh-delivery-checker/go.mod")) - w.h.Checked(judged(wDel, 72, wH72, "pass", gTrunk0)) - setChecks(w, wDel, wH72, "success", "success") - w.settleAll() - if open := w.op.open(); len(open) != 1 { - t.Fatalf("the lone pull request is asked once: %v", open) - } else { - w.op.answer(open[0], OptionApprove) - } - w.settleAll() - var order []string - steps := []struct { - repo string - number int - head string - }{{wSDK, 92, wH92}, {wCtl, 472, wH472}, {wDel, 73, wH73}} - if !gates { - // While the check gates the merge, #72's verdict went out of date with #73's merge and it waits for its - // check against the new trunk; while it gates none, #72 is handed once the group's pass ended. - steps = append(steps, struct { - repo string - number int - head string - }{wDel, 72, wH72}) - } - for _, next := range steps { - handed := handing(w) - if len(handed) != 1 || handed[0].Repository != next.repo || handed[0].Number != next.number { - t.Fatalf("after %v: handed %q, wanted only %s#%d", order, handedWords(handed), next.repo, next.number) - } - merge := strings.Repeat(string(rune('a'+next.number%20)), 40) - bridgeMerged(w, next.repo, next.number, next.head, merge) - w.h.PullMerged(merged(next.repo, next.number, next.head, merge)) - w.settleAll() - order = append(order, fmt.Sprintf("%s#%d", next.repo, next.number)) - } - if left := handing(w); len(left) != 0 { - t.Fatalf("still handed: %q", handedWords(left)) - } - }) + w := approvedGroupOf(t, lostAsk...) + w.merges.manual = true + allGreenFor(w, lostAsk...) + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + if got := handedWords(handing(w)); got != "novox/mesh-sdk#92" { + t.Fatalf("handed %q, wanted the group's first member", got) + } + // feat/890-assignments: a lone pull request into mesh-delivery, approved while the group's pass is under way. + w.h.PullUpdated(pr(wDel, 72, wH72, "feat/890-assignments", "modules/mesh-delivery-checker/go.mod")) + w.h.Checked(judged(wDel, 72, wH72, "pass", gTrunk0)) + setChecks(w, wDel, wH72, "success", "success") + w.settleAll() + if open := w.op.open(); len(open) != 1 { + t.Fatalf("the lone pull request is asked once: %v", open) + } else { + w.op.answer(open[0], OptionApprove) + } + w.settleAll() + var order []string + steps := []struct { + repo string + number int + head string + }{{wSDK, 92, wH92}, {wCtl, 472, wH472}, {wDel, 73, wH73}, {wDel, 72, wH72}} + for _, next := range steps { + handed := handing(w) + if len(handed) != 1 || handed[0].Repository != next.repo || handed[0].Number != next.number { + t.Fatalf("after %v: handed %q, wanted only %s#%d", order, handedWords(handed), next.repo, next.number) + } + merge := strings.Repeat(string(rune('a'+next.number%20)), 40) + bridgeMerged(w, next.repo, next.number, next.head, merge) + w.h.PullMerged(merged(next.repo, next.number, next.head, merge)) + w.settleAll() + order = append(order, fmt.Sprintf("%s#%d", next.repo, next.number)) + } + if left := handing(w); len(left) != 0 { + t.Fatalf("still handed: %q", handedWords(left)) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go index a73e68967..fc25f9384 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go @@ -135,18 +135,15 @@ func TestTheBridgesRefusalHandsNoMemberAndIsSaidUrgent(t *testing.T) { } } -// ADR 0441 §2: before a member is handed, every member's required statuses are read again; one that is not a success holds -// the group, even after the first member merged. -func TestTheNextMemberIsNotHandedWhileARequiredStatusOfTheGroupIsNotASuccess(t *testing.T) { +// novox/hq ADR 0661 §4 (changing ADR 0441 §2): the next member is handed on the Approve covering its head, with no read of +// the group's required statuses, after the first member merged. +func TestTheNextMemberIsHandedWithNoReadOfTheGroupsRequiredStatuses(t *testing.T) { w := answeredFirst(t) w.merges.say(forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: gCat, Number: 2, Head: gHk, Merge: strings.Repeat("a", 40), Root: "r", Trees: map[string]string{"x": "t"}}) setChecks(w, gCtl, gHc, "success", "") // the second member's repository check went missing meanwhile w.settleAll() - if len(w.merges.puts) != 1 { - t.Fatalf("the second member was handed with a required status missing: %d merges written", len(w.merges.puts)) - } - if lines := urgentLines(w); len(lines) != 1 || !strings.Contains(lines[0].Says, "mesh/repo-check (missing)") { - t.Fatalf("the held group is not said urgent: %+v", lines) + if len(w.merges.puts) != 2 || len(urgentLines(w)) != 0 { + t.Fatalf("%d merges written, lines %+v: wanted the second member handed on its Approve", len(w.merges.puts), urgentLines(w)) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps.go index c610b4c34..0bd8150cc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps.go @@ -6,7 +6,7 @@ import "git.novox.be/novox/mesh-sdk/go/announce" // issue 1790): the module says it itself, from its code, before it serves, and a reader is granted what it said // (clause 3). A state is readable when another module's manifest reads it (keeps_test.go names the readers). keeps_test.go holds the list to the // manifest's `state`, and keeps_integration_test.go that the module announces it on a bus. -var Kept = []announce.State{{Name: "asks", Readable: true}, {Name: "conditions"}, {Name: "deliveries"}, {Name: "groups"}, {Name: "merges", Readable: true}, {Name: "owed", Readable: true}, {Name: "pulls", Readable: true}, {Name: "source", Readable: true}, {Name: "turns", Readable: true}} +var Kept = []announce.State{{Name: "asks", Readable: true}, {Name: "conditions"}, {Name: "deliveries"}, {Name: "groups"}, {Name: "merges", Readable: true}, {Name: "owed", Readable: true}, {Name: "pulls", Readable: true}, {Name: "source", Readable: true}} // announceKept says Kept through keeps, which is stdio.Keeps. func announceKept(keeps func(...announce.State) error) error { return keeps(Kept...) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 8f1721dfd..468c75217 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -66,7 +66,6 @@ func main() { Operator: kvOperator{}, Merges: kvMerges{}, Source: kvSource{}, - Turns: kvTurns{}, Owed: seatOwed{say: stdio.SeatPublish}, Pulls: kvPulls{}, // The gate says what stalled to the operator itself (issue 1275, ADR 0437 §9). @@ -435,24 +434,6 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return h.Settle(ids, strArg(a, "why"), aPerson) }}, - {Name: "delivery_turns", - Description: "Every merge turn (novox/hq ADR 0548 §3): per repository, the approved merges that wait to merge into it in the order of " + - "their Approve, the check of another delivery each waits for and since when, whether the gate wrote the turn to its bucket " + - "turns yet (the checker claims no check of a repository a turn names), and the bound past which a held merge is a stalled " + - "line. Read-only.", - Run: func(map[string]any) (any, error) { return h.TurnsRead(), nil }}, - {Name: "delivery_hand_now", - Description: "On the operator's word, hand an approved merge held for another delivery's running check to the merge at once " + - "(novox/hq ADR 0548 §4): the check it waited for runs on and is carried to the new trunk after the merge. Refused for a " + - "delivery that is not an approved merge held by a check, naming why. The why is kept on the delivery.", - Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id (owner/repository@commit)"), - "why": str("the operator's word, kept on the delivery")}, "required": []string{"id", "why"}}, - Run: func(a map[string]any) (any, error) { - if err := need(a, "id", "why"); err != nil { - return nil, err - } - return h.HandNow(strArg(a, "id"), strArg(a, "why")) - }}, {Name: "delivery_merge_again", Description: "On the operator's word, ask again at once the merge of an approved delivery the forge refused, or of an approved " + "group held or half merged, at the heads the Approve named (novox/hq issue 1275): every member is read first and " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 16147a69d..dcc245082 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -120,12 +120,20 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if !reflect.DeepEqual(m.Emits, []string{"transition", "group", "merged"}) { t.Fatalf("emits %v", m.Emits) } - // Nine buckets, and no `asked` (the teller asks, ADR 0437 §6): ADR 0450 §3, the conditions it raised (issue 1275) and the - // merge turns (ADR 0548 §3). + // Eight buckets, and no `asked` (the teller asks, ADR 0437 §6): ADR 0450 §3 and the conditions it raised (issue 1275). No + // `turns`, no turn tool and no `check-gates-merge`: no check gates a merge (novox/hq ADR 0661 §1, §2, §6). buckets := m.buckets() - if !reflect.DeepEqual(buckets, []string{"deliveries", "groups", "pulls", "asks", "merges", "owed", "source", "conditions", "turns"}) { + if !reflect.DeepEqual(buckets, []string{"deliveries", "groups", "pulls", "asks", "merges", "owed", "source", "conditions"}) { t.Fatalf("state %v", buckets) } + for _, tool := range m.Tools { + if strings.Contains(tool, "turn") || strings.Contains(tool, "hand_now") { + t.Fatalf("the gate lists the turn tool %s", tool) + } + } + if raw, err := os.ReadFile("../../module.json"); err != nil || strings.Contains(string(raw), "check-gates-merge") { + t.Fatalf("the manifest still names check-gates-merge (%v)", err) + } // What it reads: the teller's answers and the checker's verdicts (issue 1302). The carry from today's module's buckets, the // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). The delivery view's reach, which // `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4); no forge tool is asked for it. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 8f69a2209..6ff3e41e3 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -164,6 +164,13 @@ func (v *Verdict) Judged() string { // outOfDate is why a delivery's verdict no longer counts: it names a trunk commit that is not its trunk's head // as this owner heard it (novox/hq ADR 0323 §7). Empty when it counts — or names no trunk commit, as a verdict // of a build seat from before the rule does, which counts as it did. +// approvedAtHead is whether the operator's Approve covers this delivery at its head: once it does, the merge judges that +// Approve alone, not the check (novox/hq ADR 0661 §3, §4). +func (d *Delivery) approvedAtHead() bool { + r := d.Ready + return r != nil && r.Approved != nil && !r.declined() && r.Heads[d.ID] == d.Commit +} + func (d *Delivery) outOfDate() string { if d.Check == nil || d.Check.Trunk == "" || d.TrunkNow == "" || sameCommit(d.Check.Trunk, d.TrunkNow) { return "" diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go index 72cd0e552..554ad13f3 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go @@ -97,7 +97,7 @@ func (h *Holder) CheckerVerdictChanged(key string, value []byte) { v.Heads = hash } if prev, ok := h.checker[v.Heads]; v.State == checkerRunning && (!ok || prev.State != checkerRunning) { - h.runningSince[v.Heads] = h.Now() // a check starts: a merge it holds says since when (ADR 0548 §2) + h.runningSince[v.Heads] = h.Now() // a check starts: its words say since when } else if v.State != checkerRunning { delete(h.runningSince, v.Heads) } @@ -171,10 +171,12 @@ func (h *Holder) takeCheckStatus(repository, sha, state, desc string) { } h.moved(d, []Transition{t}) } - if d.State == Ready && !h.checkGates() { - // The check gates no merge (check-gates-merge false): its verdict is shown on the delivery, and moves nothing. + if d.approvedAtHead() && (d.State == Ready || d.State == Merged) { + // The merge judges the operator's Approve alone (novox/hq ADR 0661 §3): a verdict after it, a carry's, is shown and + // moves nothing; a failure is said (failedAfterApprove). d.Check = v - h.Logf("[mesh-delivery] %s: its mesh/check is %s — %s (shown; the check gates no merge)", d.ID, state, oneLine(clip(desc, 120))) + h.Logf("[mesh-delivery] %s: its mesh/check is %s — %s (after the operator's Approve)", d.ID, state, oneLine(clip(desc, 120))) + h.failedAfterApprove(d, v) h.keep(d) return } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 37186e644..f3da1acf1 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -128,12 +128,11 @@ type ReadyRun struct { Warrant string `json:"warrant,omitempty"` Refused string `json:"refused,omitempty"` - // One check per delivery (novox/hq ADR 0548 §2, §4, turns.go): HeldSince is when the approved merge began to wait for the - // check of other heads HeldFor names (its verdict key), running on a repository it goes into; HandNow the operator's word - // that handed it at once past that check. - HeldSince *time.Time `json:"held_since,omitempty"` - HeldFor string `json:"held_for,omitempty"` - HandNow string `json:"hand_now,omitempty"` + // FailedAfter is a failure heard after the operator's Approve covered the head, a carry's (novox/hq ADR 0661 §5): said + // once on the pull request and as a stalled line naming the merge it follows, never a refusal of the merge; cleared by a + // later pass. FailedAfterSince is when it was heard. + FailedAfter string `json:"failed_after,omitempty"` + FailedAfterSince *time.Time `json:"failed_after_since,omitempty"` // CannotWrite is why the operator is not asked: the forge's merging account cannot write a repository the ask covers // (writes.go, issue 1945); CannotWriteSince since when. Said at once as a condition. @@ -194,7 +193,6 @@ func (h *Holder) runOf(d *Delivery, now time.Time) *ReadyRun { func (h *Holder) readyAsks(now time.Time) []func() { var asks []func() h.endStaleAsks() - gates := h.checkGates() ids := make([]string, 0, len(h.deliveries)) for id := range h.deliveries { ids = append(ids, id) @@ -205,9 +203,6 @@ func (h *Holder) readyAsks(now time.Time) []func() { if d.Number == 0 || d.State.Final() || d.MergedAs != "" { continue // not of a pull request, or past its merge } - if !gates { - h.ungate(d, now) - } if d.Ready == nil && d.State == Ready { h.runOf(d, now) h.Poke() // its check, if none is asked, is worth asking now (askCheck) @@ -276,9 +271,6 @@ func (h *Holder) readyAsks(now time.Time) []func() { if waits == "" { waits, approvalAsk = h.approval(d, now) } - if waits == "" && gates { - waits = h.turnWaits(d, now) // nothing merges into a repository while a check of other heads judges it (ADR 0548 §2) - } if waits == "" { waits = h.passWaits(d) // no merge lands in a repository between another approved group's members (issue 1275) } @@ -310,7 +302,6 @@ func (h *Holder) readyAsks(now time.Time) []func() { h.noteNotHanded(d, h.notHanded(g, d), now) // a merge never handed says why (2026-10-29: silent for an hour) } } - asks = append(asks, h.turnWrites()...) // the merge turns, after the holds of this look (ADR 0548 §3) asks = append(asks, h.settleMerges()...) asks = append(asks, h.settleForgeMerges()...) asks = append(asks, h.recordForgeMade()...) @@ -321,30 +312,6 @@ func (h *Holder) readyAsks(now time.Time) []func() { return asks } -// ungate stands a delivery where it is while mesh/check gates no merge (check-gates-merge false): a proposed one is ready -// with no verdict, and a rejected one, refused on a verdict before the setting changed, is proposed again and so ready. Its -// verdict stays on it, shown. Under the lock. -func (h *Holder) ungate(d *Delivery, now time.Time) { - f := h.factsAt(now) - if d.State == Rejected { - f.By, f.Why = byOwner, "the check gates no merge (check-gates-merge false): its verdict is shown, not waited on" - t, err := Apply(d, EvRecheck, true, f) - if err != nil { - h.refuse(err) - return - } - if d.Ready != nil { - d.Ready.Failed = "" - } - h.moved(d, []Transition{t}) - f.By, f.Why = "", "" - } - if d.State == Proposed { - h.moved(d, settle(d, f)) - h.closeGroupOf(d) - } -} - // sayAfterLine tells a pull request, once, that its description holds an `after: ` line, which is not read // (novox/hq ADR 0419 §2): the order of a delivery comes from the module graph alone, and no refusal asks for the line. // Once per pull request, not per head. Under the lock; the comment is said outside it. @@ -406,14 +373,21 @@ func (r *ReadyRun) noteRefusal(d *Delivery, answer string) { r.Repeats, r.RepeatTrunk = 1, d.TrunkNow } -// readyWaits is what a delivery's merge waits on; empty when the Approve may be asked and the merge asked. A ready -// delivery is asked. So is a proposed one whose verdict went out of date, when an Approve was asked for it already (it -// stays through a trunk move, ADR 0421 §3) and its check against the trunk's head is not on its way; one without an -// Approve waits for that check first. +// readyWaits is what a delivery's merge waits on; empty when the Approve may be asked and the merge asked. Once the +// operator's Approve covers its heads the merge judges that Approve alone (approvedWaits); until then the check gates the +// ask (askWaits), the one place a check gates anything here (novox/hq ADR 0661 §3, §4). Under the lock. func (h *Holder) readyWaits(d *Delivery) string { - if !h.checkGates() { - return h.ungatedWaits(d) + if c := h.covering(d); c != nil && c.Approved != nil { + return h.approvedWaits(d) } + return h.askWaits(d) +} + +// askWaits is why the operator's Approve is not asked yet: the check gates the ask (novox/hq ADR 0661 §3, ADR 0421 §1). A +// ready delivery is asked once the checker's verdict at exactly the heads the ask covers passed; a proposed one whose +// verdict went out of date while its ask is pending waits for its check against the trunk's head. A group is asked only +// once its composed check passed at its members' heads (ADR 0661 §4). +func (h *Holder) askWaits(d *Delivery) string { if d.judgedByCheck() && d.State == Proposed && d.outOfDate() != "" && h.covering(d) == nil { return "its mesh/check is " + d.outOfDate() + ": the checker checks it again against the trunk's head" } @@ -421,10 +395,6 @@ func (h *Holder) readyWaits(d *Delivery) string { case d.State == Ready: case d.State == Proposed && d.outOfDate() != "" && d.AskedAgain != nil: return "its check against the trunk's head is asked and has not answered" - case d.State == Proposed && d.outOfDate() != "" && d.Check.Passes() && h.covering(d) != nil: - // Approved, and its trunk moved: the checker carries its verdict to the trunk's head by its package digests (ADR 0548 - // §5, §8). The Approve stands and is not asked again; the merge waits for the carried verdict, not a fresh check. - return "approved; its verdict is being carried to the trunk's head " + shortOf(d.TrunkNow, 12) + " (" + d.outOfDate() + ")" case d.State == Proposed && d.outOfDate() != "": return "its verdict is out of date: its check against the trunk's head is asked before the operator is" case d.State == Proposed && d.AskedAgain != nil: @@ -442,10 +412,16 @@ func (h *Holder) readyWaits(d *Delivery) string { return h.groupTurnToMerge(d) } -// ungatedWaits is readyWaits while mesh/check gates no merge (check-gates-merge false): a ready delivery waits for nothing but -// a group member for every member to be ready; no verdict, composed check or trunk is waited on, -// so the Approve is asked at once. -func (h *Holder) ungatedWaits(d *Delivery) string { +// approvedWaits is what a merge the operator's Approve covers waits on: nothing of the check (novox/hq ADR 0661 §3, §4). +// No verdict, carry, composed check or trunk move is waited on, and no status is read again; a group member waits only +// for every member to be ready, at the heads the Approve covers, so the group merges in one pass. A proposed delivery +// the Approve covers (a trunk moved before the Approve took its run) is ready on it. Under the lock. +func (h *Holder) approvedWaits(d *Delivery) string { + if c := h.covering(d); d.State == Proposed && d.Ready != nil && c != nil && c.Approved != nil { + now := h.Now() + h.approval(d, now) // the covering Approve taken onto its own run, so the table reads it + h.moved(d, settle(d, h.factsAt(now))) + } if d.State != Ready { return "it is " + string(d.State) + ", not ready yet" } @@ -456,9 +432,6 @@ func (h *Holder) ungatedWaits(d *Delivery) string { } } } - // No verdict is waited on (the operator, 2026-11-07: "we aren't running any checks anymore, we're delivering"): while - // the check gates no merge, the Approve is asked once the delivery is ready; a verdict that comes later - // is shown on the pull request and the delivery, as before. return "" } @@ -517,12 +490,50 @@ func (h *Holder) sayOn(d *Delivery, body string) func() { return func() {} } +// failedAfterApprove says a failure heard after the operator's Approve covered the head: once on the pull request, and +// as the stalled line readyLines lists; the merge is not refused for it, and the operator decides the remedy, a fix or a +// revert (novox/hq ADR 0661 §5). A later pass clears it. Under the lock. +func (h *Holder) failedAfterApprove(d *Delivery, v *Verdict) { + r := d.Ready + if r == nil { + return + } + if v.Passes() { + r.FailedAfter, r.FailedAfterSince = "", nil + return + } + if r.FailedAfter != "" { + return + } + now := h.Now() + r.FailedAfter, r.FailedAfterSince = "its check failed after the operator's Approve: "+oneLine(clip(v.Summary, 120)), &now + h.Logf("[mesh-delivery] %s (%s): %s", d.ID, h.mergeFollowed(d), r.FailedAfter) + h.sayOn(d, fmt.Sprintf("mesh-delivery: **failed after the Approve.** This pull request (%s) %s. The merge is not "+ + "refused for it: the operator decides the remedy, a fix or a revert.", h.mergeFollowed(d), r.FailedAfter)) +} + +// mergeFollowed names the merge a failure after the Approve follows: the trunk commit it merged as, or its approved head +// not merged yet. +func (h *Holder) mergeFollowed(d *Delivery) string { + if d.MergedAs != "" { + return "merged as " + shortOf(d.MergedAs, 12) + " at its head " + shortOf(d.Commit, 12) + } + return "approved at its head " + shortOf(d.Commit, 12) + ", not merged yet" +} + // readyLines are the approved heads that will not merge: a check that failed at once, a merge refused past its bound; // as stalled lines, which the gate says as conditions (conditions.go). Under the lock. func (h *Holder) readyLines(now time.Time) []StalledLine { var out []StalledLine for _, d := range h.deliveries { r := d.Ready + if r != nil && r.FailedAfter != "" && r.FailedAfterSince != nil && d.State != Closed { + out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, + For: now.Sub(*r.FailedAfterSince).Round(time.Second).String(), Bound: "0s", + H2: "none: the operator decides the remedy, a fix or a revert; a later pass clears it", + Says: fmt.Sprintf("%s#%d (%s): %s", d.Repository, d.Number, h.mergeFollowed(d), r.FailedAfter)}) + continue + } if r == nil || r.Merged != nil || d.MergedAs != "" || d.State.Final() { continue } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go index 6c80aec72..2a00b03f2 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go @@ -8,7 +8,6 @@ import ( "time" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" - mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" ) // **A pull request whose checks passed and whose merge the operator approved is merged by this owner** (novox/hq issues @@ -85,22 +84,9 @@ func TestAnApprovedPullRequestIsMergedThroughTheMergeTool(t *testing.T) { {name: "gated, passed", gated: true, state: "pass", asks: 1, head: "Merge into main: app#7 Fix it", want: []string{"app#7 Fix it passed its check. Approve merges it into main at " + readyHead[:12] + ";"}, not: []string{"before its check ends", "no verdict yet"}}, - // The operator, 2026-11-07: "we aren't running any checks anymore, we're delivering": while the check gates no - // merge the Approve is asked at once, with no verdict or a running check, never waiting for one. - {name: "ungated, no verdict", asks: 1, not: []string{waitsCheckBeforeAsking}}, - {name: "ungated, running", state: "running", asks: 1, not: []string{waitsCheckBeforeAsking}}, - // Asked at once, then asked again in its words once the verdict comes (a later verdict asks a pending Approve again). - {name: "ungated, passed", state: "pass", asks: 2, - want: []string{"passed its check. Approve merges it into main at " + readyHead[:12] + ";"}, not: []string{"before its check ends"}}, - {name: "ungated, failed", state: "fail", failed: "TestThing: wanted 1, got 2", asks: 2, - head: "Check failed \u2014 merge into main: app#7 Fix it", - want: []string{"its check failed: TestThing: wanted 1, got 2", "although its check failed"}, not: []string{"passed", "anyway"}}, } { t.Run(row.name, func(t *testing.T) { v := newWorld(t) - if !row.gated { - ungated(v) - } v.op.auto = false p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") p.Title = "Fix it" @@ -158,47 +144,6 @@ func TestAnApprovedPullRequestIsMergedThroughTheMergeTool(t *testing.T) { } }) } - - // check-gates-merge false (the operator, 2026-10-25 22:56): a reviewed pull request is asked with no verdict, another - // merge moving its trunk puts it out of nothing, a failed verdict is shown on it and holds nothing, and the gate's - // status and show say the mode. - u := newWorld(t) - ungated(u) - u.op.auto = false - const other = "3333333333333333333333333333333333333333" - u.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) - u.h.PullUpdated(pr("novox/app", 8, other, "feat/y", "src/b.go")) - u.settleAll() - if u.op.count() == 0 || u.state(IDOf("novox/app", readyHead)) != Ready { - t.Fatalf("ungated, two reviewed pull requests with no verdict: %d asks, %s; wanted ready and asked at once", u.op.count(), u.state(IDOf("novox/app", readyHead))) - } - if gates, _ := u.h.Status("listening")["check-gates-merge"].(bool); gates { - t.Fatalf("delivery_status does not say the check gates no merge: %v", u.h.Status("listening")) - } - shown, _ := u.h.Show(IDOf("novox/app", other)) - if gates, _ := shown.(map[string]any)["check-gates-merge"].(bool); gates { - t.Fatalf("show does not say the check gates no merge: %v", shown) - } - u.h.Checked(judgedAt("novox/app", 8, other, "fail", "t1t1t1t1t1t1")) - checkerSays(u, "feat/y", "fail", "TestB", headOf("novox/app", 8, other)) - u.h.PullMerged(merged("novox/app", 7, readyHead, "m7m7m7m7m7m7")) - u.settleAll() - if u.op.count() == 0 { - t.Fatalf("ungated, a failed verdict holds the ask (the check gates no merge): %d asks", u.op.count()) - } - d = u.delivery(IDOf("novox/app", other)) - if d.State != Ready || d.Check == nil || d.Check.Passes() { - t.Fatalf("ungated, after a failed verdict and a trunk's move: %s, verdict %+v", d.State, d.Check) - } - for _, a := range u.op.open() { - if strings.Contains(a, "feat/y") { - u.op.answer(a, OptionApprove) - } - } - u.settleAll() - if len(u.forge.merges) != 1 || u.forge.merges[0] != "novox/app#8@"+other { - t.Fatalf("ungated, the approved head is merged %v; wanted once, with no verdict waited on", u.forge.merges) - } } // 1144/AC3: a refused merge is not asked again by any timer; only an event (here, mergeAgainNow called directly, as a @@ -342,57 +287,6 @@ func TestAGroupMergesEveryMemberInOnePassOnlyOnceComposedAndApproved(t *testing. if w.op.count() != 1 { t.Fatalf("a second Approve was asked: %d", w.op.count()) } - - // check-gates-merge false: the group is asked once with no verdict and no composed check, and merges whole in one pass. - u := newWorld(t) - ungated(u) - u.op.auto = false - u.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) - u.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) - u.settleAll() - group := []mdc.Head{headOf("novox/mesh-controller", 1, ctlHead), headOf("novox/mesh-catalog", 2, catHead)} - checkerSays(u, "feat/x", "running", "", group...) - u.settleAll() - // Issue 1966 (the operator, 2026-11-07): the group is asked at once, its check running or not; nothing waits on it. - if d := u.delivery(IDOf("novox/mesh-catalog", catHead)); u.op.count() != 1 || d.Ready == nil || - strings.Contains(d.Ready.Waits, waitsCheckBeforeAsking) { - t.Fatalf("ungated, the group is asked %d times while its check runs; wanted once at once, and the wait said: %+v", u.op.count(), d.Ready) - } - checkerSays(u, "feat/x", "pass", "", group...) - u.settleAll() - if u.op.count() != 2 { - t.Fatalf("ungated, the group is asked %d times once its check passed; wanted once at once and once again in its words", u.op.count()) - } - if e := u.op.ask(u.op.open()[0]).Explanation; !strings.Contains(e, "2 pull requests, passed its check. Approve merges all of them into main in one pass, at the heads named;") { - t.Fatalf("ungated, the group's ask does not say its check passed:\n%s", e) - } - // A member re-pushed hours after its sibling became ready: with no verdict at the new heads nothing is asked, however - // long the sibling has waited. - r := newWorld(t) - ungated(r) - r.op.auto = false - const ctlHead2 = "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3" - r.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go")) - r.h.PullUpdated(pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")) - r.settleAll() - r.later(5 * time.Hour) - r.h.PullUpdated(pr("novox/mesh-controller", 1, ctlHead2, "feat/x", "internal/a.go")) - // Made ready by a handler, before the look gives it a run: no verdict and no status at the new heads. - r.h.mu.Lock() - r.h.ungate(r.h.deliveries[IDOf("novox/mesh-controller", ctlHead2)], r.h.Now()) - r.h.mu.Unlock() - r.settleAll() - // Issue 1966: no verdict is waited on, so the new heads are asked again at once, never held for a check. - if d := r.delivery(IDOf("novox/mesh-catalog", catHead)); r.op.count() == 0 || d.Ready == nil || - strings.Contains(d.Ready.Waits, waitsCheckBeforeAsking) { - t.Fatalf("a member re-pushed hours later: asked %d times, wait %+v; wanted asked again at once", r.op.count(), d.Ready) - } - - u.op.answer(u.op.open()[0], OptionApprove) - u.settleAll() - if len(u.forge.merges) != 2 { - t.Fatalf("ungated, both members merge in one pass: %v", u.forge.merges) - } } // **A merge into another branch than main is its author's** (novox/hq ADR 0398 §4, issue 948): a green pull request diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go index c3217ae2e..50b905024 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settings.go @@ -25,10 +25,6 @@ const ( type Settings struct { RefusalFloorMinutes *int `json:"refusal-stall-floor-minutes"` AuthorBoundMinutes *int `json:"author-new-head-minutes"` - // CheckGatesMerge is whether mesh/check gates the merge (the default, true). False (the operator, 2026-10-25 22:56: - // "run the check locally only, but not as a gate, it loses us too much time"): a reviewed pull request is ready and its - // Approve asked without a verdict, a trunk's move puts nothing out of date, and a verdict is shown, never waited on. - CheckGatesMerge *bool `json:"check-gates-merge"` // RecordRetentionDays is how many days a delivery record is kept from when it opened (design 47 *The delivery // record*); nil keeps DefaultRecordRetention. RecordRetentionDays *int `json:"record-retention"` @@ -73,28 +69,13 @@ func readSettings(path string) (Settings, error) { default: return Settings{}, errors.New("record-retention is not a number") } - switch v := loose["check-gates-merge"].(type) { - case nil: - case bool: - s.CheckGatesMerge = &v - default: - return Settings{}, errors.New("check-gates-merge is not true or false") - } return s, nil } -// checkGates is whether mesh/check gates the merge: true unless the setting check-gates-merge is false. -func (h *Holder) checkGates() bool { - if h.Settings == nil { - return true - } - v := h.Settings().CheckGatesMerge - return v == nil || *v -} - -// factsAt are the facts a settle reads at now: the moment, and whether the check gates the merge. +// factsAt are the facts a settle reads at now: the moment. No setting switches whether a check gates a merge: the check +// gates the ask, and the merge judges the operator's Approve alone (novox/hq ADR 0661 §2, §3). func (h *Holder) factsAt(now time.Time) Facts { - return Facts{Now: now, Ungated: !h.checkGates()} + return Facts{Now: now} } // minutesOr is a setting's minutes, or the default when it gives none. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go index bfc56be93..771116f70 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go @@ -94,9 +94,6 @@ type Facts struct { Now time.Time By string Why string - // Ungated is the setting check-gates-merge false: mesh/check gates no merge, so a proposed delivery is accepted with no - // verdict, none is refused for one, and a trunk's move puts none back to proposed. - Ungated bool } // Row is one transition the table holds. @@ -145,10 +142,10 @@ var Table = []Row{ // cannot loop with `trunk moved`; a recheck or the merge tool's ask judges it again. {From: []State{Proposed}, Event: EvAccepted, To: Ready, Guard: "a verdict names this commit and the trunk's head as its trunk commit, the gate passed or warned, and " + - "the repository's own check did not fail; or the check gates no merge (check-gates-merge false)", + "the repository's own check did not fail; or the operator's Approve covers this head (the merge judges it alone)", Holds: func(d *Delivery, f Facts) error { - if f.Ungated { - return nil + if d.approvedAtHead() { + return nil // novox/hq ADR 0661 §3: an Approve given stands across a trunk move, and the merge judges it alone } if d.Check == nil { return errors.New("no verdict") @@ -160,10 +157,11 @@ var Table = []Row{ }}, {From: []State{Proposed}, Event: EvRefused, To: Rejected, Guard: "a verdict names this commit and the trunk's head as its trunk commit, and the gate failed or could not run, " + - "or the repository's own check failed; never while the check gates no merge", + "or the repository's own check failed; never once the operator's Approve covers this head", Holds: func(d *Delivery, f Facts) error { - if f.Ungated { - return errors.New("the check gates no merge (check-gates-merge false)") + if d.approvedAtHead() { + // novox/hq ADR 0661 §5: a failure after the Approve is said, and does not undo it. + return errors.New("the operator's Approve covers this head: a failure is said, the merge is not refused") } if d.Check == nil { return errors.New("no verdict") @@ -176,10 +174,11 @@ var Table = []Row{ // **A ready delivery whose verdict went out of date goes back to proposed** (novox/hq ADR 0323 §7): another // merge moved its trunk. A rejected one stays rejected, its status on the forge saying the verdict is out of date. {From: []State{Ready}, Event: EvTrunkMoved, To: Proposed, - Guard: "its verdict names a trunk commit that is not its trunk's head, and the check gates the merge", + Guard: "its verdict names a trunk commit that is not its trunk's head, and no Approve covers this head yet", Holds: func(d *Delivery, f Facts) error { - if f.Ungated { - return errors.New("the check gates no merge (check-gates-merge false)") + if d.approvedAtHead() { + // novox/hq ADR 0661 §3: an approved merge waits for no carry; its carry runs after the merge. + return errors.New("the operator's Approve covers this head: it merges onto the new trunk, its carry after") } return unless(d.MergedAs == "" && d.outOfDate() != "", "not merged, and its verdict is not out of date") }, diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go index bf92f3067..1469a95a5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go @@ -161,21 +161,34 @@ func TestTheTableKeepsItsRules(t *testing.T) { if _, err := Apply(d0, EvReask, true, Facts{Now: time.Now(), By: byH2, Why: "stalled"}); err == nil { t.Fatal("a check re-asked once was re-asked again") } - // While the check gates no merge (check-gates-merge false): a proposed delivery is accepted with no verdict or a failed - // one, none is refused, and a trunk's move puts no ready one back to proposed. - off := Facts{Now: time.Now(), Ungated: true} + // Once the operator's Approve covers the head (novox/hq ADR 0661 §3, §5): a proposed delivery is accepted with no + // verdict or a failed one, none is refused, and a trunk's move puts no ready one back to proposed. + at := time.Now() + approved := func(d *Delivery) *Delivery { + d.Ready = &ReadyRun{Approved: &at, Heads: map[string]string{d.ID: d.Commit}} + return d + } + f := Facts{Now: at} for _, check := range []*Verdict{nil, {Gate: "fail"}} { - p := &Delivery{ID: "novox/app@c0", State: Proposed, Number: 1, Check: check} - if _, err := Apply(p, EvRefused, false, off); err == nil { - t.Fatalf("ungated, a verdict %+v refused the delivery", check) + p := approved(&Delivery{ID: "novox/app@c0", Commit: "c0", State: Proposed, Number: 1, Check: check}) + if _, err := Apply(p, EvRefused, false, f); err == nil { + t.Fatalf("approved, a verdict %+v refused the delivery", check) } - if _, err := Apply(p, EvAccepted, false, off); err != nil || p.State != Ready { - t.Fatalf("ungated, a delivery with the verdict %+v is %s: %v", check, p.State, err) + if _, err := Apply(p, EvAccepted, false, f); err != nil || p.State != Ready { + t.Fatalf("approved, a delivery with the verdict %+v is %s: %v", check, p.State, err) + } + unapproved := &Delivery{ID: "novox/app@c0", Commit: "c0", State: Proposed, Number: 1, Check: check} + if _, err := Apply(unapproved, EvAccepted, false, f); err == nil { + t.Fatalf("with no Approve, a delivery with the verdict %+v was accepted: the check gates the ask", check) } } - stale := &Delivery{ID: "novox/app@c0", State: Ready, Check: &Verdict{Gate: "pass", Trunk: "d28ca95bbbbb"}, TrunkNow: "690966dccccc"} - if _, err := Apply(stale, EvTrunkMoved, false, off); err == nil { - t.Fatal("ungated, a trunk's move put a ready delivery back to proposed") + stale := approved(&Delivery{ID: "novox/app@c0", Commit: "c0", State: Ready, Check: &Verdict{Gate: "pass", Trunk: "d28ca95bbbbb"}, TrunkNow: "690966dccccc"}) + if _, err := Apply(stale, EvTrunkMoved, false, f); err == nil { + t.Fatal("approved, a trunk's move put a ready delivery back to proposed") + } + stale.Ready = nil + if _, err := Apply(stale, EvTrunkMoved, false, f); err != nil || stale.State != Proposed { + t.Fatalf("with no Approve, a trunk's move left a ready delivery %s: %v", stale.State, err) } // A ready delivery off the trunk is refused its merge, whatever else is true of it. d := &Delivery{ID: "novox/app@c0", State: Ready} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go deleted file mode 100644 index a5bd78f73..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/turns.go +++ /dev/null @@ -1,461 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "sort" - "strings" - "time" - - checker "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliverycheckerclient" - mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// **One check per delivery** (novox/hq ADR 0548 §2 to §4, issue 568). While a check runs, nothing merges into a repository it -// judges, and an approved merge holds a turn on its repositories so no new check starts there: -// -// - the hold (§2): an approved merge is handed to the forge bridge only when no verdict of the checker's verdicts bucket in -// the state running names, among its trunk heads, a repository the merge goes into; the verdict of the merge's own heads -// excepted. The gate watches that bucket already (onestatus.go), so the verdict leaving running gives the waiting merge its -// turn at once, with no timer. Siblings a check reads beside are not held (§2). Only while mesh/check gates the merge; -// - the turn (§3): before it reads the verdicts for a merge, the gate writes one key per repository the merge goes into in its -// bucket turns (`.`, mdc.TurnsKey): the deliveries that wait there in the order of their Approve, the check -// each waits for and since when; for a group on every member's repository until its last member merged. The checker reads -// the bucket and claims no check of a repository a turn names. The key goes when the last delivery in it merged or left; -// - the bound and the tools (§4): a merge held past turnBound is a stalled line; delivery_turns reads every turn and -// delivery_hand_now hands a waiting merge at once on the operator's word, the check it waited for then carried. - -// turnBound is how long an approved merge may wait for another delivery's check before it is a stalled line (ADR 0548 §4). -const turnBound = 30 * time.Minute - -// TurnStore is the gate's bucket turns: one key per repository, written by the gate alone. -type TurnStore interface { - // All reads every turn, by repository (lowercase). - All() (map[string]Turn, error) - Put(t Turn) error - Delete(repository string) error -} - -// kvTurns is the bucket turns over the runtime. -type kvTurns struct{} - -func (kvTurns) All() (map[string]Turn, error) { - keys, err := stdio.State("turns").Keys() - if err != nil { - return nil, err - } - out := map[string]Turn{} - for _, k := range keys { - e, err := stdio.State("turns").Get(k) - if err != nil { - return nil, err - } - if e == nil { - continue - } - repository, err := mdc.ParseTurnsKey(k) - if err != nil { - continue // not a turn's key: never the gate's, left - } - t, err := DecodeTurn(e.Value) - if err != nil { - t = Turn{Repository: repository} // unreadable: rewritten or deleted at the next look - } - out[repository] = t - } - return out, nil -} - -func (kvTurns) Put(t Turn) error { - key, err := mdc.TurnsKey(t.Repository) - if err != nil { - return err - } - raw, err := EncodeTurn(t) - if err != nil { - return err - } - _, err = stdio.State("turns").Put(key, json.RawMessage(raw)) - return err -} - -func (kvTurns) Delete(repository string) error { - key, err := mdc.TurnsKey(repository) - if err != nil { - return err - } - return stdio.State("turns").Delete(key) -} - -// approvedMerge is an approved merge not yet done: the run holding its Approve, the deliveries it covers, the hash of their -// heads as the checker keys their verdict (lowercase, onestatus.go headsOf), and the repositories it goes into (lowercase). -type approvedMerge struct { - owner *Delivery - run *ReadyRun - group string - heads string - repos []string -} - -// approvedMerges are the merges the operator approved that are not done, in the order of their Approve: each ask once, its -// run on the delivery that asked it. A declined, refused, merged, rejected or final one has left. Under the lock. -func (h *Holder) approvedMerges() []approvedMerge { - var out []approvedMerge - seen := map[string]bool{} - ids := make([]string, 0, len(h.deliveries)) - for id := range h.deliveries { - ids = append(ids, id) - } - sort.Strings(ids) // the same owner of an ask at every look, whichever member's run holds it - for _, id := range ids { - d := h.deliveries[id] - r := d.Ready - if r == nil || r.Approved == nil || r.declined() || r.AskID == "" || len(r.Heads) == 0 || seen[r.AskID] || - d.State.Final() || d.State == Rejected || d.Base != MergeBranch || r.RefusedSince != nil || h.allMerged(r.Heads) { - continue - } - seen[r.AskID] = true - var heads []mdc.Head - repos := map[string]bool{} - for id, sha := range r.Heads { - m := h.deliveries[id] - if m == nil { - continue - } - heads = append(heads, mdc.Head{Repository: strings.ToLower(m.Repository), Number: m.Number, Head: strings.ToLower(sha)}) - repos[strings.ToLower(m.Repository)] = true - } - hash, err := mdc.HeadsHash(heads) - if err != nil { - continue - } - am := approvedMerge{owner: d, run: r, group: askGroup(d), heads: hash} - for repo := range repos { - am.repos = append(am.repos, repo) - } - sort.Strings(am.repos) - out = append(out, am) - } - sort.SliceStable(out, func(i, j int) bool { - if !out[i].run.Approved.Equal(*out[j].run.Approved) { - return out[i].run.Approved.Before(*out[j].run.Approved) - } - return out[i].heads < out[j].heads - }) - return out -} - -// mergeOfDelivery is the approved merge covering d, if one is approved. -func (h *Holder) mergeOfDelivery(d *Delivery) (approvedMerge, bool) { - c := h.covering(d) - if c == nil || c.Approved == nil { - return approvedMerge{}, false - } - for _, am := range h.approvedMerges() { - // By the ask, not the run: a group's members may each hold a run of the same ask, and the merge is the ask's. - if am.run.AskID == c.AskID { - return am, true - } - } - return approvedMerge{}, false -} - -// runningCheck is a check of the checker in the state running that names repo among its trunk heads, its heads not own: its -// verdict key and since when the gate has seen it run. False when none runs there. Under the lock. -func (h *Holder) runningCheck(repo, own string) (key string, v checkerVerdict, since time.Time, ok bool) { - hashes := make([]string, 0, len(h.checker)) - for hash := range h.checker { - hashes = append(hashes, hash) - } - sort.Strings(hashes) - for _, hash := range hashes { - v := h.checker[hash] - if v.State != checkerRunning || v.Heads == own { - continue - } - for r := range v.Trunk { - if strings.EqualFold(r, repo) { - key, err := checker.VerdictKey(v.Group, v.Heads) - if err != nil { - key = v.Group + "." + v.Heads - } - return key, v, h.runningSince[v.Heads], true - } - } - } - return "", checkerVerdict{}, time.Time{}, false -} - -// turnWaits is what an approved merge waits for before it is handed (ADR 0548 §2, §3): its turns written, then the end of -// every check of other heads running on a repository it goes into. Empty when it may be handed. It keeps on the run since -// when it is held and for which check, for the stalled line and the tools. Under the lock. -func (h *Holder) turnWaits(d *Delivery, now time.Time) string { - am, ok := h.mergeOfDelivery(d) - if !ok { - return "" - } - r := am.run - if h.Turns != nil { - if !h.turnsLoaded { - return "approved; its merge turn waits for the turns the gate wrote before to be read" - } - for _, repo := range am.repos { - if !turnNames(h.turnsHeld[repo], am.heads) { - return "approved; its merge turn on " + repo + " is being written, so no check starts there before it merges" - } - } - } - if r.HandNow != "" { - if r.HeldSince != nil { - r.HeldSince, r.HeldFor = nil, "" - h.keep(am.owner) - } - return "" // handed now on the operator's word: the check it waited for is carried after the merge - } - for _, repo := range am.repos { - key, v, since, running := h.runningCheck(repo, am.heads) - if !running { - continue - } - if r.HeldSince == nil || r.HeldFor != key { - at := now - r.HeldSince, r.HeldFor = &at, key - h.keep(am.owner) - h.Logf("[mesh-delivery] %s is approved and held: the check %s runs on %s, and nothing merges there while it does (ADR 0548 §2)", am.owner.ID, key, repo) - } - words := fmt.Sprintf("approved; waits for the check of %s (heads %s) on %s", v.Group, v.Heads, repo) - if !since.IsZero() { - words += ", running since " + since.UTC().Format(time.RFC3339) - } - return words + " (novox/hq ADR 0548 §2)" - } - if r.HeldSince != nil { - r.HeldSince, r.HeldFor = nil, "" - h.keep(am.owner) - } - return "" -} - -// turnNames says whether a turn names the heads. -func turnNames(t Turn, heads string) bool { - for _, w := range t.Waiting { - if w.Heads == heads { - return true - } - } - return false -} - -// wantedTurns are the turns the approved merges hold now, by repository: each merge on every repository it goes into, in -// the order of their Approve, with the check of other heads it waits for there. Under the lock. -func (h *Holder) wantedTurns() map[string]Turn { - out := map[string]Turn{} - for _, am := range h.approvedMerges() { - for _, repo := range am.repos { - t := out[repo] - t.Repository = repo - w := TurnWait{Group: am.group, Heads: am.heads, Ask: am.run.AskID, Approved: am.run.Approved.UTC(), Since: am.run.Approved.UTC()} - if key, _, since, ok := h.runningCheck(repo, am.heads); ok { - w.WaitsFor = key - if !since.IsZero() { - w.WaitsSince = since.UTC() - } - } - t.Waiting = append(t.Waiting, w) - out[repo] = t - } - } - return out -} - -// turnWrites are the writes that bring the bucket turns to what the approved merges hold: a turn written where it changed, -// a key deleted where its last delivery merged or left. The first look reads what a gate before this one wrote. Under the -// lock; the writes run outside it, and each that lands is kept and asks for a look. -func (h *Holder) turnWrites() []func() { - if h.Turns == nil { - return nil - } - if !h.turnsLoaded { - return []func(){func() { - held, err := h.Turns.All() - h.mu.Lock() - defer h.mu.Unlock() - if err != nil { - h.Logf("[mesh-delivery] the merge turns written before cannot be read, so no merge is handed yet: %v; asked again in %s", err, askAgainAfter) - h.again("the merge turns written before, read") - return - } - h.turnsHeld, h.turnsLoaded = held, true - h.Poke() - }} - } - want := map[string]Turn{} - if h.checkGates() { - want = h.wantedTurns() // while the check gates no merge, no merge waits for one, and no turn holds a check - } - var out []func() - repos := make([]string, 0, len(want)+len(h.turnsHeld)) - for repo := range want { - repos = append(repos, repo) - } - for repo := range h.turnsHeld { - if _, ok := want[repo]; !ok { - repos = append(repos, repo) - } - } - sort.Strings(repos) - for _, repo := range repos { - t, wanted := want[repo] - held, had := h.turnsHeld[repo] - switch { - case wanted && had && sameTurnValue(held, t): - case wanted: - out = append(out, func() { - err := h.Turns.Put(t) - h.mu.Lock() - defer h.mu.Unlock() - if err != nil { - h.Logf("[mesh-delivery] the merge turn on %s cannot be written: %v; asked again in %s", repo, err, askAgainAfter) - h.again("a merge turn, written") - return - } - h.turnsHeld[repo] = t - h.Poke() - }) - default: - out = append(out, func() { - err := h.Turns.Delete(repo) - h.mu.Lock() - defer h.mu.Unlock() - if err != nil { - h.Logf("[mesh-delivery] the merge turn on %s cannot be deleted: %v; asked again in %s", repo, err, askAgainAfter) - h.again("a merge turn, deleted") - return - } - delete(h.turnsHeld, repo) - h.Logf("[mesh-delivery] the merge turn on %s is deleted: no approved merge waits on it, so the checker may start a check there", repo) - h.Poke() - }) - } - } - return out -} - -// sameTurnValue says whether two turns hold the same value, the version aside. -func sameTurnValue(a, b Turn) bool { - a.V, b.V = 0, 0 - return SameTurn(a, b) -} - -// turnLines are the stalled lines of approved merges held past turnBound by another delivery's check (ADR 0548 §4). Under the -// lock. -func (h *Holder) turnLines(now time.Time) []StalledLine { - var out []StalledLine - for _, d := range h.deliveries { - r := d.Ready - if r == nil || r.HeldSince == nil || r.Merged != nil || d.MergedAs != "" || d.State.Final() || now.Sub(*r.HeldSince) < turnBound { - continue - } - words := r.Waits - if words == "" { - words = "approved; waits for the check " + r.HeldFor - } - out = append(out, StalledLine{ID: d.ID, Number: d.Number, State: d.State, For: now.Sub(*r.HeldSince).Round(time.Second).String(), - Bound: turnBound.String(), - H2: "none: the check it waits for ends, or the operator hands the merge now with delivery_hand_now — the operator's", - Says: fmt.Sprintf("%s#%d at %s is %s, held since %s; the step that check is at is read with the checker's checker_status", - d.Repository, d.Number, shortOf(r.Head, 12), words, r.HeldSince.UTC().Format(time.RFC3339))}) - } - return out -} - -// TurnLine is one repository's turn as delivery_turns reads it: the turn as the gate holds it and, for each delivery waiting, -// the check it waits for in words. -type TurnLine struct { - Repository string `json:"repository"` - Turn Turn `json:"turn"` - Waits map[string]any `json:"waits,omitempty"` -} - -// TurnsRead is the tool delivery_turns: every turn the gate holds, with the check each waiting delivery waits for, since when, -// and whether the merge was handed now on the operator's word. -func (h *Holder) TurnsRead() map[string]any { - h.mu.Lock() - defer h.mu.Unlock() - want := map[string]Turn{} - if h.checkGates() { - want = h.wantedTurns() // as turnWrites: while the check gates no merge, no turn is written by design - } - repos := make([]string, 0, len(want)) - for repo := range want { - repos = append(repos, repo) - } - sort.Strings(repos) - lines := []TurnLine{} - for _, repo := range repos { - l := TurnLine{Repository: repo, Turn: want[repo], Waits: map[string]any{}} - for _, w := range want[repo].Waiting { - says := "nothing: no check of other heads runs on " + repo - if w.WaitsFor != "" { - says = "the check " + w.WaitsFor - if !w.WaitsSince.IsZero() { - says += ", running since " + w.WaitsSince.Format(time.RFC3339) - } - } - l.Waits[w.Group] = says - } - lines = append(lines, l) - } - out := map[string]any{"turns": lines, "check-gates-merge": h.checkGates(), "bound": turnBound.String()} - if !h.checkGates() { - out["why-none"] = "the check gates no merge (check-gates-merge false): no merge waits for a check, so no turn is written" - } - if h.Turns != nil { - out["written"] = h.turnsLoaded - var unwritten []string - for _, repo := range repos { - if held, ok := h.turnsHeld[repo]; !ok || !sameTurnValue(held, want[repo]) { - unwritten = append(unwritten, repo) - } - } - for repo := range h.turnsHeld { - if _, ok := want[repo]; !ok { - unwritten = append(unwritten, repo+" (to delete)") - } - } - sort.Strings(unwritten) - if len(unwritten) > 0 { - out["not-yet-written"] = unwritten - } - } - return out -} - -// HandNow is the tool delivery_hand_now (ADR 0548 §4): on the operator's word an approved merge held for another delivery's -// check is handed at once; that check runs on and is carried to the new trunk after the merge. Refused for a delivery that is -// not an approved merge waiting, naming why. -func (h *Holder) HandNow(id, why string) (string, error) { - h.mu.Lock() - defer h.mu.Unlock() - d := h.deliveries[id] - if d == nil { - return "", fmt.Errorf("no delivery %q", id) - } - if strings.TrimSpace(why) == "" { - return "", fmt.Errorf("say why: the hand is the operator's word, kept on the delivery") - } - am, ok := h.mergeOfDelivery(d) - if !ok { - return "", fmt.Errorf("%s is not an approved merge waiting: nothing to hand (its state is %s)", id, d.State) - } - if am.run.HeldSince == nil { - return "", fmt.Errorf("%s waits for no check of another delivery: %s", id, firstOf(am.run.Waits, "it is handed at its turn")) - } - am.run.HandNow = why - h.keep(am.owner) - h.mergeAgainNow(d) - h.Logf("[mesh-delivery] %s is handed now on the operator's word, past the check %s it waited for, which is carried after the merge: %s", - id, am.run.HeldFor, why) - return fmt.Sprintf("%s is handed to the merge at the next look; the check %s it waited for runs on and is carried to the new trunk", id, am.run.HeldFor), nil -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go index 80132df1d..20b00411e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "reflect" - "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" @@ -137,63 +136,3 @@ func SameAsk(a, b Ask) bool { a.Ask, b.Ask = nil, nil return reflect.DeepEqual(a, b) && reflect.DeepEqual(x, y) } - -// Turn is the state turns' value: the approved deliveries waiting to merge in one repository, in the order of the Approve. -type Turn struct { - V int - Repository string - Waiting []TurnWait -} - -// TurnWait is one approved delivery in a turn; WaitsFor, WaitsSince and Step say the check of another delivery it waits on. -type TurnWait struct { - Group string - Heads string - Ask string - Approved time.Time - Since time.Time - WaitsFor string - WaitsSince time.Time - Step string -} - -func (t Turn) stored() seat.TurnsByRepositoryValue { - out := seat.TurnsByRepositoryValue{Repository: t.Repository, V: int64(t.V), Waiting: []seat.TurnsByRepositoryValueWaitingItem{}} - for _, w := range t.Waiting { - item := seat.TurnsByRepositoryValueWaitingItem{Approved: w.Approved, Ask: w.Ask, Group: w.Group, Heads: w.Heads, - Since: w.Since, Step: w.Step, WaitsFor: w.WaitsFor} - if !w.WaitsSince.IsZero() { - since := w.WaitsSince - item.WaitsSince = &since - } - out.Waiting = append(out.Waiting, item) - } - return out -} - -func turnFromSeat(s seat.TurnsByRepositoryValue) Turn { - out := Turn{V: int(s.V), Repository: s.Repository} - for _, w := range s.Waiting { - item := TurnWait{Group: w.Group, Heads: w.Heads, Ask: w.Ask, Approved: w.Approved, Since: w.Since, WaitsFor: w.WaitsFor, Step: w.Step} - if w.WaitsSince != nil { - item.WaitsSince = *w.WaitsSince - } - out.Waiting = append(out.Waiting, item) - } - return out -} - -// EncodeTurn is t as the value of a turns key. -func EncodeTurn(t Turn) ([]byte, error) { return mdc.EncodeTurn(t.stored()) } - -// DecodeTurn reads the value of a turns key. -func DecodeTurn(data []byte) (Turn, error) { - s, err := mdc.DecodeTurn(data) - if err != nil { - return Turn{}, err - } - return turnFromSeat(s), nil -} - -// SameTurn says whether two turns are the same value. -func SameTurn(a, b Turn) bool { return mdc.SameTurn(a.stored(), b.stored()) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values_test.go index abb4253d7..3187afaf9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values_test.go @@ -4,7 +4,6 @@ import ( "encoding/json" "reflect" "testing" - "time" mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" ) @@ -12,8 +11,6 @@ import ( // What go/delivery wrote into the seat's states, key order and all, must decode as it did and encode to the same // content (ADR 0634 §2): the gate's working values are converted to the seat's types at the edge. func TestStoredValuesOfTheEarlierLayoutDecodeUnchanged(t *testing.T) { - approved := time.Date(2026, 11, 5, 10, 0, 0, 0, time.UTC) - since := time.Date(2026, 11, 5, 10, 5, 0, 0, time.UTC) head := mdc.Head{Repository: "novox/app", Number: 7, Head: "0123456789abcdef0123456789abcdef01234567"} rows := []struct { name string @@ -32,13 +29,6 @@ func TestStoredValuesOfTheEarlierLayoutDecodeUnchanged(t *testing.T) { {"ask keeps the router's ask as bytes", `{"v":1,"group":"feat/x","heads":[{"repository":"novox/app","number":7,"head":"` + head.Head + `"}],"ask":{"id":"a","options":["Approve"]},"asking":2}`, func(s string) (any, error) { return DecodeAsk([]byte(s)) }, func(v any) ([]byte, error) { return EncodeAsk(v.(Ask)) }, Ask{V: 1, Group: "feat/x", Heads: []mdc.Head{head}, Ask: json.RawMessage(`{"id":"a","options":["Approve"]}`), Asking: 2}}, - {"turn without and with a check waited for", `{"v":1,"repository":"novox/app","waiting":[` + - `{"group":"feat/x","heads":"0123456789ab","ask":"merge.x","approved":"2026-11-05T10:00:00Z","since":"2026-11-05T10:00:00Z"},` + - `{"group":"feat/y","heads":"ba9876543210","approved":"2026-11-05T10:00:00Z","since":"2026-11-05T10:00:00Z","waits-for":"feat/x.0123456789ab","waits-since":"2026-11-05T10:05:00Z","step":"unit"}]}`, - func(s string) (any, error) { return DecodeTurn([]byte(s)) }, func(v any) ([]byte, error) { return EncodeTurn(v.(Turn)) }, - Turn{V: 1, Repository: "novox/app", Waiting: []TurnWait{ - {Group: "feat/x", Heads: "0123456789ab", Ask: "merge.x", Approved: approved, Since: approved}, - {Group: "feat/y", Heads: "ba9876543210", Approved: approved, Since: approved, WaitsFor: "feat/x.0123456789ab", WaitsSince: since, Step: "unit"}}}}, {"source", `{"v":1,"group":"feat/x","heads":{"novox/app":"c0ffee"}}`, func(s string) (any, error) { return mdc.DecodeSource([]byte(s)) }, func(v any) ([]byte, error) { return mdc.EncodeSource(v.(Source)) }, Source{V: 1, Group: "feat/x", Heads: map[string]string{"novox/app": "c0ffee"}}}, @@ -74,8 +64,5 @@ func sameValue(a, b any) bool { if x, ok := a.(Ask); ok { return SameAsk(x, b.(Ask)) } - if x, ok := a.(Turn); ok { - return SameTurn(x, b.(Turn)) && reflect.DeepEqual(len(x.Waiting), len(b.(Turn).Waiting)) && x.Waiting[len(x.Waiting)-1].WaitsSince.Equal(b.(Turn).Waiting[len(x.Waiting)-1].WaitsSince) - } return reflect.DeepEqual(a, b) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index ff854fe20..18378d182 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -84,7 +84,7 @@ func (h *Holder) Show(id string) (any, error) { h.mu.Lock() defer h.mu.Unlock() if d := h.deliveries[id]; d != nil { - out := map[string]any{"delivery": d, "waits": h.waitsFor(d), "check-gates-merge": h.checkGates()} + out := map[string]any{"delivery": d, "waits": h.waitsFor(d)} if m := h.machines(d); m != "" { out["machines"] = m } @@ -97,7 +97,7 @@ func (h *Holder) Show(id string) (any, error) { if g := h.groups[id]; g != nil { members := h.membersOf(g) state, why := GroupState(g, members) - return map[string]any{"group": g, "state": state, "why": why, "members": members, "check-gates-merge": h.checkGates()}, nil + return map[string]any{"group": g, "state": state, "why": why, "members": members}, nil } return nil, fmt.Errorf("no delivery or group %q: `deliveries` lists them", id) } @@ -246,7 +246,6 @@ func (h *Holder) Stalled() []StalledLine { } out = append(out, h.withChecks(h.uncheckedLines(now))...) out = append(out, h.readyLines(now)...) - out = append(out, h.turnLines(now)...) // an approved merge held past its bound by another delivery's check (ADR 0548 §4) out = append(out, h.groupLines(now)...) out = append(out, h.owedLines(now)...) // an act owed the forge with no outcome past its bound (W11.17) sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) @@ -316,7 +315,7 @@ func (h *Holder) Status(listening string) map[string]any { byState[string(s)] = n } out := map[string]any{"deliveries": byState, "groups": len(h.groups), "owed": owed, "unkept": len(h.dirty), - "events": listening, "started": h.started.Format(time.RFC3339), "check-gates-merge": h.checkGates()} + "events": listening, "started": h.started.Format(time.RFC3339)} // The owed effects tried since this owner started (novox/hq issue 407): done, failed and to be tried again, // or given up, and when the last was tried. A count since the start, said with that start: two readings in // one run give a window's tries. diff --git a/modules/mesh-delivery-gate/files/settings.json b/modules/mesh-delivery-gate/files/settings.json index e164ea53c..8c179a898 100644 --- a/modules/mesh-delivery-gate/files/settings.json +++ b/modules/mesh-delivery-gate/files/settings.json @@ -1,6 +1,5 @@ { "refusal-stall-floor-minutes": 10, "author-new-head-minutes": 120, - "check-gates-merge": true, "record-retention": 90 } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index cf20565ca..e462c942a 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -316,11 +316,7 @@ "name": "source", "history": 64 }, - "conditions", - { - "name": "turns", - "history": 5 - } + "conditions" ], "reads": [ "mesh-delivery-teller.answers", @@ -328,10 +324,8 @@ "mesh-delivery-view.reach" ], "tools": [ - "delivery_hand_now", "delivery_merge_again", "delivery_status", - "delivery_turns", "gate_consumer", "gate_record", "gate_record_end", @@ -354,12 +348,6 @@ "trusted": false, "default": 120 }, - "check-gates-merge": { - "type": "boolean", - "description": "Whether mesh/check gates the merge. True (the default): the operator's Approve is asked, and a merge handed, only on a passing verdict judged against the trunk's head, and a trunk's move puts a verdict out of date (ADR 0447, ADR 0505 §4). False (the operator, 2026-10-25 22:56: \"run the check locally only, but not as a gate, it loses us too much time\"; developers run the unit and integration levels locally before a pull request): a ready pull request (no review line is waited for, ADR 0659) has its Approve is asked once the check at its heads (a group: of all its heads together) has a verdict, a pass or a failure said in the ask, or after 60 minutes with none, said so (the operator, 2026-10-27 23:54: \"if you don't want me to approve it, don't send the approval request\"); an approved merge waits for no verdict; no merge is refused for a check judged on an older trunk; a later verdict is shown on the pull request and the delivery, and asks a pending Approve again in its words; a group still merges all its members in one pass. Set it false together with the gitea module's check-gates-merge, which lets its merge tool merge an approved head without a passing mesh/check.", - "trusted": false, - "default": true - }, "record-retention": { "type": "number", "description": "days a delivery record is kept from when it opened (design 47 *The delivery record*): ninety covers every trace asked of a past delivery so far; when a record is removed at its retention, the gate says freed for each graph revision it pinned. Nothing else is removed to make room.", @@ -385,7 +373,6 @@ "takes-config": [ "refusal-stall-floor-minutes", "author-new-head-minutes", - "check-gates-merge", "record-retention" ], "merge": "json", -- 2.54.0 From 8ee28d08c33c8e8e3fe834a4a1205af0f8016954 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:44:17 +0100 Subject: [PATCH 054/243] The gate says its records' ledger filling, full and room, and holds a merge whose record finds no room (milestone 47/G, W11.20) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Design 47: a store filling is said before it is full. The gate reads its ledger's fill through the tool runner's mesh/ledger.fill, says filling of the seat mesh-delivery when a record takes the ledger past 80 % of its cap, refuses the next record when none fits and says full naming it, and keeps the merge waiting for room on its delivery (held_full), so a restarted gate judges alike. Once the ledger is below 80 % again, after the retention's removals, it says room and hands the merge. gate_records says the fill against its cap, or why it is unread. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 061be00d.) Builds: design 47 G §181 (GATE12) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/main.go | 2 +- .../cmd/mesh-delivery-gate/merges.go | 8 + .../cmd/mesh-delivery-gate/ready.go | 5 +- .../cmd/mesh-delivery-gate/record.go | 12 +- .../cmd/mesh-delivery-gate/record_test.go | 125 +++++++++++ .../cmd/mesh-delivery-gate/recordfill.go | 201 ++++++++++++++++++ .../cmd/mesh-delivery-gate/recordtools.go | 6 +- .../cmd/mesh-delivery-gate/switch.go | 29 +-- 8 files changed, 369 insertions(+), 19 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 468c75217..bf165028f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -74,7 +74,7 @@ func main() { } h.init() h.SaySeat = stdio.SeatPublish - h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish} + h.Records = &Records{Ledger: sdkLedger{}, Now: h.Now, Retention: h.recordRetention, Say: stdio.SeatPublish, Logf: logf} h.Merges = recordedMerges{MergeStore: kvMerges{}, records: h.Records, members: h.groupMembers} h.Old = kvOld{} h.Consumer = &StepConsumer{Records: h.Records, Families: stepFamilies, Now: h.Now, Logf: logf} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index 3f32bfb77..e9701a0af 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -291,6 +291,9 @@ func (h *Holder) hand(gid, name, key, target string, reads []memberRead, approve if gid != "" && !h.groupMergeable(gid, target, reads) { return } + if h.waitForRoom(target) { + return + } h.mu.Lock() value, ok := h.mergeValue(name, key, approved, target, mdc.MergeMerging) h.mu.Unlock() @@ -306,6 +309,11 @@ func (h *Holder) hand(gid, name, key, target string, reads []memberRead, approve err = h.Merges.Put(name, value) } if err != nil { + if errors.Is(err, ErrRecordsFull) { + h.mu.Lock() + h.holdFull(target, h.Now()) + h.mu.Unlock() + } h.giveBack(target, err) return } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index f3da1acf1..649f99a27 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -136,7 +136,10 @@ type ReadyRun struct { // CannotWrite is why the operator is not asked: the forge's merging account cannot write a repository the ask covers // (writes.go, issue 1945); CannotWriteSince since when. Said at once as a condition. - CannotWrite string `json:"cannot_write,omitempty"` + CannotWrite string `json:"cannot_write,omitempty"` + // HeldFull is since when its merge waits for room in the delivery records' ledger, which refused its record as full + // (recordfill.go): handed once the ledger is below 80 % of its cap again. + HeldFull *time.Time `json:"held_full,omitempty"` CannotWriteSince *time.Time `json:"cannot_write_since,omitempty"` // NotHanded is why an approved merge that waits for nothing is not handed to the forge bridge (a member of its group that diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 9cd40c3ce..38b81225a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -345,8 +345,11 @@ type Records struct { Now func() time.Time // Retention is how long a record is kept; nil keeps DefaultRecordRetention. Retention func() time.Duration - // Say says the seat mesh-delivery's events of the records: pinned, freed, record-ended (recordevents.go); nil says none. + // Say says the seat mesh-delivery's events of the records: pinned, freed, record-ended (recordevents.go), filling, + // full and room (recordfill.go); nil says none. Say SeatSay + // Logf says what the records could not say; nothing is said when nil. + Logf func(format string, args ...any) // held are the records kept, by instance, read once from the ledger (Current) and kept as the gate writes them: the // gate is their one writer. @@ -475,6 +478,10 @@ func (rs *Records) Open(milestone string, heads map[string]string, forgeMade boo if err := rs.Sweep(now); err != nil { return nil, false, fmt.Errorf("the records past their retention cannot be removed, so no record is opened: %w", err) } + before, read, err := rs.refuseFull(milestone, inst) + if err != nil { + return nil, false, err + } body := map[string]any{"milestone": milestone, "heads": heads, "opened": now, "form": RecordForm} if forgeMade { body["forge-made"] = true @@ -485,6 +492,9 @@ func (rs *Records) Open(milestone string, heads map[string]string, forgeMade boo if _, err := rs.Ledger.Apply(heldInstance(now), OpHold, heldEntry{Instance: inst, Opened: now}); err != nil { return nil, false, fmt.Errorf("the record %s is opened but not listed: %w", inst, err) } + if read { + rs.sayFilling(inst, before) + } r, err = rs.Read(inst) if err == nil && r != nil { rs.keep(r) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 2a04a3214..87efb4b97 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -30,6 +30,15 @@ type ledgerFake struct { seq map[string]uint64 history []string failOn string + // fill is what Fill answers; nil, the ledger says no fill. + fill *buscalls.LedgerFilled +} + +func (f *ledgerFake) Fill() (buscalls.LedgerFilled, error) { + if f.fill == nil { + return buscalls.LedgerFilled{}, errors.New("the runtime answers no mesh/ledger.fill") + } + return *f.fill, nil } func newLedgerFake(t *testing.T) *ledgerFake { @@ -742,3 +751,119 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { } }) } + +// W11.20 (design 47 *The delivery record*: a store filling is said before it is full): the gate reads its ledger's fill, +// says filling when a record takes it past 80 % of its cap, refuses the next record as full and holds its merge, and +// says room and hands the merge once the ledger is below 80 % again; the read tool says the fill. +func TestTheRecordsLedgerSaysFillingFullAndRoom(t *testing.T) { + const capBytes = 10 << 20 + filled := func(bytes uint64) *buscalls.LedgerFilled { + return &buscalls.LedgerFilled{History: buscalls.Fill{Count: 1, Bytes: bytes, MaxBytes: capBytes}, + State: buscalls.Fill{Bytes: 1, MaxBytes: 64 << 20}} + } + heads := map[string]string{"novox/app@111111111111": strings.Repeat("1", 40)} + at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) + type said struct { + subject, id string + body meshdelivery.Fill + } + saying := func(out *[]said) SeatSay { + return func(subject string, body any, id string) error { + f, _ := body.(meshdelivery.Fill) + *out = append(*out, said{subject, id, f}) + return nil + } + } + + t.Run("filling once past 80 %, full refuses the next record, an unread fill refuses nothing", func(t *testing.T) { + l := newLedgerFake(t) + var events []said + rs := &Records{Ledger: l, Now: func() time.Time { return at }, Say: saying(&events)} + l.fill = filled(capBytes * 7 / 10) + r, opened, err := rs.Open("47/G", heads, false) + if err != nil || !opened { + t.Fatalf("open below the mark: %v %v", opened, err) + } + if len(events) != 0 { + t.Fatalf("said %+v below the mark", events) + } + // The record took the ledger past 80 %: said once as filling, with the store, its bytes and its cap. + l.fill = filled(capBytes * 85 / 100) + rs.sayFilling(r.Instance, buscalls.Fill{Bytes: capBytes * 7 / 10, MaxBytes: capBytes}) + if len(events) != 1 || events[0].subject != meshdelivery.SubjectFilling || events[0].body.Store != RecordsLedger || + events[0].body.Cap != capBytes || events[0].body.Bytes != capBytes*85/100 { + t.Fatalf("filling said as %+v", events) + } + // Already past the mark: a next record is opened and says nothing again. + events = nil + if _, opened, err := rs.Open("47/H", heads, false); err != nil || !opened || len(events) != 0 { + t.Fatalf("a record past the mark: %v %v, said %+v", opened, err, events) + } + // No room for one more record: refused, said as full naming what it refuses, nothing opened. + l.fill = filled(capBytes - RecordRoom/2) + n := len(l.history) + if _, opened, err := rs.Open("47/I", heads, false); !errors.Is(err, ErrRecordsFull) || opened || len(l.history) != n { + t.Fatalf("a full ledger opened %v (%v), %d operations", opened, err, len(l.history)-n) + } + if len(events) != 1 || events[0].subject != meshdelivery.SubjectFull || !strings.Contains(events[0].body.Refused, "47/I") { + t.Fatalf("full said as %+v", events) + } + if line, _ := rs.Lines(); !strings.HasPrefix(fmt.Sprint(line["fill"].(map[string]any)["state"]), "full") { + t.Fatalf("gate_records says the fill %v", line["fill"]) + } + // A ledger that says no fill (a runner before the call): the record is opened, and the read tool says why. + l.fill = nil + if _, opened, err := rs.Open("47/J", heads, false); err != nil || !opened { + t.Fatalf("an unread fill refused the record: %v %v", opened, err) + } + if line, _ := rs.Lines(); line["fill"].(map[string]any)["unread"] == nil { + t.Fatalf("gate_records says the unread fill as %v", line["fill"]) + } + }) + + t.Run("a merge refused for a full ledger is held, then handed with room once below 80 %", func(t *testing.T) { + w := approvedGroup(t) + l := newLedgerFake(t) + var events []said + w.h.Records = &Records{Ledger: l, Now: w.h.Now, Say: saying(&events)} + w.h.Merges = recordedMerges{MergeStore: w.merges, records: w.h.Records, members: w.h.groupMembers} + l.fill = filled(capBytes) + allGreen(w) + w.op.answer(w.op.open()[0], OptionApprove) + w.settleAll() + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("a full ledger let %v merge", w.forge.merges) + } + w.h.mu.Lock() + since := w.h.heldFull() + w.h.mu.Unlock() + if since == nil || len(events) == 0 || events[0].subject != meshdelivery.SubjectFull { + t.Fatalf("the merge is not held for room (%v), said %+v", since, events) + } + // Room for a record but past 80 %: still held. + l.fill = filled(capBytes * 85 / 100) + w.later(askAgainAfter + time.Second) + w.settleAll() + if len(w.forge.merges) != 0 { + t.Fatalf("a ledger past 80 %% let %v merge after a full", w.forge.merges) + } + // Below 80 %: room said once, the held mark cleared, the group merged. + l.fill = filled(capBytes / 2) + w.later(askAgainAfter + time.Second) + w.settleAll() + w.settleAll() + rooms := 0 + for _, e := range events { + if e.subject == meshdelivery.SubjectRoom { + rooms++ + } + } + w.h.mu.Lock() + since = w.h.heldFull() + w.h.mu.Unlock() + if rooms != 1 || since != nil || len(w.forge.merges) == 0 { + t.Fatalf("room said %d times, held since %v, merges %v", rooms, since, w.forge.merges) + } + }) +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go new file mode 100644 index 000000000..33d856012 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go @@ -0,0 +1,201 @@ +package main + +// The delivery records' ledger against its cap (design 47 *The delivery record*: "A store filling is said before it is +// full"): the gate reads its ledger's fill through the tool runner (mesh-sdk buscalls.LedgerFill), says `filling` of the +// seat mesh-delivery when a record it opens takes the ledger past 80 % of its cap, refuses the next record when the +// ledger has no room for one and says `full`, and holds that record's merge until the ledger is below 80 % again, when +// it says `room` and hands the merge. Nothing is dropped to make room: only the retention removes a record. The fill +// is read from the bus at each judgement, and which merges wait for room is kept on their deliveries (Ready.HeldFull), +// so a restarted gate judges as the one before it. + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "git.novox.be/novox/mesh-sdk/go/buscalls" +) + +// FillingPercent is the share of its cap past which the ledger is said filling, and below which room is said again. +const FillingPercent = 80 + +// RecordRoom is the room one record needs at most: an operation is at most one value of the bus. +const RecordRoom = 256 << 10 + +// ErrRecordsFull is a record refused because the records' ledger has no room for it. +var ErrRecordsFull = errors.New("the delivery records' ledger is full") + +// fillReader is a ledger that says its fill against its cap. +type fillReader interface { + Fill() (buscalls.LedgerFilled, error) +} + +func (sdkLedger) Fill() (buscalls.LedgerFilled, error) { + out, err := ledgerCall(buscalls.LedgerFill, buscalls.LedgerRequest{Ledger: RecordsLedger}) + if err != nil { + return buscalls.LedgerFilled{}, err + } + var got buscalls.LedgerFilled + if err := json.Unmarshal(out, &got); err != nil { + return got, fmt.Errorf("the ledger's fill is not readable: %w", err) + } + return got, nil +} + +// Fill is the records' ledger's store nearest its cap. +func (rs *Records) Fill() (buscalls.Fill, error) { + fr, ok := rs.Ledger.(fillReader) + if !ok { + return buscalls.Fill{}, errors.New("this gate's ledger says no fill") + } + f, err := fr.Fill() + if err != nil { + return buscalls.Fill{}, fmt.Errorf("the records' ledger cannot say its fill: %w", err) + } + return f.Fullest(), nil +} + +// pastFilling is a fill at or past FillingPercent of its cap. +func pastFilling(f buscalls.Fill) bool { + return f.MaxBytes > 0 && f.Bytes*100 >= f.MaxBytes*FillingPercent +} + +// noRoom is a fill that leaves no room for one more record. +func noRoom(f buscalls.Fill) bool { return f.MaxBytes > 0 && f.Bytes+RecordRoom > f.MaxBytes } + +// sayFill says filling, full or room of the seat under an id of the event and what it is about, so said again it is +// heard once. +func (rs *Records) sayFill(subject, about string, f buscalls.Fill, refused string) { + at := rs.now().UTC() + body := meshdelivery.Fill{Store: RecordsLedger, Bytes: int64(f.Bytes), Cap: int64(f.MaxBytes), Refused: refused, At: &at} + if err := rs.say(subject, body, about); err != nil { + rs.logf("[mesh-delivery] %s of the records' ledger (%d of %d bytes) is not said: %v", subject, f.Bytes, f.MaxBytes, err) + } +} + +func (rs *Records) logf(format string, args ...any) { + if rs.Logf != nil { + rs.Logf(format, args...) + } +} + +// refuseFull says full and answers ErrRecordsFull when the ledger has no room for the record inst; a fill it cannot +// read refuses nothing (the read tools say why it is not read). +func (rs *Records) refuseFull(milestone, inst string) (before buscalls.Fill, read bool, err error) { + before, fillErr := rs.Fill() + if fillErr != nil { + rs.logf("[mesh-delivery] the record %s is opened with its ledger's fill unread: %v", inst, fillErr) + return before, false, nil + } + if !noRoom(before) { + return before, true, nil + } + rs.sayFill(meshdelivery.SubjectFull, "full-"+inst, before, + "the next record, "+inst+" of "+milestone+"; its merge is held until room is said") + return before, true, fmt.Errorf("%w (%d of %d bytes): the record %s of %s is not opened, and its merge is held until "+ + "room is said: the setting record-retention or a larger cap is the operator's", ErrRecordsFull, before.Bytes, before.MaxBytes, inst, milestone) +} + +// sayFilling says filling when the record just opened took the ledger past FillingPercent of its cap. +func (rs *Records) sayFilling(inst string, before buscalls.Fill) { + if pastFilling(before) { + return + } + after, err := rs.Fill() + if err == nil && pastFilling(after) { + rs.sayFill(meshdelivery.SubjectFilling, "filling-"+inst, after, "") + } +} + +// RoomAgain is whether a merge held for a full ledger since then may be handed now: the records past their retention +// removed first, the ledger below FillingPercent of its cap, said once as room. A fill it cannot read gives no room. +func (rs *Records) RoomAgain(since time.Time) (bool, error) { + if err := rs.Sweep(rs.now()); err != nil { + return false, err + } + f, err := rs.Fill() + if err != nil { + return false, err + } + if pastFilling(f) { + return false, fmt.Errorf("the records' ledger holds %d of %d bytes, past %d %% of its cap: the merge waits for room", + f.Bytes, f.MaxBytes, FillingPercent) + } + rs.sayFill(meshdelivery.SubjectRoom, "room-"+since.UTC().Format(time.RFC3339Nano), f, "") + return true, nil +} + +// fillLine is the ledger's fill as the read tools say it. +func (rs *Records) fillLine() map[string]any { + f, err := rs.Fill() + if err != nil { + return map[string]any{"store": RecordsLedger, "unread": err.Error()} + } + line := map[string]any{"store": RecordsLedger, "bytes": f.Bytes, "cap": f.MaxBytes, "count": f.Count} + switch { + case noRoom(f): + line["state"] = "full: the next record is refused and its merge held" + case pastFilling(f): + line["state"] = fmt.Sprintf("filling: past %d %% of its cap", FillingPercent) + default: + line["state"] = "room" + } + return line +} + +// heldFull is the earliest time a merge was held for a full ledger, of every delivery that waits for room. Under the +// lock. +func (h *Holder) heldFull() *time.Time { + var since *time.Time + for _, d := range h.deliveries { + if r := d.Ready; r != nil && r.HeldFull != nil && (since == nil || r.HeldFull.Before(*since)) { + since = r.HeldFull + } + } + return since +} + +// roomFor clears every delivery's wait for room. Under the lock. +func (h *Holder) roomFor() { + for _, d := range h.deliveries { + if r := d.Ready; r != nil && r.HeldFull != nil { + r.HeldFull = nil + h.keep(d) + } + } +} + +// holdFull keeps that target's merge waits for room. Under the lock. +func (h *Holder) holdFull(target string, now time.Time) { + if d := h.deliveries[target]; d != nil && d.Ready != nil && d.Ready.HeldFull == nil { + at := now + d.Ready.HeldFull = &at + h.keep(d) + } +} + +// waitForRoom is whether a merge must wait because the records' ledger was full: while any merge waits for room, none +// is handed until RoomAgain gives it. +func (h *Holder) waitForRoom(target string) bool { + if h.Records == nil { + return false + } + h.mu.Lock() + since := h.heldFull() + h.mu.Unlock() + if since == nil { + return false + } + ok, err := h.Records.RoomAgain(*since) + if !ok { + h.giveBack(target, err) + return true + } + h.mu.Lock() + h.roomFor() + h.mu.Unlock() + h.Logf("[mesh-delivery] the records' ledger has room again: the merges held for it are handed") + return false +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 72faf22d1..4941ed254 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -105,7 +105,8 @@ func (rs *Records) Lines() (map[string]any, error) { } lines = append(lines, l) } - return map[string]any{"records": lines, "open": open, "kept": len(lines), "retention-days": int(rs.retention().Hours() / 24)}, nil + return map[string]any{"records": lines, "open": open, "kept": len(lines), "retention-days": int(rs.retention().Hours() / 24), + "fill": rs.fillLine()}, nil } // EndByHand ends one open record by hand, with the reason, recorded in the hand-act log first: nothing is ended when the @@ -161,7 +162,8 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool { }}, {Name: "gate_records", Description: "Every delivery record kept, oldest first: its milestone, when it opened and ended, its entries, " + - "outcomes, nodes away and pins, its size against the bus's value limit, and while open what it waits for.", + "outcomes, nodes away and pins, its size against the bus's value limit, and while open what it waits for; and " + + "the records' ledger's fill against its cap (filling past 80 %, full when no record fits, else room).", Run: func(map[string]any) (any, error) { if h.Records == nil { return none() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go index 8b4c14264..0a2ff4e52 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/switch.go @@ -38,22 +38,23 @@ type recordedMerges struct { members func(group string) []string } +// Put writes the group's merge into its delivery record first, so a ledger with no room refuses it before the merge is +// handed to the forge bridge (G §181), then into the state merges the bridge follows. func (m recordedMerges) Put(group string, v Merge) error { - if err := m.MergeStore.Put(group, v); err != nil { - return err + if m.records != nil && len(v.Approved) > 0 { + r, _, err := m.records.Open(group, v.Approved, false) + if err != nil { + return fmt.Errorf("the merge of %s is handed but not in its delivery record: %w", group, err) + } + var members []string + if m.members != nil { + members = m.members(group) + } + if err := m.records.SetGroup(r, recordGroupOf(v, members)); err != nil { + return err + } } - if m.records == nil || len(v.Approved) == 0 { - return nil - } - r, _, err := m.records.Open(group, v.Approved, false) - if err != nil { - return fmt.Errorf("the merge of %s is handed but not in its delivery record: %w", group, err) - } - var members []string - if m.members != nil { - members = m.members(group) - } - return m.records.SetGroup(r, recordGroupOf(v, members)) + return m.MergeStore.Put(group, v) } // switchGroup is one group in flight as the switch finds it: its milestone, its heads and its members. -- 2.54.0 From bf6bb3d09491c1b790fba74699c8f0c2cd293bd6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:44:17 +0100 Subject: [PATCH 055/243] The gate names the issues linked in the issue tracker and the forge's author, and reads no description (milestone 47/G, W11.5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0659 §4 kept three description reads that shape words only until they move to the bus: the issue numbers the ask names, the author a stalled line names and the notice that after: lines are not read. The ask now names the issues linked to each member's pull request, read once per head through the seat issue-tracker's list (a tracker that does not answer is said, never waited on); the author is the forge's pull request record's, from the seat git's events; the after: notice is gone. The gate's forms of a pull request and a delivery hold no description field. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from ddd93a78.) Builds: design 47 G §50, §143 (GATE15) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-delivery-gate/README.md | 5 +- .../cmd/mesh-delivery-gate/adr505_test.go | 20 ++-- .../cmd/mesh-delivery-gate/approval.go | 25 ++--- .../cmd/mesh-delivery-gate/bus_test.go | 2 - .../cmd/mesh-delivery-gate/events_test.go | 83 ++++---------- .../cmd/mesh-delivery-gate/holder.go | 45 +++----- .../cmd/mesh-delivery-gate/holder_test.go | 71 ++++++++---- .../cmd/mesh-delivery-gate/issue1114_test.go | 3 - .../cmd/mesh-delivery-gate/issue1726_test.go | 6 +- .../cmd/mesh-delivery-gate/issuelinks.go | 106 ++++++++++++++++++ .../cmd/mesh-delivery-gate/main.go | 1 + .../cmd/mesh-delivery-gate/manifest_test.go | 3 +- .../cmd/mesh-delivery-gate/model.go | 55 +++------ .../mesh-delivery-gate/onestatus_bus_test.go | 1 - .../cmd/mesh-delivery-gate/ready.go | 23 ---- modules/mesh-delivery-gate/module.json | 3 +- 16 files changed, 234 insertions(+), 218 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issuelinks.go diff --git a/modules/mesh-delivery-gate/README.md b/modules/mesh-delivery-gate/README.md index a7ed14ada..726019984 100644 --- a/modules/mesh-delivery-gate/README.md +++ b/modules/mesh-delivery-gate/README.md @@ -33,8 +33,9 @@ names every refusal and rejection at once, and the pull request is told at once; `refusal-stall-floor-minutes`, 10). Every other refusal is stalled at once. - a delivery that needs its author's new head (its check failed after its Approve: a conflict, a failed check, a rejected group) is stalled only once no new head came for the setting `author-new-head-minutes` (120). Its line says - `needs: new-head` and names its author: the branch, and the session a `Pushed-by: ` line in the pull - request's description names. + `needs: new-head` and names its author: the branch, and who opened the pull request as the forge's record says + (the seat git's events). The ask names the issues linked to each pull request in the issue tracker (the seat + issue-tracker's `list`); nothing is read from a description. ## A merge the forge account cannot carry out is not asked (issue 1945) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go index 3848448a7..b3c0355a0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go @@ -131,15 +131,13 @@ func TestAnyOtherRefusalIsStalledAtOnce(t *testing.T) { } // rejectedBeforeItsAnswer is a delivery approved and then rejected: its check failed after the ask (as in events_test.go), -// its pull request's description naming who pushed it when pushedBy is given. -func rejectedBeforeItsAnswer(t *testing.T, pushedBy string) (*world, string) { +// its pull request opened by author as the forge's record says, when given. +func rejectedBeforeItsAnswer(t *testing.T, author string) (*world, string) { t.Helper() w := newWorld(t) w.op.auto = false p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") - if pushedBy != "" { - p.Body += "\nPushed-by: " + pushedBy - } + p.Author = author w.h.PullUpdated(p) w.settleAll() id := IDOf("novox/app", readyHead) @@ -170,7 +168,7 @@ func rejectedBeforeItsAnswer(t *testing.T, pushedBy string) (*world, string) { // with no new head, in words naming its author (its branch, and the session that pushed it) and needing a new head — // never "stop it", never "the operator's". func TestARejectedDeliveryIsItsAuthorsForTwoHours(t *testing.T) { - w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries@shanks") + w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries") said := false for _, c := range w.forge.comments { said = said || strings.Contains(c, "not merged") @@ -188,7 +186,7 @@ func TestARejectedDeliveryIsItsAuthorsForTwoHours(t *testing.T) { t.Fatalf("not the operator's after two hours with no new head: %+v", got) } l := got[0] - if l.Needs != NeedsNewHead || l.Author != "the author of feat/x (quiet-deliveries on shanks)" || l.Branch != "feat/x" { + if l.Needs != NeedsNewHead || l.Author != "the author of feat/x (quiet-deliveries)" || l.Branch != "feat/x" { t.Fatalf("the line does not name its author and the new head it needs: %+v", l) } for _, never := range []string{"operator's", "stop it"} { @@ -196,13 +194,13 @@ func TestARejectedDeliveryIsItsAuthorsForTwoHours(t *testing.T) { t.Fatalf("the line says %q: %+v", never, l) } } - if !strings.Contains(l.Says, "needs a new head from the author of feat/x (quiet-deliveries on shanks)") { + if !strings.Contains(l.Says, "needs a new head from the author of feat/x (quiet-deliveries)") { t.Fatalf("the words do not say a new head is needed from its author: %q", l.Says) } } -// ADR 0505 §5: with no Pushed-by line, the branch alone names its author. -func TestARejectedDeliveryWithNoPushedByLineIsNamedByItsBranch(t *testing.T) { +// ADR 0505 §5: with no author in the forge's record, the branch alone names its author. +func TestARejectedDeliveryWithNoAuthorIsNamedByItsBranch(t *testing.T) { w, id := rejectedBeforeItsAnswer(t, "") w.later(2*time.Hour + time.Minute) if got := linesFor(w, id); len(got) != 1 || got[0].Author != "the author of feat/x" { @@ -212,7 +210,7 @@ func TestARejectedDeliveryWithNoPushedByLineIsNamedByItsBranch(t *testing.T) { // ADR 0505 §5: a new head within the two hours ends it: the rejected head is never the operator's. func TestANewHeadWithinTwoHoursKeepsTheRejectedHeadFromTheOperator(t *testing.T) { - w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries@shanks") + w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries") w.later(time.Hour) w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go")) w.settleAll() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index f23058fa5..e322d10c4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -98,26 +98,14 @@ func phone(s string, n int) string { return s } -var issueNamed = regexp.MustCompile(`(?i)\bissues?\s*#?(\d+)`) - -// IssuesIn reads the issues a pull request's title and body name ("issue 948"), ascending, once each. -func IssuesIn(text string) []int { - seen := map[int]bool{} - var out []int - for _, m := range issueNamed.FindAllStringSubmatch(text, -1) { - if n, err := strconv.Atoi(m[1]); err == nil && !seen[n] { - seen[n] = true - out = append(out, n) - } - } - sort.Ints(out) - return out -} - +// issuesSaid is the issues linked to the deliveries' pull requests in the issue tracker, as the ask names them. func issuesSaid(ds []*Delivery) string { seen := map[int]bool{} var out []string for _, d := range ds { + if d.IssuesUnread != "" { + out = append(out, pullSaid(d)+"'s unread (the issue tracker did not answer)") + } for _, n := range d.Issues { if !seen[n] { seen[n] = true @@ -126,7 +114,7 @@ func issuesSaid(ds []*Delivery) string { } } if len(out) == 0 { - return "none named" + return "none linked" } return strings.Join(out, ", ") } @@ -440,6 +428,9 @@ func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { if waits := h.writeWaits(d, members, now); waits != "" { return waits, nil // asked only once the forge account can carry the merge out (writes.go, issue 1945) } + if read := h.issuesToRead(members); read != nil { + return "the issues linked to its pull request are read from the issue tracker", read + } group := "" if len(members) > 1 { group = d.Group diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index d52c1e910..3ebee8272 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -562,7 +562,6 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB announced := pr("novox/app", 7, head, "fix/x", "src/a.go") announced.BaseSHA = "a1a1a1a1a1a1" announced.Title = "Fix the apostrophe (issue 975)" - announced.Body = "The description, with no review line yet." b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(announced)) rt.until("the pull request's key to hold its head", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") @@ -754,7 +753,6 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe for _, p := range []PullEvent{pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go"), pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")} { p.BaseSHA = "a1a1a1a1a1a1" - p.Body = "Reviewed-by: lead@shanks at " + p.HeadSHA b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(p)) } // Both members pass at the checker, whose one verdict covers both heads: the group is asked once, no composed check is asked diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go index c71ad614a..7e3192cdb 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go @@ -2,6 +2,7 @@ package main import ( gitseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/git" + "reflect" "encoding/json" "strings" @@ -120,29 +121,31 @@ func (*askedAgainError) Error() string { return "the forge refused gitea_merge_pull_request: novox/app#7 is not merged: its verdict was judged against trunk aaaaaaaaaaaa, and main is now bbbbbbbbbbbb. The check of head cccccccccccc merged into trunk bbbbbbbbbbbb was asked; took 3s. Ask again at cccccccccccc once both statuses pass. Nothing was merged" } -// The gate reads nothing from a pull request's description (novox/hq ADR 0493 §6, ADR 0659, which ends ADR 0496): a pull request whose description -// holds no review line, or one naming another head, is asked once its other waits are met, and the ask names no review. +// The gate reads nothing from a pull request's description (novox/hq ADR 0493 §6, ADR 0659, which ends ADR 0496): its form +// of a pull request and its delivery hold no description, no review line, no pushed-by and no after: line, so no wait can +// read one; a pull request is asked once its other waits are met, and the ask names no review. func TestAPullRequestIsAskedWhateverItsDescriptionSays(t *testing.T) { - for _, c := range []struct{ name, body string }{ - {"no line", "Fixes issue 7."}, - {"another head", "Reviewed-by: lead@shanks at " + pushed}, - {"empty", ""}, - } { - t.Run(c.name, func(t *testing.T) { - w := newWorld(t) - w.op.auto = false - p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") - p.Body = c.body - w.h.PullUpdated(p) - hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) - open := w.op.open() - if len(open) != 1 { - t.Fatalf("an unreviewed pull request with its other waits met was not asked: %d", len(open)) + for _, form := range []any{PullEvent{}, Delivery{}} { + ty := reflect.TypeOf(form) + for i := 0; i < ty.NumField(); i++ { + name := strings.ToLower(ty.Field(i).Name) + for _, never := range []string{"body", "description", "pushed", "after", "review"} { + if strings.Contains(name, never) { + t.Errorf("%s holds the field %s, a description's", ty.Name(), ty.Field(i).Name) + } } - if e := w.op.ask(open[0]).Explanation; strings.Contains(strings.ToLower(e), "review") { - t.Fatalf("the ask names a review:\n%s", e) - } - }) + } + } + w := newWorld(t) + w.op.auto = false + w.h.PullUpdated(pr("novox/app", 7, readyHead, "feat/x", "src/a.go")) + hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) + open := w.op.open() + if len(open) != 1 { + t.Fatalf("an unreviewed pull request with its other waits met was not asked: %d", len(open)) + } + if e := w.op.ask(open[0]).Explanation; strings.Contains(strings.ToLower(e), "review") { + t.Fatalf("the ask names a review:\n%s", e) } } @@ -170,41 +173,3 @@ func TestAPushAfterTheAskEndsItAndSaysSoOnceOnThePullRequest(t *testing.T) { t.Fatalf("said more than once: %v", w.forge.comments) } } - -// ADR 0659 clause 6: no wait of ready.go reads a description field. A delivery and a group whose descriptions differ only -// (no line, a Reviewed-by line naming the head, another head, garbage, an after: and a Pushed-by line) answer the same -// readyWaits, ungatedWaits and group state. -func TestNoWaitReadsADescription(t *testing.T) { - heads := map[string]string{"novox/a": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "novox/b": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"} - bodies := []string{"", "Fixes issue 7.", "Reviewed-by: lead@shanks at %s", "Reviewed-by: lead@shanks at " + pushed, - "\x00garbage: Reviewed-by pending\nafter: novox/zzz\nPushed-by: x@y"} - var first []string - for i, body := range bodies { - w := newWorld(t) - w.op.auto = false - for n, repo := range []string{"novox/a", "novox/b"} { - p := pr(repo, n+1, heads[repo], "feat/y", "x") - p.Body = strings.ReplaceAll(body, "%s", heads[repo]) - w.h.PullUpdated(p) - w.h.Checked(verdict(repo, n+1, heads[repo], "pass")) - } - w.h.Checked(CheckedEvent{Group: "feat/y", ID: "check-1", Verdict: "pass", Summary: "compose"}) - w.settleAll() - g := w.h.groups["feat/y"] - state, why := GroupState(g, w.h.membersOf(g)) - ds := []*Delivery{w.delivery(IDOf("novox/a", heads["novox/a"])), w.delivery(IDOf("novox/b", heads["novox/b"]))} - w.h.mu.Lock() - got := []string{state, why} - for _, d := range ds { - got = append(got, w.h.askWaits(d), w.h.approvedWaits(d)) - } - w.h.mu.Unlock() - if i == 0 { - first = got - continue - } - if strings.Join(got, "|") != strings.Join(first, "|") { - t.Fatalf("description %q changes the waits:\n got %q\nwant %q", body, got, first) - } - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go index a427ebdad..d2f1b5ed4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "slices" "sort" "strings" "sync" @@ -45,6 +44,8 @@ type Holder struct { Consumer *StepConsumer // SaySeat says an event of the seat mesh-delivery (recordevents.go): heads-replaced here; nil says none. SaySeat SeatSay + // Links reads the issues linked to a pull request from the issue tracker (issuelinks.go); nil reads none. + Links IssueLinks // Pulls keeps the pull requests' keys on the bus (pullkey.go); nil keeps none. Pulls PullKeys // Say says a condition to the operator, and Conditions keeps which the gate raised (conditions.go); nil says none. @@ -297,7 +298,6 @@ type PullEvent struct { Repo string `json:"repo"` Number int `json:"number"` Title string `json:"title,omitempty"` - Body string `json:"body,omitempty"` Base string `json:"base"` Head string `json:"head"` HeadSHA string `json:"head_sha"` @@ -357,14 +357,9 @@ func (h *Holder) pullUpdated(p PullEvent) (isNew bool) { h.fillFromAnnouncement(d, p) // A pull request retargeted to another base (review of issue 1422): what it merges into is the forge's word now. h.retarget(d, p) - // So is the line naming who pushed it (ADR 0505 §5). - if by := PushedIn(p.Body); p.Body != "" && by != d.PushedBy { - d.PushedBy = by - h.keep(d) - } - // An `after:` line written into the description later is told about too (novox/hq ADR 0419 §2). - if after := AfterIn(p.Body); p.Body != "" && !slices.Equal(after, d.After) { - d.After = after + // So is who opened it, from the forge's pull request record (ADR 0505 §5, ADR 0659 §4). + if p.Author != "" && p.Author != d.Author { + d.Author = p.Author h.keep(d) } if d.State == Proposed && d.outOfDate() != "" && d.AskedAgain == nil { @@ -449,8 +444,9 @@ func (h *Holder) fillFromAnnouncement(d *Delivery, p PullEvent) { d.Title, d.Base, d.Branch, d.HTMLURL, d.CloneURL = p.Title, p.Base, p.Head, p.HTMLURL, p.CloneURL d.Paths, d.PathsTruncated, d.Removed = p.Paths, p.PathsTruncated, p.Removed d.ModuleDirs, d.ModuleDirsSaid = p.ModuleDirs, p.ModuleDirsSaid - d.Issues, d.After = IssuesIn(p.Title+"\n"+p.Body), AfterIn(p.Body) - d.PushedBy = PushedIn(p.Body) + if p.Author != "" { + d.Author = p.Author + } if p.BaseSHA != "" { d.BaseSHA, d.BaseSHAAt = p.BaseSHA, h.Now() } @@ -485,9 +481,9 @@ func (h *Holder) retarget(d *Delivery, p PullEvent) { h.keep(d) } -// PullEdited takes a pull request's edit (novox/hq issue 1079): its `after:` lines, the `Pushed-by` line and the issues it names -// are read again, exactly as an announcement reads them; nothing here holds or withdraws an ask. An edit with no description, -// or of a pull request with no open delivery at that head, changes nothing. +// PullEdited takes a pull request's edit (novox/hq issue 1079): a delivery made from its verdict is filled from it, and a +// record stopped under another, closed pull request at this head is taken over; nothing of its description is read +// (novox/hq ADR 0659), and nothing here holds or withdraws an ask. func (h *Holder) PullEdited(p PullEvent) { if p.HeadSHA == "" || p.Number == 0 { return @@ -515,20 +511,6 @@ func (h *Holder) PullEdited(p PullEvent) { if d != nil && d.Number == p.Number { h.fillFromAnnouncement(d, p) // a delivery made from its verdict, its head never heard (issue 1726) } - if p.Body == "" { - return - } - if d == nil || d.Number != p.Number || d.State.Final() || d.MergedAs != "" { - return - } - d.PushedBy = PushedIn(p.Body) - d.After = AfterIn(p.Body) - title := p.Title - if title == "" { - title = d.Title - } - d.Issues = IssuesIn(title + "\n" + p.Body) - h.keep(d) } // withOther is others with p put in, replacing an earlier announcement of the same pull request. @@ -581,11 +563,10 @@ func (h *Holder) freshAnnouncement(p PullEvent, now time.Time, startedOver strin d := &Delivery{ID: IDOf(p.repository(), p.HeadSHA), Repository: p.repository(), Commit: p.HeadSHA, Number: p.Number, Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid, - After: AfterIn(p.Body), Issues: IssuesIn(p.Title + "\n" + p.Body), Created: now, BaseSHA: p.BaseSHA} + Author: p.Author, Created: now, BaseSHA: p.BaseSHA} if p.BaseSHA != "" { d.BaseSHAAt = now } - d.PushedBy = PushedIn(p.Body) why := fmt.Sprintf("%s#%d's head announced", d.Repository, d.Number) if startedOver != "" { why = fmt.Sprintf("%s; %s", why, startedOver) @@ -756,7 +737,7 @@ func (h *Holder) pullMerged(p PullEvent, late bool) { d = &Delivery{ID: IDOf(p.repository(), head), Repository: p.repository(), Commit: head, Number: p.Number, Title: p.Title, Base: p.Base, Branch: p.Head, HTMLURL: p.HTMLURL, CloneURL: p.CloneURL, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, ModuleDirs: p.ModuleDirs, ModuleDirsSaid: p.ModuleDirsSaid, - After: AfterIn(p.Body), Issues: IssuesIn(p.Title + "\n" + p.Body), Created: now, FromForge: true} + Author: p.Author, Created: now, FromForge: true} // Its check, as the forge holds it on the head: the verdict the pull request merged with. if s := p.HeadChecks[ctxCheck]; s != "" { d.Check = verdictOfCheck(s, "", now) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go index 73ec828ae..c369104d7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go @@ -2,8 +2,12 @@ package main import ( "encoding/json" + "errors" seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" + "sort" + "strconv" "strings" + "sync" "testing" "time" @@ -263,13 +267,16 @@ func TestAMergedGroupEndsMergedWithSourceMainMoved(t *testing.T) { } } -// ADR 0419 §2, issue 978: a description's `after:` line is not read, and says so once on its pull request. The -// composed check is asked as for any other group: no order is asked of the controller first, and none gates it. -func TestAnAfterLineIsNotReadAndSaidOnceAndNothingWaitsForAnOrder(t *testing.T) { +// W11.5 (novox/hq ADR 0659 §4, ADR 0493 §4): the issues an ask names are the issue tracker's links of each member's pull +// request, read through the seat issue-tracker, never the title; a tracker that does not answer is said, and holds nothing; +// the author is the forge's pull request record's. ADR 0419 §2's after: notice is gone with the description. +func TestTheAskNamesTheIssuesLinkedInTheIssueTracker(t *testing.T) { w := newWorld(t) w.op.auto = false + links := &fakeLinks{linked: map[string][]int{"novox/a#1": {948, 12}}, failing: map[string]bool{"novox/b#2": true}} + w.h.Links = links pa := pr("novox/a", 1, "a1a1a1a1a1a1a1", "feat/z") - pa.Body = "Goes after the other.\n\nafter: novox/b\n" + pa.Title, pa.Author = "Fix issue 77", "quiet-deliveries" w.h.PullUpdated(pa) w.h.PullUpdated(pr("novox/b", 2, "b2b2b2b2b2b2b2", "feat/z")) w.h.Checked(verdict("novox/a", 1, "a1a1a1a1a1a1a1", "pass")) @@ -277,29 +284,49 @@ func TestAnAfterLineIsNotReadAndSaidOnceAndNothingWaitsForAnOrder(t *testing.T) w.settleAll() w.later(time.Minute) w.settleAll() - said := 0 + open := w.op.open() + if len(open) != 1 { + t.Fatalf("the group with an unread tracker is not asked: %v", open) + } + e := w.op.ask(open[0]).Explanation + if !strings.Contains(e, "Issues: issue 12, issue 948, b#2's unread (the issue tracker did not answer)") || strings.Contains(e, "77") { + t.Fatalf("the ask's issues are not the links:\n%s", e) + } + if d := w.delivery(IDOf("novox/a", "a1a1a1a1a1a1a1")); d.Author != "quiet-deliveries" || authorOf(d) != "the author of feat/z (quiet-deliveries)" { + t.Fatalf("the author is %q, said %q", d.Author, authorOf(d)) + } + if links.asked["novox/a#1"] != 1 { + t.Fatalf("the links of novox/a#1 were read %d times at one head, want once", links.asked["novox/a#1"]) + } for _, c := range w.forge.comments { - if strings.Contains(c, "not read") && strings.Contains(c, "novox/a#1") { - said++ + if strings.Contains(c, "after:") { + t.Fatalf("an after: notice is said: %q", c) } } - if said != 1 { - t.Fatalf("the after: line is not said once on its pull request: %v", w.forge.comments) +} + +// fakeLinks is the issue tracker's links of pull requests, by `#`. +type fakeLinks struct { + mu sync.Mutex + linked map[string][]int + failing map[string]bool + asked map[string]int +} + +func (f *fakeLinks) Linked(repository string, number int) ([]int, error) { + f.mu.Lock() + defer f.mu.Unlock() + key := repository + "#" + strconv.Itoa(number) + if f.asked == nil { + f.asked = map[string]int{} } - // A new head of the same pull request, with the line still there, is not told again. - pa2 := pr("novox/a", 1, "a9a9a9a9a9a9a9", "feat/z") - pa2.Body = pa.Body - w.h.PullUpdated(pa2) - w.settleAll() - said = 0 - for _, c := range w.forge.comments { - if strings.Contains(c, "not read") { - said++ - } - } - if said != 1 { - t.Fatalf("said again for a new head: %v", w.forge.comments) + f.asked[key]++ + if f.failing[key] { + return nil, errors.New("no responders") } + out := append([]int(nil), f.linked[key]...) + sort.Ints(out) + return out, nil } // Stalled: a delivery past its state's bound is listed with what H2 may do; close takes only that. Only a proposed diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1114_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1114_test.go index 427a8f01b..8f5416c20 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1114_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1114_test.go @@ -18,7 +18,6 @@ func TestClosingThePullRequestOfASharedHeadStartsTheOtherOnesDeliveryOver(t *tes id := IDOf(gCtl, head1114) w.h.PullUpdated(pr(gCtl, 378, head1114, "feat/chain", "internal/a.go")) b := pr(gCtl, 379, head1114, "feat/chain-2", "internal/a.go") - b.Body = "Fixes issue 379." w.h.PullUpdated(b) // announced while #378 is live: the same delivery heard again if d := w.delivery(id); d.Number != 378 { t.Fatalf("setup: the record is #%d, wanted #378 still", d.Number) @@ -56,7 +55,6 @@ func TestAnEditOfAPullRequestWhoseSharedHeadsRecordIsStoppedUnderAnotherTakesItO t.Fatalf("setup: %s", d.State) } e := pr(gCtl, 379, head1114, "feat/chain-2") - e.Body = "Fixes issue 379." w.h.PullEdited(e) d := w.delivery(id) if d.Number != 379 || d.State != Proposed || len(d.Paths) == 0 { @@ -77,7 +75,6 @@ func TestAMergedRecordIsNeverTakenOverByTheOtherPullRequestOfItsHead(t *testing. w.h.PullMerged(merged(gCtl, 378, head1114, "f378f378f378f378")) w.h.PullClosed(pr(gCtl, 378, head1114, "feat/chain")) e := pr(gCtl, 379, head1114, "feat/chain-2") - e.Body = "text" w.h.PullEdited(e) if d := w.delivery(id); d.Number != 378 || d.MergedAs == "" { t.Fatalf("a merged record was taken over: #%d merged as %q", d.Number, d.MergedAs) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go index 120653b6d..99861eaa7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go @@ -11,8 +11,6 @@ import ( // branch, title, URLs or review record. The ready pass skipped it in silence, so no Approve was ever asked, and the // announcement of the same head, heard later, filled none of it. -const verdictFirstBody = "Fixes a bug.\n\nReviewed-by: reviewer@shanks at " + readyHead + "\n" - // verdictFirst is the live shape: the verdict heard, the head not, and the delivery ready. func verdictFirst(t *testing.T) (*world, string) { w := newWorld(t) @@ -48,7 +46,7 @@ func TestADeliveryWithoutItsHeadSaysWhatItWaitsForAndAsksNoOne(t *testing.T) { func TestAnAnnouncementFillsADeliveryMadeFromItsVerdictAndTheOperatorIsAsked(t *testing.T) { w, id := verdictFirst(t) p := pr("novox/app", 419, readyHead, "fix/x", "src/a.go") - p.Title, p.Body, p.CloneURL = "Fix x", verdictFirstBody, "https://forge.invalid/novox/app.git" + p.Title, p.CloneURL = "Fix x", "https://forge.invalid/novox/app.git" w.h.PullUpdated(p) w.settleAll() d := w.delivery(id) @@ -112,7 +110,7 @@ func TestAnAnnouncementFillingAVerdictFirstHeadSupersedesTheOlderHeadAndItsAsk(t func TestAnEditOfThePullRequestFillsADeliveryMadeFromItsVerdict(t *testing.T) { w, id := verdictFirst(t) p := pr("novox/app", 419, readyHead, "fix/x") - p.Title, p.Body = "Fix x", verdictFirstBody + p.Title = "Fix x" w.h.PullEdited(p) w.settleAll() d := w.delivery(id) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issuelinks.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issuelinks.go new file mode 100644 index 000000000..e30287678 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issuelinks.go @@ -0,0 +1,106 @@ +package main + +// The issues an ask names, from the issue tracker's links (novox/hq ADR 0493 §4, ADR 0659 §4): a pull request is linked to +// the issue it fixes by a tool call of the seat issue-tracker, never matched from its title or description, and the gate +// reads those links through the seat's verb `list`. Words only: an ask is never held for them; a read that fails is said in +// the ask as the tracker unread, and read again at the pull request's next head. + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" +) + +// TrackerListVerb is the seat issue-tracker's verb the gate reads a pull request's issues with (the manifest invokes it). +const TrackerListVerb = "seat:issue-tracker.list" + +// IssueLinks reads the issues linked to a pull request in the issue tracker. +type IssueLinks interface { + // Linked is the issues linked to the pull request number of repository (owner/name), ascending. + Linked(repository string, number int) ([]int, error) +} + +// seatIssues is the issue tracker through its seat. +type seatIssues struct{ ask asking } + +func (s seatIssues) Linked(repository string, number int) ([]int, error) { + // A pull request is linked as `#`, lower-case, as the tracker keeps it. + target := "pull-request:" + repoName(repository) + "#" + strconv.Itoa(number) + raw, err := s.ask(TrackerListVerb, map[string]any{"linked": target, "limit": 200}) + if err != nil { + return nil, err + } + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("the seat issue-tracker refused list: %s", lastLine(output)) + } + var listed struct { + Rows []struct { + Number int `json:"number"` + } `json:"rows"` + Error string `json:"error"` + } + if err := json.Unmarshal(answer, &listed); err != nil { + return nil, fmt.Errorf("the seat issue-tracker's list is not readable: %s", clip(string(answer), 200)) + } + if listed.Error != "" { + return nil, fmt.Errorf("the seat issue-tracker could not list %s: %s", target, listed.Error) + } + out := make([]int, 0, len(listed.Rows)) + for _, r := range listed.Rows { + out = append(out, r.Number) + } + sort.Ints(out) + return out, nil +} + +// issuesToRead is the read of the linked issues of every member not read at its head yet, done outside the lock; nil when +// all are read or no issue tracker is reached. Under the lock. +func (h *Holder) issuesToRead(members []*Delivery) func() { + if h.Links == nil { + return nil + } + type read struct { + id, repository, head, why string + number int + issues []int + } + var reads []read + for _, m := range members { + if m.Number != 0 && m.IssuesRead != m.Commit { + reads = append(reads, read{id: m.ID, repository: m.Repository, head: m.Commit, number: m.Number}) + } + } + if len(reads) == 0 { + return nil + } + for _, m := range members { + if m.issuesReading == m.Commit && m.IssuesRead != m.Commit { + return func() {} // read already: its end looks again + } + } + for _, m := range members { + m.issuesReading = m.Commit + } + return func() { + for i := range reads { + r := &reads[i] + issues, err := h.Links.Linked(r.repository, r.number) + r.issues = issues + if err != nil { + r.why = err.Error() + h.Logf("[mesh-delivery] %s: the issues linked to its pull request cannot be read: %v; its ask says so", r.id, err) + } + } + h.mu.Lock() + for _, r := range reads { + if d := h.deliveries[r.id]; d != nil && d.Commit == r.head { + d.Issues, d.IssuesRead, d.IssuesUnread = r.issues, r.head, r.why + h.keep(d) + } + } + h.mu.Unlock() + h.lookAt(h.Now(), "an ask whose issues were read from the issue tracker") + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index bf165028f..5c47895e1 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -61,6 +61,7 @@ func main() { Settings: settings, Store: kvStore{}, Forge: seatForge{ask: stdio.Ask}, + Links: seatIssues{ask: stdio.Ask}, Emit: func(event string, body any) error { return stdio.Emit(event, body) }, Logf: logf, Operator: kvOperator{}, diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index dcc245082..fc3194694 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -144,7 +144,8 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { // merge state through the one statuses tool the seat does not carry yet (seatForge.mergeState); the forge's writes are the // bridge's (ADR 0437 §8). It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks // (ADR 0437 §6). - want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", "seat:mesh-hand-acts.hand-act"} + want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", "seat:mesh-hand-acts.hand-act", + TrackerListVerb} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 6ff3e41e3..340bd7dac 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "regexp" "sort" "strings" "time" @@ -32,15 +31,16 @@ type Delivery struct { Removed []string `json:"removed,omitempty"` ModuleDirs []string `json:"module_dirs,omitempty"` ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` - // After are the repositories its pull request's description names in `after: ` lines. They are not read - // (novox/hq ADR 0419 §2): kept only to say once on the pull request that the line is not read. AfterSaid is whether - // that was said, for this pull request. - After []string `json:"after,omitempty"` - AfterSaid bool `json:"after_said,omitempty"` // Comments is how many comments this owner owed on the pull request at this head, which numbers the next. Comments int `json:"comments,omitempty"` - // Issues are the issues its title and body name ("issue 948"), shown to the operator who is asked to approve its merge. - Issues []int `json:"issues,omitempty"` + // Issues are the issues linked to its pull request in the issue tracker (novox/hq ADR 0493 §4, ADR 0659 §4), shown to the + // operator who is asked to approve its merge; IssuesRead is the head they were read at, IssuesUnread why the issue + // tracker could not say them. Words only: nothing waits on them. + Issues []int `json:"issues,omitempty"` + IssuesRead string `json:"issues_read,omitempty"` + IssuesUnread string `json:"issues_unread,omitempty"` + // issuesReading is a read of them in flight, so a look meanwhile does not ask it again; not kept. + issuesReading string // Group is the delivery group it is in: its head branch name. Empty for a lone delivery. Group string `json:"group,omitempty"` @@ -99,9 +99,9 @@ type Delivery struct { // Ready is what this owner did with the operator's Approve of its merge into main, and with the merge (novox/hq ADR // 0421, ready.go). The json key is the one the ready mark's record had, so rows written before still read. Ready *ReadyRun `json:"ready,omitempty"` - // PushedBy is the session that pushed its head, as its pull request's description names it in a line - // `Pushed-by: ` (playbook 07): named with its branch when a new head is needed (novox/hq ADR 0505 §5). - PushedBy string `json:"pushed_by,omitempty"` + // Author is who opened its pull request, as the forge's pull request record says (the seat git's events): named with + // its branch when a new head is needed (novox/hq ADR 0505 §5, ADR 0659 §4). + Author string `json:"author,omitempty"` // FreshWanted is whether a passing verdict that arrived out of date is to have its check asked against the // trunk's head, before the operator is asked (novox/hq ADR 0421 §3, issue 1022). Cleared when it is asked. FreshWanted bool `json:"fresh_wanted,omitempty"` @@ -284,44 +284,19 @@ func sameCommit(a, b string) bool { return len(a) >= 7 && strings.HasPrefix(strings.ToLower(b), strings.ToLower(a)) } -// pushedLine is the line naming the session that pushed a pull request's head: `Pushed-by: ` (ADR 0505 §5). -var pushedLine = regexp.MustCompile(`(?im)^\s*Pushed-by:\s*([A-Za-z0-9_.\-]+(?:@[A-Za-z0-9_.\-]+)?)\s*$`) - -// PushedIn reads it: the last such line, or "" when there is none. -func PushedIn(body string) string { - who := "" - for _, m := range pushedLine.FindAllStringSubmatch(body, -1) { - who = m[1] - } - return who -} - -// authorOf is a delivery's author as the operator reads it: "the author of ", and the session that pushed it -// when its description names one ("who on node": the router refuses an @ in its words). +// authorOf is a delivery's author as the operator reads it: "the author of ", and who opened its pull request as +// the forge's record says (the router refuses an @ in its words). func authorOf(d *Delivery) string { who := "the author of " + d.Branch if d.Branch == "" { who = "its author" } - if d.PushedBy != "" { - who += " (" + strings.Replace(d.PushedBy, "@", " on ", 1) + ")" + if d.Author != "" { + who += " (" + strings.Replace(d.Author, "@", " on ", 1) + ")" } return who } -// afterLines are the `after: ` lines of a pull request's description. -var afterLines = regexp.MustCompile(`(?im)^\s*after:\s*([A-Za-z0-9_.\-/]+)\s*$`) - -// AfterIn reads them. -func AfterIn(body string) []string { - var out []string - for _, m := range afterLines.FindAllStringSubmatch(body, -1) { - out = append(out, strings.TrimSuffix(m[1], ".git")) - } - sort.Strings(out) - return out -} - // Group is a delivery group: two or more deliveries sharing a head branch name, one level only. type Group struct { ID string `json:"id"` diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go index d5dfaa0c0..ae815a941 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go @@ -39,7 +39,6 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) { Statuses: []string{}, Checks: map[string]HeadCheck{}}} rt.mu.Unlock() announced := pr("novox/app", 8, head, "fix/y", "src/b.go") - announced.Body = "Reviewed-by: lead@shanks at " + head b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(announced)) // The checker's verdict at the head, then the forge's word that its bridge set mesh/check. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 649f99a27..7ce09b063 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -212,9 +212,6 @@ func (h *Holder) readyAsks(now time.Time) []func() { h.Logf("[mesh-delivery] %s is ready (%s); the operator's Approve of its merge into main is asked once it may merge", d.ID, h.checkAt([]*Delivery{d}).Words) } - if a := h.sayAfterLine(d); a != nil { - asks = append(asks, a) - } r := d.Ready if r == nil || r.Merged != nil || r.declined() { continue @@ -315,26 +312,6 @@ func (h *Holder) readyAsks(now time.Time) []func() { return asks } -// sayAfterLine tells a pull request, once, that its description holds an `after: ` line, which is not read -// (novox/hq ADR 0419 §2): the order of a delivery comes from the module graph alone, and no refusal asks for the line. -// Once per pull request, not per head. Under the lock; the comment is said outside it. -func (h *Holder) sayAfterLine(d *Delivery) func() { - if len(d.After) == 0 || d.AfterSaid { - return nil - } - d.AfterSaid = true - for _, o := range h.deliveries { - if o != d && o.Number == d.Number && o.Repository == d.Repository && o.AfterSaid { - h.keep(d) - return nil // said on this pull request for an earlier head - } - } - h.keep(d) - h.Logf("[mesh-delivery] %s: its description holds an after: line (%s), which is not read", d.ID, strings.Join(d.After, ", ")) - return h.sayOn(d, "mesh-delivery: this description holds an `after:` line ("+strings.Join(d.After, ", ")+"). It is not read: "+ - "the order of a delivery comes from the module graph alone, and nothing asks for the line (novox/hq ADR 0419).") -} - // mergeAgainNow lets an approved delivery's merge be asked at once: something that decides whether it may merge happened (a // verdict, an Approve, a group's composed check), so the wait after a refusal does not hold it. Under the lock. func (h *Holder) mergeAgainNow(d *Delivery) { diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index e462c942a..9677d8801 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -294,7 +294,8 @@ "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", - "seat:mesh-hand-acts.hand-act" + "seat:mesh-hand-acts.hand-act", + "seat:issue-tracker.list" ], "state": [ "deliveries", -- 2.54.0 From ed188268a0fb9bfd9c9a902d31a5c5d87b96b284 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:44:20 +0100 Subject: [PATCH 056/243] The gate asks the ledger's calls and its fill of its machine's tool runner (milestone 47/G, W11.20 ruling) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate reads its records' ledger fill before it opens a record; its manifest asked no capability, so it could run beside a runner answering neither. It now asks ledger-calls and ledger-fill, the fill a capability of its own. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from c20044a2.) Builds: design 47 G §181, §462, §488 (GATE12) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/manifest_test.go | 8 ++++++++ modules/mesh-delivery-gate/module.json | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index fc3194694..138f4f22a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -12,6 +12,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest" ) @@ -35,6 +36,8 @@ type manifest struct { State []any `json:"state"` Reads []string `json:"reads"` Tools []string `json:"tools"` + // Capabilities are what the gate's machine's tool runner must state (buscalls.Capabilities). + Capabilities []string `json:"capabilities"` } // buckets are the names of the state the manifest declares, each a bare name or an object with a name and a history. @@ -134,6 +137,11 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if raw, err := os.ReadFile("../../module.json"); err != nil || strings.Contains(string(raw), "check-gates-merge") { t.Fatalf("the manifest still names check-gates-merge (%v)", err) } + // W11.20: the gate keeps its records in a ledger and reads its fill before it opens one, so it is held off a machine whose + // tool runner answers neither (the fill a capability of its own, the lead's ruling). + if !reflect.DeepEqual(m.Capabilities, []string{buscalls.LedgerCalls, buscalls.LedgerFillCalls}) { + t.Fatalf("capabilities %v, want %s and %s", m.Capabilities, buscalls.LedgerCalls, buscalls.LedgerFillCalls) + } // What it reads: the teller's answers and the checker's verdicts (issue 1302). The carry from today's module's buckets, the // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). The delivery view's reach, which // `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4); no forge tool is asked for it. diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 9677d8801..138321257 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -319,6 +319,10 @@ }, "conditions" ], + "capabilities": [ + "ledger-calls", + "ledger-fill" + ], "reads": [ "mesh-delivery-teller.answers", "mesh-delivery-checker.verdicts", -- 2.54.0 From 6fe1bba0673850f3ba3837e78a45e0f662d076ac Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:45:50 +0100 Subject: [PATCH 057/243] =?UTF-8?q?The=20gate=20hears=20the=20teller's=20a?= =?UTF-8?q?nswered=20and=20the=20view's=20reached,=20reads=20their=20answe?= =?UTF-8?q?rs=20and=20reach=20through=20their=20tools,=20and=20reads=20nei?= =?UTF-8?q?ther=20state=20(milestone=2047/G,=20ruling=203:=20ADR=200596=20?= =?UTF-8?q?=C2=A72-=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from ace22b9f.) Builds: design 47 G §431 (the reads of the teller's answers and the view's reach are by their events); the commit's calls of teller_answers and view_reach are kept whole, the map names them no line Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/approval.go | 17 +-- .../cmd/mesh-delivery-gate/bus_test.go | 30 +++-- .../cmd/mesh-delivery-gate/main.go | 39 ++----- .../cmd/mesh-delivery-gate/manifest_test.go | 18 ++- .../cmd/mesh-delivery-gate/ownerreads.go | 107 ++++++++++++++++++ .../cmd/mesh-delivery-gate/ownerreads_test.go | 79 +++++++++++++ .../cmd/mesh-delivery-gate/reached.go | 3 - modules/mesh-delivery-gate/module.json | 10 +- 8 files changed, 244 insertions(+), 59 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads_test.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index e322d10c4..32e42916f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -794,8 +794,8 @@ func askOf(a Ask) (oc.Ask, error) { return occ.Carried(a.Ask, mdc.AskingID(key, a.Asking)) } -// kvOperator is the operator over the runtime: the ask is a key of the gate's bucket asks, the answer a key of the teller's -// bucket answers. +// kvOperator is the operator over the runtime: the ask is a key of the gate's bucket asks, the answer the teller's, read +// through its tool teller_answers. type kvOperator struct{} func (kvOperator) Ask(key string, a Ask) error { @@ -817,17 +817,8 @@ func (kvOperator) Ask(key string, a Ask) error { func (kvOperator) Cancel(key string) error { return stdio.State("asks").Delete(key) } -func (kvOperator) Where(key string) (*Answer, error) { - e, err := stdio.State(tellerModule + ".answers").Get(key) - if err != nil || e == nil { - return nil, err - } - a, err := mdc.DecodeAnswer(e.Value) - if err != nil { - return nil, fmt.Errorf("the answer %s is not readable: %w", key, err) - } - return &a, nil -} +// Where reads the teller's answer through its tool (ADR 0596 §3), never its bucket. +func (kvOperator) Where(key string) (*Answer, error) { return answersOf(stdio.Ask, key) } // standing is whether an ask is written under a key in the gate's asks, as this owner last knew. Under the lock. func (h *Holder) standing(key string) bool { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 3ebee8272..3a40582f9 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -53,6 +53,9 @@ type testBus struct { nc *nats.Conn js jetstream.JetStream kvs map[string]jetstream.KeyValue + // the teller's answers, as its tool teller_answers says them, and the delivery view's reach as view_reach says it. + mu sync.Mutex + answers map[string][]byte } func newTestBus(t *testing.T) *testBus { @@ -78,7 +81,7 @@ func newTestBus(t *testing.T) *testBus { if err != nil { t.Fatal(err) } - b := &testBus{nc: nc, js: js, kvs: map[string]jetstream.KeyValue{}} + b := &testBus{nc: nc, js: js, kvs: map[string]jetstream.KeyValue{}, answers: map[string][]byte{}} // The controller's job: a bucket per state the manifest declares. m, _ := readManifest(t) for _, name := range m.buckets() { @@ -88,8 +91,8 @@ func newTestBus(t *testing.T) *testBus { } b.kvs[name] = kv } - // The buckets of other modules the manifest reads (`.` is the bucket `_`): the teller's answers - // and the checker's verdicts. + // The buckets of other modules the manifest reads (`.` is the bucket `_`): the checker's verdicts. + // The teller's answers and the view's reach are heard as events and asked of their tools (ADR 0596). for _, name := range m.Reads { kv, err := js.CreateKeyValue(context.Background(), jetstream.KeyValueConfig{Bucket: strings.Replace(name, ".", "_", 1), History: 10}) if err != nil { @@ -100,7 +103,7 @@ func newTestBus(t *testing.T) *testBus { // The streams the events are on (mesh.mod..event., mesh.seat..event.). if _, err := js.CreateStream(context.Background(), jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.seat.git.event.>", "mesh.seat." + oc.Seat + ".event.>", - "mesh.mod.mesh-delivery-forge-bridge.event.>"}}); err != nil { + "mesh.mod.mesh-delivery-forge-bridge.event.>", "mesh.mod.mesh-delivery-teller.event.>"}}); err != nil { t.Fatal(err) } return b @@ -379,6 +382,17 @@ func (r *runtime) answer(m rpcMessage) { // Green: mesh/check a success at the head asked, as the seat answers. commit, _ := a.Body["commit"].(string) return gitseat.StatusesAnswer{Commit: commit, Statuses: []gitseat.Status{{Context: "mesh/check", State: "success"}}}, nil + case "mesh-delivery-teller.teller_answers": + key, _ := a.Body["key"].(string) + r.b.mu.Lock() + raw, found := r.b.answers[key] + r.b.mu.Unlock() + if !found { + return map[string]any{"key": key, "found": false}, nil + } + return map[string]any{"key": key, "found": true, "answer": json.RawMessage(raw)}, nil + case "mesh-delivery-view.view_reach": + return map[string]any{"reach": []json.RawMessage{}}, nil case "gitea.gitea_note_append", "gitea.gitea_delivery_view", "gitea.gitea_commit_status", "gitea.gitea_merge_pull_request", "gitea.gitea_add_comment": return toolAnswer(map[string]any{}), nil @@ -715,9 +729,11 @@ func (b *testBus) answer(t *testing.T, key, option string) time.Time { if err != nil { t.Fatal(err) } - if _, err := b.kvs["mesh-delivery-teller.answers"].Put(context.Background(), key, raw); err != nil { - t.Fatal(err) - } + // The teller writes the answer under the ask's key and says it as the event answered (ADR 0596 §2). + b.mu.Lock() + b.answers[key] = raw + b.mu.Unlock() + b.publish(t, "mesh.mod.mesh-delivery-teller.event.answered", map[string]any{"key": key, "ask": asked.ID, "answer": json.RawMessage(raw)}) return time.Now() } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 5c47895e1..fdb7beceb 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -35,7 +35,7 @@ func (l *listening) get() string { l.mu.Lock(); defer l.mu.Unlock(); return l.n // The events this owner follows: the forge's pull requests, as the seat git says them (gitseat.go). The forge's status events // say that a check answered (events.go). The step events the delivery record is built from follow (consumer.go). -var followed = append(append(append([]string{}, gitEvents...), forgePattern, owedEndedEvent, ConfigChangedEvent), stepPatterns()...) +var followed = append(append(append([]string{}, gitEvents...), forgePattern, owedEndedEvent, answeredEvent, reachedEvent, ConfigChangedEvent), stepPatterns()...) // forgePattern is the forge bridge's messages on FORGE (novox/hq ADR 0437 §8): the gate follows a merge and the refusal of one. const forgePattern = "mesh-delivery-forge-bridge.forge.**" @@ -112,32 +112,6 @@ func run(h *Holder, l *listening) { time.Sleep(wait) } } - // The teller's answers: a change of its bucket asks for a look, which reads the answers to the asks pending. - go func() { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - err := stdio.State(tellerModule+".answers").Watch("", func(stdio.StateChange) error { h.AnswersChanged(); return nil }) - if err == nil { - return - } - logf("[mesh-delivery] not hearing the teller's answers yet (%v); asking again in %s", err, wait) - time.Sleep(wait) - } - }() - // The delivery view's reach (novox/hq ADR 0556 clause 4): how far each repository reached, which `deliveries` and `show` - // derive whether a merge is on the machines from. Each change is kept; nothing is moved by it. - go func() { - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - err := stdio.State(reachState).Watch("", func(c stdio.StateChange) error { - h.ReachChanged(c.Key, c.Value, c.Op == "delete") - return nil - }) - if err == nil { - return - } - logf("[mesh-delivery] not hearing the delivery view's reach yet (%v); asking again in %s", err, wait) - time.Sleep(wait) - } - }() // The checker's verdicts (issue 1302, ADR 0447 §2): the Approve on mesh/check is asked only on its verdict at exactly the // heads an ask covers, and a group's verdict is that one; each change asks for a look. go func() { @@ -194,6 +168,10 @@ func switchToRecords(h *Holder, l *listening) { // read by the first look, and the asks the router closed meanwhile are read here. func readOnStart(h *Holder) { h.wantForge() + // The delivery view's reach, said while the gate was away, is read once through its tool; from then on its events carry it. + if err := h.ReachRead(stdio.Ask); err != nil { + logf("[mesh-delivery] the delivery view's reach cannot be read yet, so merges read as not known until it says one (%v)", err) + } h.hearAsks() h.SourceCatchUp() // a merge missed or marked unwritten while the gate was away reaches source.main, and so the checker (issue 1726) } @@ -218,6 +196,13 @@ func take(h *Holder, e stdio.Envelope) error { return nil } switch e.Key { + case answeredEvent: + h.AnswersChanged() // the teller wrote an answer: the look that follows reads the answers to the asks pending + return nil + case reachedEvent: + return h.ReachHeard(e.Body) + } + switch e.Key { case gitEvent(gitseat.EventPullUpdated), gitEvent(gitseat.EventPullMerged), gitEvent(gitseat.EventPullClosed): p, err := gitPullEvent(e.Key, e.Body) if err != nil { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 138f4f22a..1575a13af 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -142,18 +142,26 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if !reflect.DeepEqual(m.Capabilities, []string{buscalls.LedgerCalls, buscalls.LedgerFillCalls}) { t.Fatalf("capabilities %v, want %s and %s", m.Capabilities, buscalls.LedgerCalls, buscalls.LedgerFillCalls) } - // What it reads: the teller's answers and the checker's verdicts (issue 1302). The carry from today's module's buckets, the - // seat's holder and the machines' haves went with the seat's hand-over (ADR 0539 §17). The delivery view's reach, which - // `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4); no forge tool is asked for it. - if !reflect.DeepEqual(m.Reads, []string{tellerModule + ".answers", checkerVerdicts, viewModule + ".reach"}) { + // What it reads: the checker's verdicts alone (issue 1302; they go with the checker). The teller's answers and the delivery + // view's reach, which `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4), are no + // state it reads (ADR 0596 §2-§4): it hears their owners' events answered and reached and asks their tools. + if !reflect.DeepEqual(m.Reads, []string{checkerVerdicts}) { t.Fatalf("reads %v", m.Reads) } + for _, c := range []string{"mesh-delivery-teller.answered", "mesh-delivery-view.reached"} { + if !slices.Contains(m.Consumes, c) { + t.Errorf("the manifest does not consume %s", c) + } + } + if answeredEvent != "mesh-delivery-teller.answered" || reachedEvent != "mesh-delivery-view.reached" { + t.Errorf("the events heard are %q and %q", answeredEvent, reachedEvent) + } // It calls no controller verb and reads the forge through the seat git's verbs (design 47 *The seat git*), and the forge's // merge state through the one statuses tool the seat does not carry yet (seatForge.mergeState); the forge's writes are the // bridge's (ADR 0437 §8). It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks // (ADR 0437 §6). want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", "seat:mesh-hand-acts.hand-act", - TrackerListVerb} + TrackerListVerb, tellerAnswersTool, viewReachTool} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads.go new file mode 100644 index 000000000..6e123eb69 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads.go @@ -0,0 +1,107 @@ +package main + +import ( + "encoding/json" + "fmt" + + mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate" + "git.novox.be/novox/mesh-delivery/sdk/trunk" +) + +// What the gate reads of the teller and of the delivery view it reads through their owners' events and tools, never their +// state (ADR 0596 §2-§4): the teller says `answered` after each answer it writes and answers its answers through +// `teller_answers`; the view says `reached` after each reach it writes and answers its reach through `view_reach`. An event +// asks a look (the answers) or takes the reach it carries; the start reads what was said while the gate was away, once. + +// The tools the gate asks (the manifest invokes them). +const ( + tellerAnswersTool = tellerModule + ".teller_answers" + viewReachTool = viewModule + ".view_reach" +) + +// The events the gate hears: `.`, as the manifest consumes them. +var ( + answeredEvent = declared("answered") + reachedEvent = declared("reached") +) + +// answersOf reads the teller's answer to an ask through its tool: nil when the teller kept none. +func answersOf(ask asking, key string) (*Answer, error) { + raw, err := ask(tellerAnswersTool, map[string]any{"key": key}) + if err != nil { + return nil, err + } + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("the teller refused teller_answers: %s", lastLine(output)) + } + var r struct { + Found bool `json:"found"` + Answer json.RawMessage `json:"answer"` + } + if err := json.Unmarshal(answer, &r); err != nil { + return nil, fmt.Errorf("the teller's answer to %s is not readable: %s", key, clip(string(answer), 200)) + } + if !r.Found { + return nil, nil + } + a, err := mdc.DecodeAnswer(r.Answer) + if err != nil { + return nil, fmt.Errorf("the answer %s is not readable: %w", key, err) + } + return &a, nil +} + +// reachedBody is the body of the view's event reached: the key of the reach and the reach as the view wrote it. +type reachedBody struct { + Key string `json:"key"` + Reach json.RawMessage `json:"reach"` +} + +// ReachHeard takes the view's event reached. +func (h *Holder) ReachHeard(body []byte) error { + var e reachedBody + if err := json.Unmarshal(body, &e); err != nil || e.Key == "" { + h.Refuse(fmt.Errorf("the delivery view's reach event is not readable: %v", err)) + return nil + } + h.ReachChanged(e.Key, e.Reach, false) + return nil +} + +// ReachRead takes every reach the view's tool answers: what was said while the gate was away. +func (h *Holder) ReachRead(ask asking) error { + raw, err := ask(viewReachTool, map[string]any{}) + if err != nil { + return err + } + answer, output, ok := answerOf(raw) + if !ok { + return fmt.Errorf("the delivery view refused view_reach: %s", lastLine(output)) + } + var all struct { + Reach []json.RawMessage `json:"reach"` + } + if err := json.Unmarshal(answer, &all); err != nil { + return fmt.Errorf("the delivery view's reach is not readable: %s", clip(string(answer), 200)) + } + for _, r := range all.Reach { + v, err := viewstate.DecodeReach(r) + if err != nil { + h.Logf("[mesh-delivery] a reach the delivery view answers is not readable, so its merges read as not known: %v", err) + continue + } + key, err := trunk.RepositoryKey(v.Repository) + if err != nil { + continue + } + h.mu.Lock() + _, heard := h.reach[key] + h.mu.Unlock() + if !heard { // a reach the event said is as new as this read: it is kept + h.ReachChanged(key, r, false) + } + } + return nil +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads_test.go new file mode 100644 index 000000000..677ef8db3 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads_test.go @@ -0,0 +1,79 @@ +package main + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-view/viewstate" + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// ADR 0596 §2-§4: the gate reads the teller's answer through the teller's tool, found or not, never the bucket. +func TestTheTellersAnswerIsReadThroughItsTool(t *testing.T) { + answer, err := mdc.EncodeAnswer(mdc.Answer{Option: "approve", Warrant: "w1", Words: "the operator chose Approve"}) + if err != nil { + t.Fatal(err) + } + var asked []string + ask := func(key string, body any) (json.RawMessage, error) { + asked = append(asked, fmt.Sprint(key, body)) + if body.(map[string]any)["key"] == "k1" { + return json.Marshal(map[string]any{"key": "k1", "found": true, "answer": json.RawMessage(answer)}) + } + return json.RawMessage(`{"key":"k2","found":false}`), nil + } + a, err := answersOf(ask, "k1") + if err != nil || a == nil || a.Option != "approve" || a.Warrant != "w1" { + t.Fatalf("the answer = %+v, %v", a, err) + } + if a, err = answersOf(ask, "k2"); err != nil || a != nil { + t.Errorf("no answer = %+v, %v", a, err) + } + if len(asked) != 2 || !strings.HasPrefix(asked[0], "mesh-delivery-teller.teller_answers") { + t.Errorf("asked %v", asked) + } + if _, err = answersOf(func(string, any) (json.RawMessage, error) { return nil, fmt.Errorf("no tool runner") }, "k"); err == nil { + t.Errorf("a tool that cannot be asked was read as no answer") + } +} + +// The events answered and reached are heard: the first asks a look, the second carries the reach, which a read of the view's +// tool at start fills in only where no event said one. +func TestTheViewsReachIsHeardAndReadAtStart(t *testing.T) { + w := newWorld(t) + reach := func(head string) json.RawMessage { + raw, err := viewstate.EncodeReach(viewstate.Reach{Repository: "novox/web", Reached: head, ReachedSrc: 3}) + if err != nil { + t.Fatal(err) + } + return raw + } + body, _ := json.Marshal(reachedBody{Key: "novox/web", Reach: reach(strings.Repeat("a", 40))}) + if err := take(w.h, stdio.Envelope{Key: reachedEvent, Body: body}); err != nil { + t.Fatal(err) + } + if got := w.h.reach["novox/web"].Reached; got != strings.Repeat("a", 40) { + t.Fatalf("the reach heard is %q", got) + } + read := func(string, any) (json.RawMessage, error) { + return json.Marshal(map[string]any{"reach": []json.RawMessage{reach(strings.Repeat("b", 40)), func() json.RawMessage { + raw, _ := viewstate.EncodeReach(viewstate.Reach{Repository: "novox/api", Reached: strings.Repeat("c", 40), ReachedSrc: 4}) + return raw + }()}}) + } + if err := w.h.ReachRead(read); err != nil { + t.Fatal(err) + } + if got := w.h.reach["novox/web"].Reached; got != strings.Repeat("a", 40) { + t.Errorf("the read at start overwrote the reach the event said: %q", got) + } + if got := w.h.reach["novox/api"].Reached; got != strings.Repeat("c", 40) { + t.Errorf("the reach the event never said is read at start: %q", got) + } + if err := take(w.h, stdio.Envelope{Key: reachedEvent, Body: []byte(`{"reach":`)}); err != nil { + t.Errorf("a reach event that cannot be read is said, never left to be heard again: %v", err) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go index c9f6ae8be..806ce1581 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go @@ -14,9 +14,6 @@ import ( // The order of two merges into one trunk is the gate's own: the moment each was merged, the trunk being a line of merges. A // reached head the gate holds no merge of is said, never asked of the forge. -// reachState is the delivery view's state the gate reads. -const reachState = viewModule + ".reach" - // ReachChanged takes one change of the view's reach. func (h *Holder) ReachChanged(key string, value []byte, deleted bool) { h.mu.Lock() diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 138321257..3c663d2a2 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -29,6 +29,8 @@ "git.repo.writable", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-forge-bridge.owed.ended", + "mesh-delivery-teller.answered", + "mesh-delivery-view.reached", "mesh-config.config-changed", "mesh-delivery-indexer.take-in.taken" ], @@ -295,7 +297,9 @@ "seat:git.open-pulls", "seat:git.contains", "seat:mesh-hand-acts.hand-act", - "seat:issue-tracker.list" + "seat:issue-tracker.list", + "mesh-delivery-teller.teller_answers", + "mesh-delivery-view.view_reach" ], "state": [ "deliveries", @@ -324,9 +328,7 @@ "ledger-fill" ], "reads": [ - "mesh-delivery-teller.answers", - "mesh-delivery-checker.verdicts", - "mesh-delivery-view.reach" + "mesh-delivery-checker.verdicts" ], "tools": [ "delivery_merge_again", -- 2.54.0 From 435d1206e477a6a8cd9ecc8c8fd2cabce8f9e2e0 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:45:50 +0100 Subject: [PATCH 058/243] A verdict, composed check or status of the same heads that changes what a refusal waited on starts the count of repeated refusals again (milestone 47/G, W11.7, issue 1785) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 80d401c4.) Builds: design 47 G (main file G row: issues 1785, 1794) (GATE) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/events.go | 2 +- .../cmd/mesh-delivery-gate/issue1348_test.go | 20 +++++++++++++++++++ .../cmd/mesh-delivery-gate/onestatus.go | 4 ++-- .../cmd/mesh-delivery-gate/ready.go | 10 ++++++++++ 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go index 1cd053de7..18bdfef88 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go @@ -81,7 +81,7 @@ func (h *Holder) CommitStatus(e ForgeStatusEvent) { head = sha } if sameCommit(head, e.SHA) { - h.mergeAgainNow(d) // its approved head's check turned success on the forge: the merge is asked again at once + h.causeChanged(d) // its approved head's check turned success on the forge: the merge is asked again at once, the count anew } } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go index fc25f9384..fdcebc717 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go @@ -224,3 +224,23 @@ func TestARefusalThatRepeatsWithNoTrunkMoveIsNotAskedAgainWithoutEnd(t *testing. t.Fatal("a trunk's move did not let the merge be asked again") } } + +// 1785: a verdict, a composed check or a status of the same heads that changes what the refusal waited on starts the count of +// the same words again, so an approved merge whose cause cleared is asked again without a second Approve; a status that is no +// success changes nothing. +func TestANewCauseOfTheSameHeadsStartsTheCountOfARepeatedRefusalAgain(t *testing.T) { + w := answeredFirst(t) + d := w.h.deliveries[IDOf(gCat, gHk)] + d.Ready.Repeats, d.Ready.RepeatTrunk = maxRepeatedRefusals, d.TrunkNow + d.Ready.LastTry = w.now + owner, repo, _ := strings.Cut(gCat, "/") + w.h.CommitStatus(ForgeStatusEvent{Owner: owner, Repo: repo, SHA: gHk, Context: "mesh/check", State: "pending"}) + if d.Ready.Repeats != maxRepeatedRefusals || d.Ready.LastTry.IsZero() { + t.Fatalf("a status that is no success changed the refusal: %d repeats", d.Ready.Repeats) + } + w.h.CommitStatus(ForgeStatusEvent{Owner: owner, Repo: repo, SHA: gHk, Context: "mesh/check", State: "success"}) + if d.Ready.Repeats != 0 || !d.Ready.LastTry.IsZero() { + t.Fatalf("the check turned success on the same head: %d repeats, last try %v; the merge is asked again and the count starts anew", + d.Ready.Repeats, d.Ready.LastTry) + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go index 554ad13f3..9afb0c564 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go @@ -185,7 +185,7 @@ func (h *Holder) takeCheckStatus(repository, sha, state, desc string) { return // past its verdict: the merge reads the forge's status itself } d.Check, d.AskedAgain, d.CheckAsk = v, nil, nil - h.mergeAgainNow(d) + h.causeChanged(d) if why := d.outOfDate(); why != "" { h.Logf("[mesh-delivery] %s: its mesh/check is %s; the checker checks it again", d.ID, why) } @@ -299,7 +299,7 @@ func (h *Holder) composeFromChecker(g *Group, members []*Delivery, now time.Time h.keepGroup(g) if c.Verdict != "" { for _, d := range members { - h.mergeAgainNow(d) + h.causeChanged(d) // a composed check that differs from the last: what a refusal waited on changed (issue 1785) } h.Logf("[mesh-delivery] group %s: %s — %s", g.ID, c.Verdict, c.Summary) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 7ce09b063..18749f9f6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -323,6 +323,16 @@ func (h *Holder) mergeAgainNow(d *Delivery) { } } +// causeChanged takes a verdict, a composed check or a status of the same heads that decides whether the merge may go: what the +// refusal waited on changed, so the count of the same words starts again and the merge is asked at once (issue 1785); only a +// refusal that truly repeats, with nothing new behind it, stops the asking (issue 1106). Under the lock. +func (h *Holder) causeChanged(d *Delivery) { + if d != nil && d.Ready != nil && d.Ready.Merged == nil { + d.Ready.Repeats = 0 + } + h.mergeAgainNow(d) +} + // maxRepeatedRefusals is how many refusals in a row, with the same words and the trunk not moved, end the asking again: a // refusal that repeats with no trunk move is not a trunk's move, and the check asked after each only brings the same words. const maxRepeatedRefusals = 3 -- 2.54.0 From 46c125c07cf4cd4642b6f19d15f579dc14565f52 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:45:50 +0100 Subject: [PATCH 059/243] The gate keeps no mark of an own ask of a check: the dead field is gone and a row holds it (milestone 47/G, W11.8, issue 1794: its cause went with the gate's checks, issue 1318) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 70ded58f.) Builds: design 47 G (main file G row: issues 1785, 1794) (GATE) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/adr661_test.go | 19 +++++++++++++++++++ .../cmd/mesh-delivery-gate/ready.go | 3 --- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go index 164d3a1f7..19e7d353e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go @@ -1,6 +1,8 @@ package main import ( + "encoding/json" + "reflect" "strings" "testing" "time" @@ -134,3 +136,20 @@ func TestNoCheckGatesAMergeAtTheGate(t *testing.T) { t.Run(name, run) } } + +// Issue 1794: the gate keeps no mark of an own ask of a check, so no verdict of another check of the head can clear one while +// the gate's own is in flight: since issue 1318 the gate asks no check, only the checker's verdict at the heads is read, and a +// run holds no field that marks an ask of one. +func TestTheGateAsksNoCheckOfItsOwnSoNoMarkOfOneCanBeClearedByAnother(t *testing.T) { + typ := reflect.TypeOf(ReadyRun{}) + for i := 0; i < typ.NumField(); i++ { + if f := typ.Field(i).Name; strings.HasPrefix(f, "OwnAsk") || strings.HasPrefix(f, "OwnCheck") { + t.Errorf("ReadyRun holds %s: a mark of a check the gate asked itself, which it no longer asks", f) + } + } + for _, key := range []string{"own_asked_at", "own_ask"} { + if raw, _ := json.Marshal(ReadyRun{}); strings.Contains(string(raw), key) { + t.Errorf("a run is kept with %s", key) + } + } +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go index 18749f9f6..3898e05de 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go @@ -81,9 +81,6 @@ type ReadyRun struct { // (merges.go). MergeCommit string `json:"merge_commit,omitempty"` SourceMoved bool `json:"source_moved,omitempty"` - // OwnAskedAt is when its own check was asked after the merge tool refused it for a pending status of its own (issue - // 1049): once for the head this run is for. - OwnAskedAt time.Time `json:"own_asked_at,omitempty"` // Urgent is set once a refusal of this head was one that is urgent at once: its severity never steps down to a // warning after (issue 1275, AC6). Urgent bool `json:"urgent,omitempty"` -- 2.54.0 From 1f1aa70580c0126001284850f1c68ede5d5e914a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:45:50 +0100 Subject: [PATCH 060/243] The gate's whole-module test traces a delivery in one record and starts the gate again over the same bus and ledger: the record whole, nothing handed or asked again (milestone 47/G, W11.16: acceptance 34, 39) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from d162f899.) Builds: design 47 G §160, tests 34, 39 (GATE2) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bus_test.go | 66 ++++++++++++++++--- 1 file changed, 58 insertions(+), 8 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 3a40582f9..bcbf1772a 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -15,6 +15,7 @@ import ( "io" "os" "os/exec" + "reflect" "strings" "sync" "testing" @@ -56,6 +57,8 @@ type testBus struct { // the teller's answers, as its tool teller_answers says them, and the delivery view's reach as view_reach says it. mu sync.Mutex answers map[string][]byte + // the gate's ledger `records`, as the tool runner keeps it: it outlives one run of the gate, as the bus's does. + ledger *ledgerFake } func newTestBus(t *testing.T) *testBus { @@ -119,8 +122,6 @@ type runtime struct { next int wait map[string]chan json.RawMessage - // the gate's ledger, as the tool runner keeps it. - ledger *ledgerFake // what the bundle subscribed to, and asked of the mesh. subs []string asked []askedOfMesh @@ -406,24 +407,24 @@ func (r *runtime) answer(m rpcMessage) { switch m.Method { case buscalls.LedgerApply, buscalls.LedgerGet: // The gate's ledger `records` (W11.1), as the tool runner keeps it: the record test's ledger in memory. - r.mu.Lock() - defer r.mu.Unlock() - if r.ledger == nil { - r.ledger = newLedgerFake(r.t) + r.b.mu.Lock() + defer r.b.mu.Unlock() + if r.b.ledger == nil { + r.b.ledger = newLedgerFake(r.t) } if m.Method == buscalls.LedgerGet { var in buscalls.InstanceRequest if err := json.Unmarshal(m.Params, &in); err != nil { return nil, err } - state, seq, err := r.ledger.Get(in.Instance) + state, seq, err := r.b.ledger.Get(in.Instance) return buscalls.InstanceState{Instance: in.Instance, State: state, Sequence: seq}, err } var in buscalls.ApplyRequest if err := json.Unmarshal(m.Params, &in); err != nil { return nil, err } - seq, err := r.ledger.Apply(in.Instance, in.Operation, in.Body) + seq, err := r.b.ledger.Apply(in.Instance, in.Operation, in.Body) return buscalls.Applied{Taken: err == nil, Sequence: seq}, err } kv := r.b.kvs[p.State] @@ -656,6 +657,49 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB return err == nil && json.Unmarshal(e.Value(), &d) == nil && d.State == Merged }) rt.noControllerAsk() + // W11.16 (acceptance 34, 39): the delivery is traced in one record of the ledger, which says its fill; the gate is stopped + // and started again over the same bus and ledger: the record is whole and opened once, the merge is not handed again, and + // the ask of the Approve already given is not asked again. + recordsNow := func(r *runtime) map[string]any { + var said struct { + Result struct { + Content []struct{ Text string } `json:"content"` + } `json:"result"` + } + raw := r.call(t, "gate_records", map[string]any{}) + if err := json.Unmarshal([]byte(raw), &said); err != nil || len(said.Result.Content) == 0 { + t.Fatalf("gate_records answered %s", raw) + } + var out map[string]any + if err := json.Unmarshal([]byte(said.Result.Content[0].Text), &out); err != nil { + t.Fatalf("gate_records says %s", said.Result.Content[0].Text) + } + return out + } + before := recordsNow(rt) + if kept, _ := before["kept"].(float64); kept != 1 || before["fill"] == nil || before["fill"] == "" { + t.Fatalf("one delivery is traced in %v records, fill %v", before["kept"], before["fill"]) + } + merges := len(rt.mergesSaid()) + _ = rt.in.Close() + _ = rt.cmd.Wait() + rt2 := b.start(t) + evctx2, cancel2 := context.WithCancel(ctx) + defer cancel2() + rt2.deliverEvents(evctx2) + rt2.untilListening() + after := recordsNow(rt2) + if !reflect.DeepEqual(before["records"], after["records"]) || after["kept"] != before["kept"] { + t.Fatalf("the record changed over a restart: %v then %v", before["records"], after["records"]) + } + rt2.until("the restarted gate to hold the delivery merged", eventWithin, func() bool { + e, err := b.kvs["deliveries"].Get(ctx, kvKey(IDOf("novox/app", head))) + var d Delivery + return err == nil && json.Unmarshal(e.Value(), &d) == nil && d.State == Merged + }) + if n := len(rt2.mergesSaid()); n != 0 || len(onlyAskKeys(b)) > 1 { + t.Fatalf("the restarted gate said the merge %d more times (%d before), or asked again: %v", n, merges, onlyAskKeys(b)) + } // Quiet follows: nothing asks the forge or the controller again with no event to say so. quiet := len(rt.asked) time.Sleep(1500 * time.Millisecond) @@ -667,6 +711,12 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB } } +// onlyAskKeys are the keys of every ask in the gate's bucket asks. +func onlyAskKeys(b *testBus) []string { + keys, _ := b.kvs["asks"].Keys(context.Background()) + return keys +} + // onlyAsk is the key of the one ask in the gate's bucket asks, or empty. func onlyAsk(b *testBus) string { keys, err := b.kvs["asks"].Keys(context.Background()) -- 2.54.0 From 8484d349aa5f4cda641474660092cdbb63f79d69 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:45:50 +0100 Subject: [PATCH 061/243] The gate says record-opened when it opens a delivery record (milestone 47/G, W9.3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 649fa9b4.) Builds: design 47 G §304, §305 (GATE) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/record.go | 15 +++++++++++++++ .../cmd/mesh-delivery-gate/record_test.go | 12 +++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 38b81225a..f99abaa03 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -21,6 +21,7 @@ import ( "sync" "time" + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/buscalls" ) @@ -498,10 +499,24 @@ func (rs *Records) Open(milestone string, heads map[string]string, forgeMade boo r, err = rs.Read(inst) if err == nil && r != nil { rs.keep(r) + // Said once, under the record's id: the watchman times it against 24 hours, ended by record-ended. A say that fails is + // said again by the next write of the record (the id keeps it once). + if serr := rs.say(meshdelivery.SubjectRecordOpened, recordOpened{Record: inst, Milestone: milestone, ForgeMade: forgeMade, At: now.UTC()}, + "record-opened."+inst); serr != nil && rs.Logf != nil { + rs.Logf("[mesh-delivery] the record %s is opened and its opening is not said: %v", inst, serr) + } } return r, err == nil && r != nil, err } +// recordOpened is the body of the seat's event record-opened. +type recordOpened struct { + Record string `json:"record"` + Milestone string `json:"milestone"` + ForgeMade bool `json:"forge-made,omitempty"` + At time.Time `json:"at"` +} + // List is every record kept, opened within the retention and a month more, oldest first. func (rs *Records) List() ([]*Record, error) { now := rs.now() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 87efb4b97..fcbb9336f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -290,6 +290,9 @@ func TestTheDeliveryRecordIsALedgerInstanceKeyedByMilestoneAndHeads(t *testing.T } var got []said rs := &Records{Ledger: l, Now: func() time.Time { return at }, Say: func(subject string, body any, id string) error { + if subject == meshdelivery.SubjectRecordOpened { + return nil + } got = append(got, said{subject, id, body.(meshdelivery.Pins)}) return nil }} @@ -694,7 +697,11 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { return rs, r, &said } t.Run("gate_record and gate_records read a record whole and its fill", func(t *testing.T) { - rs, r, _ := setup(t) + rs, r, said := setup(t) + // W9.3: the record's opening is said once, under its id, for the watchman to time against 24 hours. + if !slices.Contains(*said, meshdelivery.SubjectRecordOpened+" record-opened."+r.Instance) { + t.Fatalf("the record's opening is not said: %v", *said) + } got, err := rs.RecordOf(r.Instance, "") if err != nil || len(got) != 1 || got[0].Record.Milestone != "47/G" || got[0].Waits != "waiting for mesh-x on ace" || got[0].Bytes == 0 || got[0].Of != 256<<10 { @@ -769,6 +776,9 @@ func TestTheRecordsLedgerSaysFillingFullAndRoom(t *testing.T) { } saying := func(out *[]said) SeatSay { return func(subject string, body any, id string) error { + if subject == meshdelivery.SubjectRecordOpened { + return nil // the opening of a record, not a fill + } f, _ := body.(meshdelivery.Fill) *out = append(*out, said{subject, id, f}) return nil -- 2.54.0 From 85023b3b525407b268be4cba2df33ba52211e0fc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:45:57 +0100 Subject: [PATCH 062/243] gate_record_repair asks the owners' read verbs for the outcomes a record's node entries lack and applies them as the consumer would (milestone 47/G, gate_record_repair) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 46d3053c.) Builds: design 47 G §159 (GATE13) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/manifest_test.go | 2 +- .../cmd/mesh-delivery-gate/record_test.go | 61 ++++++ .../cmd/mesh-delivery-gate/recordrepair.go | 195 ++++++++++++++++++ .../cmd/mesh-delivery-gate/recordtools.go | 15 ++ modules/mesh-delivery-gate/module.json | 3 + 5 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordrepair.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 1575a13af..141ad05c0 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -161,7 +161,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { // bridge's (ADR 0437 §8). It uses operator-channel only to say its conditions (ADR 0437 §9, issue 1275): the teller asks // (ADR 0437 §6). want := []string{"seat:git.pull", "seat:git.statuses", "seat:git.open-pulls", "seat:git.contains", "seat:mesh-hand-acts.hand-act", - TrackerListVerb, tellerAnswersTool, viewReachTool} + TrackerListVerb, WantsEntryVerb, EngineCompositionVerb, tellerAnswersTool, viewReachTool} if !reflect.DeepEqual(m.Uses, []string{"operator-channel"}) { t.Fatalf("uses %v: operator-channel, for its conditions alone", m.Uses) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index fcbb9336f..9d72a2777 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -742,6 +742,51 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { t.Fatal("a record ended is ended again") } }) + t.Run("gate_record_repair asks the owners for the outcomes an entry lacks, applies what they answer, and ends the record", func(t *testing.T) { + rs, r, said := setup(t) + owners := &ownersFake{wanted: "src2:f1", fp: "src2:f1", phase: "running"} + log := &handActsFake{} + if _, err := rs.Repair(r.Instance, "", "jochen@shanks", owners, log); err == nil { + t.Fatal("repaired with no reason") + } + log.refuse = errors.New("no responders") + if _, err := rs.Repair(r.Instance, "the events were lost", "jochen@shanks", owners, log); err == nil || len(owners.asked) != 0 { + t.Fatalf("repaired though the hand-act log refused it: %v, asked %v", err, owners.asked) + } + log.refuse = nil + // The owner says the module runs another fingerprint: the entry is left. + owners.fp = "src2:other" + got, err := rs.Repair(r.Instance, "the events were lost", "jochen@shanks", owners, log) + if err != nil || got["ended"] != false { + t.Fatalf("%v %v", got, err) + } + if back, _ := rs.Read(r.Instance); back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "ace", StepRunning)].Outcome != "" { + t.Fatalf("an outcome the owner did not answer was applied: %+v", back.Outcomes) + } + // The owners answer: wanted and running are applied as the consumer would, the record ends, and the steps with no read + // verb are named. + owners.fp = "src2:f1" + got, err = rs.Repair(r.Instance, "the events were lost", "jochen@shanks", owners, log) + if err != nil || got["ended"] != true { + t.Fatalf("%v %v", got, err) + } + back, _ := rs.Read(r.Instance) + if back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "ace", StepRunning)].Outcome != OutcomeRunning || + back.Outcomes[OutcomeKey("mesh-x", "src2:f1", "ace", StepWanted)].Outcome != OutcomeWanted || back.Ended == nil { + t.Fatalf("outcomes %+v ended %+v", back.Outcomes, back.Ended) + } + if unread := got["not-readable-yet"].([]string); len(unread) != 4 || !strings.HasPrefix(unread[0], "built:") { + t.Fatalf("not readable yet: %v", unread) + } + if !slices.Contains(*said, meshdelivery.SubjectRecordEnded+" record-ended."+r.Instance) || len(log.took) != 2 { + t.Fatalf("said %v, hand acts %d", *said, len(log.took)) + } + // A repair repeated changes nothing. + again, _ := rs.Repair(r.Instance, "again", "jochen@shanks", owners, log) + if lines := again["lines"].([]repairLine); len(lines) != 0 { + t.Fatalf("a second repair did something: %+v", lines) + } + }) t.Run("a removal that keeps the record's operations is an error", func(t *testing.T) { was := ledgerCall t.Cleanup(func() { ledgerCall = was }) @@ -877,3 +922,19 @@ func TestTheRecordsLedgerSaysFillingFullAndRoom(t *testing.T) { } }) } + +// ownersFake answers the owners' reads: the fingerprint the assignment graph names, and the fingerprint and phase of the have. +type ownersFake struct { + wanted, fp, phase string + asked []string +} + +func (o *ownersFake) Wanted(node, module string) (string, error) { + o.asked = append(o.asked, "wanted "+node+" "+module) + return o.wanted, nil +} + +func (o *ownersFake) Running(node, module string) (string, string, error) { + o.asked = append(o.asked, "running "+node+" "+module) + return o.fp, o.phase, nil +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordrepair.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordrepair.go new file mode 100644 index 000000000..221143c32 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordrepair.go @@ -0,0 +1,195 @@ +package main + +// gate_record_repair (design 47 *The delivery record*, its tools against the four lines): for one record, asks each owner for +// the outcome of each entry that lacks one, through that owner's read verb, and applies the answers as the consumer would. The +// reads that exist as seat verbs are the seat mesh-wants' `entry` (is the node's entry at the fingerprint: wanted) and the seat +// node-engine's `composition` of that node (does the module run at the fingerprint: running). The indexer's take-in read, the +// builder's build read and the checker's check read are no seat verb yet: an entry missing one of those steps is said as +// unreadable, naming the owner, and is left to its event. + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + + "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" +) + +// The verbs the repair reads through (the manifest invokes them). +const ( + WantsEntryVerb = "seat:mesh-wants.entry" + EngineCompositionVerb = "seat:node-engine.composition" +) + +// StepWanted is the step whose outcome OutcomeWanted says the assignment graph names the entry on its node. +const StepWanted = "wanted" + +// ownerReads are the reads of the owners the repair asks. +type ownerReads interface { + // Wanted is the fingerprint the assignment graph names for the module on the node; empty when it names none. + Wanted(node, module string) (fingerprint string, err error) + // Running is the fingerprint and phase of the module's have on the node, from the node-engine seat's verb composition. + Running(node, module string) (fingerprint, phase string, err error) +} + +// seatOwners reads the owners through their seats. +type seatOwners struct{ ask asking } + +func (s seatOwners) call(verb string, in map[string]any) (json.RawMessage, error) { + raw, err := s.ask(verb, in) + if err != nil { + return nil, err + } + answer, output, ok := answerOf(raw) + if !ok { + return nil, fmt.Errorf("%s refused: %s", verb, lastLine(output)) + } + return answer, nil +} + +func (s seatOwners) Wanted(node, module string) (string, error) { + answer, err := s.call(WantsEntryVerb, map[string]any{"node": node, "module": module}) + if err != nil { + return "", err + } + var a struct { + Entry struct { + Fp string `json:"fp"` + } `json:"entry"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return "", fmt.Errorf("the entry of %s on %s is not readable: %s", module, node, clip(string(answer), 200)) + } + return a.Entry.Fp, nil +} + +func (s seatOwners) Running(node, module string) (string, string, error) { + answer, err := s.call(EngineCompositionVerb+"@"+node, map[string]any{"module": module}) + if err != nil { + return "", "", err + } + var a struct { + Have *struct { + Fp string `json:"fp"` + Phase string `json:"phase"` + } `json:"have"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return "", "", fmt.Errorf("the composition of %s on %s is not readable: %s", module, node, clip(string(answer), 200)) + } + if a.Have == nil { + return "", "", nil + } + return a.Have.Fp, a.Have.Phase, nil +} + +// unreadable are the steps whose owners offer no read verb yet, with the owner and the read it lacks. +var unreadable = map[string]string{ + OutcomeTakenIn: "the indexer has no read of what it took in", + OutcomeBuilt: "the builder has no read of a build by entry", + OutcomeChecked: "the checker has no read of a check by entry", + OutcomeComposed: "the assigner has no read of a composition by entry", +} + +// repairLine is what the repair did for one entry and step. +type repairLine struct { + Entry string `json:"entry"` + Step string `json:"step"` + Did string `json:"did"` + Because string `json:"because,omitempty"` +} + +// Repair asks the owners for each outcome the record's node entries lack, applies what they answer as the consumer would +// (an outcome with its why naming the read), and ends the record if every present node then runs its entries. Entries on +// no node (the catalog half's) are read for the steps no node-less owner offers: said unreadable. +func (rs *Records) Repair(instance, why, by string, owners ownerReads, acts handActs) (map[string]any, error) { + if strings.TrimSpace(why) == "" { + return nil, errors.New("say why the record is repaired by hand") + } + r, err := rs.Read(instance) + if err != nil { + return nil, err + } + if r == nil { + return nil, fmt.Errorf("no delivery record %s is kept", instance) + } + if owners == nil { + return nil, errors.New("the owners cannot be asked from here, so nothing was repaired") + } + if acts == nil { + return nil, errors.New("the hand-act log cannot be reached here, so nothing was repaired") + } + if _, err := acts.Record(meshhandactsclient.Request{What: "asked the owners for the outcomes the delivery record " + instance + " lacks", + Why: why, Cause: "gate_record_repair", Verb: "record-repair"}); err != nil { + return nil, fmt.Errorf("the hand act was not recorded through mesh-hand-acts, so nothing was repaired: %w", err) + } + keys := make([]string, 0, len(r.Entries)) + for k := range r.Entries { + keys = append(keys, k) + } + sort.Strings(keys) + var lines []repairLine + apply := func(e RecordEntry, step, outcome, because string) { + o := RecordOutcome{Module: e.Module, Fingerprint: e.Fingerprint, Node: e.Node, Step: step, Outcome: outcome, + Why: because, At: rs.Now(), Event: "repair." + OutcomeKey(e.Module, e.Fingerprint, e.Node, step)} + applied, err := rs.Outcome(r, o) + switch { + case err != nil: + lines = append(lines, repairLine{Entry: e.Key, Step: step, Did: "failed", Because: err.Error()}) + case applied: + lines = append(lines, repairLine{Entry: e.Key, Step: step, Did: "applied", Because: because}) + } + } + for _, k := range keys { + e := r.Entries[k] + if e.Node == "" { + continue + } + for _, step := range []string{StepWanted, StepRunning} { + if _, has := r.Outcomes[OutcomeKey(e.Module, e.Fingerprint, e.Node, step)]; has { + continue + } + switch step { + case StepWanted: + fp, err := owners.Wanted(e.Node, e.Module) + switch { + case err != nil: + lines = append(lines, repairLine{Entry: k, Step: step, Did: "unread", Because: err.Error()}) + case fp != e.Fingerprint: + lines = append(lines, repairLine{Entry: k, Step: step, Did: "left", Because: fmt.Sprintf("the assignment graph names %q on %s", fp, e.Node)}) + default: + apply(e, step, OutcomeWanted, "read from the seat mesh-wants: the entry of "+e.Module+" on "+e.Node+" names this fingerprint") + } + case StepRunning: + fp, phase, err := owners.Running(e.Node, e.Module) + switch { + case err != nil: + lines = append(lines, repairLine{Entry: k, Step: step, Did: "unread", Because: err.Error()}) + case fp != e.Fingerprint || phase != "running": + lines = append(lines, repairLine{Entry: k, Step: step, Did: "left", Because: fmt.Sprintf("its have on %s is %q at %q", e.Node, phase, fp)}) + default: + apply(e, step, OutcomeRunning, "read from the seat node-engine's composition on "+e.Node+": its have runs this fingerprint") + } + } + } + } + steps := make([]string, 0, len(unreadable)) + for s := range unreadable { + steps = append(steps, s) + } + sort.Strings(steps) + var unread []string + for _, s := range steps { + unread = append(unread, s+": "+unreadable[s]) + } + ended := false + if r.Ended == nil { + if ended, err = rs.EndIfRun(r); err != nil { + return nil, err + } + } + rs.keep(r) + return map[string]any{"instance": instance, "by": by, "lines": lines, "ended": ended, "not-readable-yet": unread, "record": viewOf(r)}, nil +} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go index 4941ed254..6d4a6cff8 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordtools.go @@ -179,6 +179,21 @@ func recordTools(h *Holder, acts handActs) []stdio.Tool { } return map[string]any{"families": h.Consumer.Heard()}, nil }}, + {Name: "gate_record_repair", + Description: "Do by hand what the consumer does itself, for one delivery record: ask the owners for the outcome of each " + + "node entry that lacks one, through their read verbs (the seat mesh-wants' entry for the wanted step, the seat " + + "node-engine's composition of that node for the running step), and apply the answers as the consumer would; an " + + "entry whose owner says otherwise is left, a read that failed is said, and the record ends when every present node " + + "runs its entries. The steps whose owners offer no read verb yet are named. Recorded in the hand-act log first.", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "instance": str("the record's instance"), "why": str("why it is repaired by hand, in plain words")}, + "required": []string{"instance", "why"}}, + Run: func(a map[string]any) (any, error) { + if h.Records == nil { + return none() + } + return h.Records.Repair(strArg(a, "instance"), strArg(a, "why"), stdio.CallerOf(a), seatOwners{ask: stdio.Ask}, acts) + }}, {Name: "gate_record_end", Description: "End a delivery record left open by hand (a node gone for good, an entry that never runs), with the " + "reason; recorded in the hand-act log of the seat mesh-hand-acts first, and said as record-ended.", diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 3c663d2a2..141daa397 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -298,6 +298,8 @@ "seat:git.contains", "seat:mesh-hand-acts.hand-act", "seat:issue-tracker.list", + "seat:mesh-wants.entry", + "seat:node-engine.composition", "mesh-delivery-teller.teller_answers", "mesh-delivery-view.view_reach" ], @@ -336,6 +338,7 @@ "gate_consumer", "gate_record", "gate_record_end", + "gate_record_repair", "gate_records" ], "config": { -- 2.54.0 From 6f7d070769c5168bd28b3a0c24219a0a6f93e910 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:46:10 +0100 Subject: [PATCH 063/243] M1-M5: the router holds back the asker's oldest ask behind a fourth and offers it again when one ends; the seat operator-channel's ask-again offers it now (milestone 47/G, issue 1966) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KK8HD1gufnqyFxMFdW ask-again is the seat operator-channel's verb (key in; state or error out), not the module tool messenger_ask_again, and records no hand act (G §394, §450). (Joined from 681975ec.) Builds: design 47 G §40, §83, §403 (RTR1, RTR2) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../messenger/cmd/messenger/answers_test.go | 7 +- modules/messenger/cmd/messenger/asks.go | 47 +++-- modules/messenger/cmd/messenger/held_test.go | 10 +- modules/messenger/cmd/messenger/holdback.go | 197 ++++++++++++++++++ .../messenger/cmd/messenger/holdback_test.go | 113 ++++++++++ modules/messenger/cmd/messenger/main.go | 11 +- .../messenger/cmd/messenger/manifest_test.go | 2 +- modules/messenger/go.mod | 2 + modules/messenger/module.json | 6 +- 9 files changed, 371 insertions(+), 24 deletions(-) create mode 100644 modules/messenger/cmd/messenger/holdback.go create mode 100644 modules/messenger/cmd/messenger/holdback_test.go diff --git a/modules/messenger/cmd/messenger/answers_test.go b/modules/messenger/cmd/messenger/answers_test.go index b02a26ec4..be28d5c0e 100644 --- a/modules/messenger/cmd/messenger/answers_test.go +++ b/modules/messenger/cmd/messenger/answers_test.go @@ -592,12 +592,15 @@ func TestAnAskIsRefusedInWordsToItsAsker(t *testing.T) { } } all := strings.Join(refused, "\n") - for _, want := range []string{"d-1: the ask is refused: an ask that destroys lasts at most", "d-2: its words carry", - "d-6: its asker already has 3 open asks"} { + // A fourth offered ask is never refused (ADR 0666 §12): it is taken and the asker's oldest is held back. + for _, want := range []string{"d-1: the ask is refused: an ask that destroys lasts at most", "d-2: its words carry"} { if !strings.Contains(all, want) { t.Errorf("not refused: %q in\n%s", want, all) } } + if strings.Contains(all, "d-6") || store.asks["mesh-delivery.d-6"].State != StateOpen || store.asks["mesh-delivery.d-3"].State != StateHeldBack { + t.Errorf("the fourth ask was refused or the oldest not held back: d-6 %s, d-3 %s", store.asks["mesh-delivery.d-6"].State, store.asks["mesh-delivery.d-3"].State) + } if _, kept := store.asks["mesh-delivery.d-1"]; kept { t.Error("a refused ask was recorded") } diff --git a/modules/messenger/cmd/messenger/asks.go b/modules/messenger/cmd/messenger/asks.go index 4e0a9c5ee..0cba1d7b7 100644 --- a/modules/messenger/cmd/messenger/asks.go +++ b/modules/messenger/cmd/messenger/asks.go @@ -65,6 +65,9 @@ type AskRecord struct { Warrant *oc.Warrant `json:"warrant,omitempty"` // Told: the warrant, or the end, was published to the asker. Told bool `json:"told,omitempty"` + // HeldBackSince is, while the ask is held back, since when: kept open behind newer asks of its asker, its tickets + // withdrawn, offered again when one of its asker's offered asks ends (novox/hq ADR 0666 §12). + HeldBackSince time.Time `json:"held-back-since,omitempty"` // HeldSince and HeldWhy say, while the ask is held, since when and why no channel can carry it: per kind, the // reason from the judgement that judges an answer. The asker reads them in this record, as it reads // the state (the asks protocol has no word for a held ask on `decided`, which means an ask ended). @@ -558,23 +561,24 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { h.refuseAsk(asker, a, "its whole words carry "+refusal.String()+", which may not leave the mesh on any channel: withhold it") return nil } + // A fourth offered ask is taken, never refused: the asker's oldest offered one is held back once this one is recorded + // (holdBackOldest). An ask with the same `about` replaces the open one, a held-back one too. open, replaced := 0, "" h.mu.Lock() for k, e := range h.asks { - if e.rec.Asker != asker || (e.rec.State != StateOpen && e.rec.State != StateHeld) { + if e.rec.Asker != asker || !liveState(e.rec.State) { continue } if a.About != "" && e.rec.Ask.About == a.About { replaced = k continue } - open++ + if e.rec.State != StateHeldBack { + open++ + } } h.mu.Unlock() - if open >= occ.MostOpen { - h.refuseAsk(asker, a, fmt.Sprintf("its asker already has %d open asks", open)) - return nil - } + holdBack := open >= occ.MostOpen rec := AskRecord{Asker: asker, Ask: a, State: StateOpen, Opened: now, Tickets: h.ticketsFor(a)} if len(rec.Tickets) == 0 { // Held, not refused (ADR 0397): the ask breaks no rule, and the operator is asked when a channel can carry it. @@ -605,6 +609,9 @@ func (h *Holder) TakeAsk(verb, asker string, body json.RawMessage) error { } else { h.keepAsk(key, rec, 0) } + if holdBack { + h.holdBackOldest(asker, key) + } if rec.State == StateHeld { h.Logf("[messenger] %s asks %q (%s): held, not shown: %s", asker, a.Headline, a.ID, rec.HeldWhy) return nil @@ -897,12 +904,14 @@ func (h *Holder) closeAsk(key string, outcome string, words string, w *oc.Warran rec, rev = *got, r } } - if rec.State != StateOpen && rec.State != StateHeld { + if !liveState(rec.State) { h.keepAsk(key, rec, rev) return ErrChanged } now := h.Now() + freed := rec.State == StateOpen || rec.State == StateHeld // a slot of the asker's offered asks, unless a newer ask takes it rec.State = string(outcome) + rec.HeldBackSince = time.Time{} rec.HeldWhy = "" rec.Closed = now if t, ok := rec.Tickets[used]; ok { @@ -926,6 +935,9 @@ func (h *Holder) closeAsk(key string, outcome string, words string, w *oc.Warran rev = newRev } h.keepAsk(key, rec, rev) + if freed && outcome != occ.OutcomeReplaced { + defer h.offerHeldBack(rec.Asker) + } h.sayClosed(key, rec, outcome, words) if rec.Asker == SelfAsker { // The router's own ask: it does what was decided, and tells nobody else (selfact.go). @@ -1016,7 +1028,7 @@ func (h *Holder) endLeftovers() { if r.Origin != "ask" || r.Ask == "" || !r.Cleared.IsZero() { continue } - if e := h.asks[r.Ask]; e != nil && e.rec.State != StateOpen && e.rec.Record == r.Key { + if e := h.asks[r.Ask]; e != nil && !liveState(e.rec.State) && e.rec.Record == r.Key { ended = append(ended, e.rec) } } @@ -1081,9 +1093,9 @@ func (h *Holder) sweepAsks(now time.Time) { var expired, untold []string for k, e := range h.asks { switch { - case (e.rec.State == StateOpen || e.rec.State == StateHeld) && !now.Before(askExpires(e.rec.Ask)): + case liveState(e.rec.State) && !now.Before(askExpires(e.rec.Ask)): expired = append(expired, k) - case e.rec.State != StateOpen && e.rec.State != StateHeld && !e.rec.Told && e.rec.Warrant != nil: + case !liveState(e.rec.State) && !e.rec.Told && e.rec.Warrant != nil: untold = append(untold, k) } } @@ -1195,6 +1207,8 @@ func (h *Holder) judge(kind string, c oc.Choice, key string, root func(KindInfo) return "chosen", nil } switch { + case rec.State == StateHeldBack: + return h.refuseChoice(kind, c, "the question waits behind newer questions of its asker, and is offered again when one of them ends") case !issued: return h.refuseChoice(kind, c, "it belongs to no ask this router holds") case t.Kind != kind: @@ -1386,6 +1400,10 @@ type AskView struct { // HeldSince and HeldWhy: while the ask is held, since when and why no channel can carry it yet (ADR 0397 §4). HeldSince string `json:"held_since,omitempty"` HeldWhy string `json:"held_why,omitempty"` + // HeldBackSince and Behind: while the ask is held back (ADR 0666 §12), since when and the keys of the newer asks of its + // asker it waits behind, in the names of the seat operator-channel's asks answer. + HeldBackSince string `json:"held-since,omitempty"` + Behind []string `json:"behind,omitempty"` } // AsksNow is every ask, open first, newest first. @@ -1400,6 +1418,9 @@ func (h *Holder) AsksNow() []AskView { if e.rec.State == StateHeld { v.HeldSince, v.HeldWhy = e.rec.HeldSince.UTC().Format(time.RFC3339), e.rec.HeldWhy } + if e.rec.State == StateHeldBack { + v.HeldBackSince, v.Behind = e.rec.HeldBackSince.UTC().Format(time.RFC3339), h.behindLocked(k, e.rec) + } kinds := map[string]bool{} for _, t := range e.rec.Tickets { kinds[t.Kind] = true @@ -1418,7 +1439,7 @@ func (h *Holder) AsksNow() []AskView { out = append(out, v) } sort.Slice(out, func(i, j int) bool { - oi, oj := out[i].State == StateOpen || out[i].State == StateHeld, out[j].State == StateOpen || out[j].State == StateHeld + oi, oj := liveState(out[i].State), liveState(out[j].State) if oi != oj { return oi } @@ -1479,7 +1500,7 @@ func (h *Holder) sayClosed(key string, rec AskRecord, outcome string, words stri } } -// AsksQuery narrows what the tool `operator-channel.asks` answers (issue 595): Key one ask, State open (open and held) or +// AsksQuery narrows what the tool `operator-channel.asks` answers (issue 595): Key one ask, State open (open, held and held-back) or // ended (all takes both), Since the asks opened at or after that time, Bound the most bytes of the answer's asks. With no // argument at all the query is the open asks within DefaultAsksBound (issue 1974), and the answer says so. type AsksQuery struct { @@ -1543,7 +1564,7 @@ func (q AsksQuery) Narrow(all []AskView) AsksAnswer { } var kept []AskView for _, v := range all { - open := v.State == StateOpen || v.State == StateHeld + open := liveState(v.State) switch { case q.Key != "" && v.Key != q.Key, q.State == "open" && !open, diff --git a/modules/messenger/cmd/messenger/held_test.go b/modules/messenger/cmd/messenger/held_test.go index b21b916a3..b04abc95f 100644 --- a/modules/messenger/cmd/messenger/held_test.go +++ b/modules/messenger/cmd/messenger/held_test.go @@ -105,17 +105,17 @@ func TestAHeldAskExpiresAsAnyOpenAskDoes(t *testing.T) { } } -func TestAHeldAskCountsTowardTheOpenAsksOfItsAsker(t *testing.T) { - h, _, _, c, _, w := newRouter(t) +func TestAHeldAskCountsTowardTheOffersOfItsAskerAndTheOldestIsHeldBack(t *testing.T) { + h, _, _, c, store, w := newRouter(t) withoutVerifiedSender(h) for i := 1; i <= 4; i++ { a := heldAsk(c, "d-"+string(rune('0'+i))) a.About = "about-" + string(rune('0'+i)) submit(t, h, "mesh-delivery", a) + c.pass(time.Second) } - got := w.of("mesh-delivery") - if len(got) != 1 || got[0].Outcome != occ.OutcomeRefused || !strings.Contains(got[0].Words, "already has 3 open asks") { - t.Fatalf("%+v", got) + if got := w.of("mesh-delivery"); len(got) != 0 || store.asks["mesh-delivery.d-1"].State != StateHeldBack || store.asks["mesh-delivery.d-4"].State != StateHeld { + t.Fatalf("%+v: a fourth is taken, never refused; the oldest is held back (%s)", got, store.asks["mesh-delivery.d-1"].State) } } diff --git a/modules/messenger/cmd/messenger/holdback.go b/modules/messenger/cmd/messenger/holdback.go new file mode 100644 index 000000000..b9aa3a018 --- /dev/null +++ b/modules/messenger/cmd/messenger/holdback.go @@ -0,0 +1,197 @@ +package main + +import ( + "fmt" + oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + "sort" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" +) + +// An ask held back (novox/hq ADR 0666 §12, design 46 § 6): an asker has at most MostOpen asks offered at once; a fourth +// is offered and nothing is refused, so no ask is hidden: the asker's oldest offered ask is held back, its tickets +// withdrawn and its message edited to say it waits behind newer asks, and it is offered again, with new tickets, when +// one of the asker's offered asks ends. `held` is another state, an ask no ready channel can carry (ADR 0397). + +// StateHeldBack is an ask kept open behind newer asks of its asker. +const StateHeldBack = "held-back" + +// liveState is whether an ask has not ended: offered, held for want of a channel, or held back. +func liveState(state string) bool { + return state == StateOpen || state == StateHeld || state == StateHeldBack +} + +// behindLocked are the keys of the newer offered asks of the asker of rec, oldest first; h.mu is held. +func (h *Holder) behindLocked(key string, rec AskRecord) []string { + var behind []string + for k, e := range h.asks { + if k != key && e.rec.Asker == rec.Asker && (e.rec.State == StateOpen || e.rec.State == StateHeld) && e.rec.Opened.After(rec.Opened) { + behind = append(behind, k) + } + } + sort.Slice(behind, func(i, j int) bool { + a, b := h.asks[behind[i]].rec, h.asks[behind[j]].rec + if !a.Opened.Equal(b.Opened) { + return a.Opened.Before(b.Opened) + } + return behind[i] < behind[j] + }) + return behind +} + +// pickLocked is the key of the asker's asks in one of the states, the oldest first (by opening, then key); "" when +// there is none, never except; h.mu is held. +func (h *Holder) pickLocked(asker, except string, states ...string) string { + pick := "" + for k, e := range h.asks { + if e.rec.Asker != asker || k == except { + continue + } + in := false + for _, s := range states { + in = in || e.rec.State == s + } + if !in { + continue + } + if pick == "" || e.rec.Opened.Before(h.asks[pick].rec.Opened) || (e.rec.Opened.Equal(h.asks[pick].rec.Opened) && k < pick) { + pick = k + } + } + return pick +} + +// offeredCount is how many of the asker's asks are offered now: open, or held for want of a channel. +func (h *Holder) offeredCount(asker string) int { + h.mu.Lock() + defer h.mu.Unlock() + n := 0 + for _, e := range h.asks { + if e.rec.Asker == asker && (e.rec.State == StateOpen || e.rec.State == StateHeld) { + n++ + } + } + return n +} + +// freshAsk is an ask's record read again from the store, where there is one. +func (h *Holder) freshAsk(key string) (AskRecord, uint64, bool) { + e := h.askOf(key) + if e == nil { + return AskRecord{}, 0, false + } + rec, rev := e.rec, e.rev + if h.Asks != nil { + if got, r, err := h.Asks.GetAsk(key); err == nil && got != nil { + rec, rev = *got, r + } + } + return rec, rev, true +} + +// holdBackOldest holds back the asker's oldest offered ask other than except (the newest, just taken): its tickets +// are withdrawn, it stays open, and its own message is edited to say it waits. One held for want of a channel counts as offered +// and is held back the same way. The work lock is held. +func (h *Holder) holdBackOldest(asker, except string) { + h.mu.Lock() + key := h.pickLocked(asker, except, StateOpen, StateHeld) + h.mu.Unlock() + if key == "" { + return + } + rec, rev, ok := h.freshAsk(key) + if !ok || (rec.State != StateOpen && rec.State != StateHeld) { + return + } + now := h.Now() + rec.State, rec.HeldBackSince, rec.Tickets, rec.HeldSince, rec.HeldWhy = StateHeldBack, now, nil, time.Time{}, "" + if !h.saveHeld(key, rec, rev) { + return + } + h.Logf("[messenger] %s's ask %q (%s) is held back behind its newer asks: it is offered again when one ends", asker, rec.Ask.Headline, rec.Ask.ID) + h.mu.Lock() + r := h.open[rec.Record] + h.mu.Unlock() + if r == nil || r.Origin != "ask" { + return // a condition's message is the newer ask's to show + } + h.mu.Lock() + r.Answered = "Waits behind newer questions of " + asker + ".\nIt is offered again, with its answers, when one of them ends." + h.mu.Unlock() + h.edit(r, "answered", false) + h.persist(r) +} + +// offerHeldBack offers the asker's oldest held-back ask again when a slot is free: new tickets, its message edited back +// to the question. The work lock is held. +func (h *Holder) offerHeldBack(asker string) { + if h.offeredCount(asker) >= occ.MostOpen { + return + } + h.mu.Lock() + key := h.pickLocked(asker, "", StateHeldBack) + h.mu.Unlock() + if key != "" { + h.offerOneHeldBack(key) + } +} + +// offerOneHeldBack offers one held-back ask again now. +func (h *Holder) offerOneHeldBack(key string) { + rec, rev, ok := h.freshAsk(key) + if !ok || rec.State != StateHeldBack { + return + } + now := h.Now() + rec.HeldBackSince = time.Time{} + rec.Tickets = h.ticketsFor(rec.Ask) + rec.State = StateOpen + if len(rec.Tickets) == 0 { + rec.Tickets, rec.State, rec.HeldSince, rec.HeldWhy = nil, StateHeld, now, h.holdWhy(rec.Ask) + } + if !h.saveHeld(key, rec, rev) { + return + } + h.Logf("[messenger] %s's ask %q (%s) is offered again", rec.Asker, rec.Ask.Headline, rec.Ask.ID) + if rec.State != StateOpen { + return + } + h.mu.Lock() + r := h.open[rec.Record] + h.mu.Unlock() + if r != nil && r.Origin == "ask" { + h.mu.Lock() + r.Answered, r.Ask = "", key + h.mu.Unlock() + h.offerAsk(r) + h.persist(r) + return + } + rec.Record = h.recordFor(rec.Asker, key, rec.Ask) + h.showAsk(key, rec, now) +} + +// AskAgain is the seat operator-channel's verb ask-again (G §394, §450): offer a held-back ask now, holding back the +// asker's oldest offered ask in its place when the asker already has the most offered. It answers the ask's state after it, +// or the error that says why it was not offered. +func (h *Holder) AskAgain(key string) oc.AskAgainAnswer { + h.work.Lock() + defer h.work.Unlock() + h.mu.Lock() + h.init() + h.mu.Unlock() + rec, _, ok := h.freshAsk(key) + switch { + case !ok: + return oc.AskAgainAnswer{Error: fmt.Sprintf("this router holds no ask %q: its key is ., as operator-channel.asks lists", key)} + case rec.State != StateHeldBack: + return oc.AskAgainAnswer{Error: fmt.Sprintf("the ask %s is %s, not held back: only a held-back ask is offered again", key, rec.State)} + } + if h.offeredCount(rec.Asker) >= occ.MostOpen { + h.holdBackOldest(rec.Asker, key) + } + h.offerOneHeldBack(key) + got, _, _ := h.freshAsk(key) + return oc.AskAgainAnswer{State: string(got.State)} +} diff --git a/modules/messenger/cmd/messenger/holdback_test.go b/modules/messenger/cmd/messenger/holdback_test.go new file mode 100644 index 000000000..5b92f1d2f --- /dev/null +++ b/modules/messenger/cmd/messenger/holdback_test.go @@ -0,0 +1,113 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" +) + +// ADR 0666 §12, design 46 §6 and test 71: an asker has at most three asks offered at once; a fourth is offered and the +// oldest is held back (its tickets withdrawn, its message edited, its record open), offered again with new tickets +// when one of the three ends, offered now by messenger_ask_again; an ask with the same `about` replaces a held-back +// one, and a Decline ends an ask whose asker may ask again at once. +func TestNoAskIsHiddenTheOldestOfFourIsHeldBackAndOfferedAgain(t *testing.T) { + h, tg, _, c, store, w := newRouter(t) + link(h, store, c, "42", c.now().Add(-2*time.Hour)) + ask := func(id, about string) { + a := ackAsk(c, id) + a.About = about + submit(t, h, "mesh-delivery", a) + c.pass(time.Second) + settle(h, c) + } + state := func(id string) string { return store.asks["mesh-delivery."+id].State } + for i, id := range []string{"d-1", "d-2", "d-3"} { + ask(id, "about-"+string(rune('1'+i))) + } + oldTicket := "" + for tk := range store.asks["mesh-delivery.d-1"].Tickets { + oldTicket = tk + } + ask("d-4", "about-4") + // M1: the fourth reaches the channel, the oldest is held back, none is refused. + if len(w.of("mesh-delivery")) != 0 || state("d-4") != StateOpen || state("d-1") != StateHeldBack { + t.Fatalf("refused %+v; d-4 %s, d-1 %s", w.of("mesh-delivery"), state("d-4"), state("d-1")) + } + if rec := store.asks["mesh-delivery.d-1"]; len(rec.Tickets) != 0 || rec.HeldBackSince.IsZero() { + t.Errorf("held back %+v: tickets withdrawn, since set", rec) + } + if m, ok := tg.edits["1"]; !ok || len(m.Actions) != 0 || !strings.Contains(m.Note, "Waits behind newer questions of mesh-delivery") { + t.Errorf("its message was not edited to say it waits: %v %+v", ok, m) + } + if _, err := h.Chosen("telegram", choiceOf(oldTicket, "42", c.now())); err == nil || !strings.Contains(err.Error(), "waits behind newer questions") || len(w.of("mesh-delivery")) != 0 { + t.Errorf("a tap of a withdrawn ticket: %v %+v", err, w.of("mesh-delivery")) + } + // M3: asks lists it with since when and the newer asks it waits behind; open includes it. + q, _ := AsksQueryOf(map[string]any{"key": "mesh-delivery.d-1"}) + raw, _ := json.Marshal(q.Narrow(h.AsksNow())) + for _, want := range []string{`"state":"held-back"`, `"held-since":"`, `"behind":["mesh-delivery.d-2","mesh-delivery.d-3","mesh-delivery.d-4"]`} { + if !strings.Contains(string(raw), want) { + t.Errorf("asks lacks %s: %s", want, raw) + } + } + if q, _ = AsksQueryOf(nil); !strings.Contains(string(mustJSON(q.Narrow(h.AsksNow()))), "mesh-delivery.d-1") { + t.Error("the open asks leave out the held-back one") + } + // M5: an ask with the same about replaces a held-back one; the three offered stay three, so the oldest of them is held back. + ask("d-5", "about-1") + if state("d-1") != string(occ.OutcomeReplaced) || state("d-2") != StateHeldBack || state("d-5") != StateOpen || state("d-3") != StateOpen { + t.Fatalf("after d-5 about the held-back ask's thing: d-1 %s d-2 %s d-3 %s d-5 %s", state("d-1"), state("d-2"), state("d-3"), state("d-5")) + } + // M2: when one of the offered ends, the oldest held back is offered again with new tickets and its message asks again. + if a := h.AskAgain("mesh-delivery.d-3"); !strings.Contains(a.Error, "not held back") { + t.Errorf("ask-again of an offered ask: %+v", a) + } + if err := h.TakeAsk("cancel", "mesh-delivery", []byte(`{"id":"d-3"}`)); err != nil { + t.Fatal(err) + } + settle(h, c) + if rec := store.asks["mesh-delivery.d-2"]; rec.State != StateOpen || len(rec.Tickets) == 0 || !rec.HeldBackSince.IsZero() { + t.Fatalf("d-2 after d-3 ended: %+v", rec) + } + if m := tg.edits["2"]; len(m.Actions) == 0 || strings.Contains(m.Note, "Waits behind") { + t.Errorf("its message does not ask again: %+v", m) + } + // M4: ask-again by hand offers a held-back ask now, holding back the oldest offered one; the seat operator-channel's ask-again (G §450). + ask("d-6", "about-6") + ask("d-7", "about-7") + held := "" + for _, id := range []string{"d-2", "d-4", "d-5", "d-6", "d-7"} { + if state(id) == StateHeldBack { + held = id + } + } + if held == "" { + t.Fatal("no ask is held back after two more") + } + if a := h.AskAgain("mesh-delivery.d-none"); a.Error == "" || a.State != "" { + t.Errorf("ask-again of an ask the router holds none of: %+v", a) + } + if a := h.AskAgain("mesh-delivery." + held); a.Error != "" || a.State != string(StateOpen) || state(held) != StateOpen { + t.Fatalf("%s is %s; ask-again answered %+v", held, state(held), a) + } + n := 0 + for _, id := range []string{"d-2", "d-4", "d-5", "d-6", "d-7"} { + if state(id) == StateHeldBack { + n++ + } + } + if n != 2 || state("d-5") != StateHeldBack { + t.Errorf("%d asks held back after ask-again (d-5 %s): the oldest offered one took its place", n, state("d-5")) + } + // A Decline ends an ask; its asker may ask again at once, and nothing is refused. + for _, wt := range w.of("mesh-delivery") { + if wt.Outcome == occ.OutcomeRefused { + t.Errorf("refused: %+v", wt) + } + } +} + +func mustJSON(v any) []byte { raw, _ := json.Marshal(v); return raw } diff --git a/modules/messenger/cmd/messenger/main.go b/modules/messenger/cmd/messenger/main.go index fd4654725..e115da0f1 100644 --- a/modules/messenger/cmd/messenger/main.go +++ b/modules/messenger/cmd/messenger/main.go @@ -741,7 +741,7 @@ func tools(h *Holder, l *listening) []stdio.Tool { "most bytes of the answer.", Input: map[string]any{ "key": str("one ask, by its key (asker.id); none for the open asks"), - "state": str("open (open and held), ended or all; none is open when no key or since is given, else both"), + "state": str("open (open, held and held-back), ended or all; none is open when no key or since is given, else both"), "since": str("only the asks opened at or after this time, RFC 3339"), "bound": map[string]any{"type": "integer", "description": "the most bytes of the asks in the answer; said when it cuts the answer short"}}, Run: func(a map[string]any) (any, error) { @@ -751,6 +751,15 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return q.Narrow(h.AsksNow()), nil }}, + {Name: "operator-channel." + oc.VerbAskAgain, + Description: "Offer a held-back ask again now (G §394, §450): an asker has at most three asks offered at once, and a fourth " + + "holds its oldest back behind it (operator-channel.asks lists it with state held-back, since when and the newer asks it waits behind). " + + "It is offered again by itself when one of the asker's asks ends; this offers it now, holding back the asker's oldest offered ask in its " + + "place. Answers the ask's state, or why it was not offered (an ask that is not held back).", + Input: map[string]any{"key": str("the held-back ask, by its key (asker.id)")}, + Run: func(a map[string]any) (any, error) { + return h.AskAgain(strArg(a, "key")), nil + }}, {Name: "messenger_identities", Description: "The accounts linked as the operator, one per channel kind: the account, the name its " + "service shows, when it was linked, and from when it can approve (an hour after linking).", diff --git a/modules/messenger/cmd/messenger/manifest_test.go b/modules/messenger/cmd/messenger/manifest_test.go index ffb17331b..354ed6f63 100644 --- a/modules/messenger/cmd/messenger/manifest_test.go +++ b/modules/messenger/cmd/messenger/manifest_test.go @@ -68,7 +68,7 @@ func readManifest(t *testing.T) (manifest, string) { func TestItDeclaresTheSeatsOfAnAskAndHoldsTheOperatorChannel(t *testing.T) { // 1217/AC6: condition beside ask and cancel m, _ := readManifest(t) - verbs := []string{"open", "history", "notify", "tell", "asks"} + verbs := []string{"open", "history", "notify", "tell", "asks", oc.VerbAskAgain} want := []seat{ {Name: oc.Seat, Scope: "mesh", Serves: verbs, Accepts: []string{"ask", "cancel", "condition"}, Emits: []string{"decided"}, ByCaller: []string{"ask", "cancel", "condition", "decided"}, Records: []string{"asks"}}, diff --git a/modules/messenger/go.mod b/modules/messenger/go.mod index 00ad0328e..2d2a82112 100644 --- a/modules/messenger/go.mod +++ b/modules/messenger/go.mod @@ -3,6 +3,7 @@ module messenger go 1.26.0 require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 @@ -12,6 +13,7 @@ require ( ) require ( + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect github.com/antithesishq/antithesis-sdk-go v0.8.0-default-no-op // indirect github.com/google/go-tpm v0.9.8 // indirect github.com/klauspost/compress v1.20.1 // indirect diff --git a/modules/messenger/module.json b/modules/messenger/module.json index 7fb4aba5c..c66ff9074 100644 --- a/modules/messenger/module.json +++ b/modules/messenger/module.json @@ -12,7 +12,8 @@ "history", "notify", "tell", - "asks" + "asks", + "ask-again" ], "accepts": [ "ask", @@ -66,7 +67,8 @@ "history", "notify", "tell", - "asks" + "asks", + "ask-again" ] } ], -- 2.54.0 From 6d4ad2c2492e8b12b80671d6b249c3f3905a4fd4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:48:07 +0100 Subject: [PATCH 064/243] Test that the operator's Approve says accepted, by whom and on which warrant (issue 595, W15.1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 65c6ca40.) Builds: design 47 G §319, test 59 (KB1) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go index caea95c34..ac8379afb 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/review_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "fmt" "strings" "testing" "time" @@ -283,6 +284,12 @@ func TestARecordThatNeedsTheOperatorIsAcceptedOnlyOnTheWarrant(t *testing.T) { if last := vs[len(vs)-1]; !strings.Contains(last.Why, "warrant") || last.By != "the operator" { t.Fatalf("the accepting version: %+v", last) } + // Issue 595: a session waiting on the Approve wakes on this event (node-tools' `waits` names it); it says who + // accepted and on which warrant. + if got := w.said.of(EventAccepted); len(got) != 1 || got[0]["by"] != "the operator" || got[0]["record"] != itoaKey(n) || + !strings.Contains(fmt.Sprint(got[0]["what"]), "warrant") { + t.Fatalf("the accepted event: %v", got) + } if said, _ := w.k.Reconcile(); len(said) != 0 { t.Fatalf("decided twice: %v", said) } -- 2.54.0 From 4839884d936a099e07c1d082eb34835d4033cdb3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:48:07 +0100 Subject: [PATCH 065/243] A decision links the milestone that builds it: link kind built-in (milestone 47/G, W15.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0493 section 4 names the links; the knowledge base took no milestone as a target. Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from 5f4e4d29.) Builds: design 47 G §318 (KB1) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-knowledge/record.go | 18 +++++++++++++++--- .../mesh-knowledge/cmd/mesh-knowledge/rules.go | 4 ++++ .../cmd/mesh-knowledge/rules_test.go | 10 +++++++++- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/record.go b/modules/mesh-knowledge/cmd/mesh-knowledge/record.go index 1cc60e6e9..c2c48d6b0 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/record.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/record.go @@ -75,10 +75,12 @@ const ( LinkCites = "cites" LinkRestsOn = "rests-on" LinkTouches = "touches" + // LinkBuiltIn is from a decision the milestone that builds it, design/id such as 47/G (ADR 0493 §4, design 47). + LinkBuiltIn = "built-in" ) // LinkKinds are the kinds a link may be. -var LinkKinds = []string{LinkSupersedes, LinkChanges, LinkExtends, LinkBecame, LinkCites, LinkRestsOn, LinkTouches} +var LinkKinds = []string{LinkSupersedes, LinkChanges, LinkExtends, LinkBecame, LinkCites, LinkRestsOn, LinkTouches, LinkBuiltIn} // LinkMeanings says what each kind of link means, as the knowledge base's own review is told (hq issue 492): the // rules it sees list LinkKinds, which propose holds a link to, so the two cannot drift. @@ -90,12 +92,15 @@ var LinkMeanings = map[string]string{ LinkCites: "any record, issue or design it names", LinkRestsOn: "from a decision, a decision it stands on, which may not be superseded under it", LinkTouches: "from a research effort, a record, issue or design its effort bears on", + LinkBuiltIn: "from a decision, the milestone that builds it, named as milestone 47/G", } // What a link may point at besides a record of this knowledge base: an issue in the tracker, a design in hq. const ( TargetIssue = "issue" TargetDesign = "design" + // TargetMilestone is a milestone of the issue tracker, design/id; its Path holds that. + TargetMilestone = "milestone" ) // Key is a record's key: its kind and number together. @@ -166,6 +171,8 @@ func (r Ref) String() string { return fmt.Sprintf("issue %d", r.Number) case TargetDesign: return "design " + r.Path + case TargetMilestone: + return "milestone " + r.Path } return r.Kind } @@ -174,6 +181,8 @@ var ( refRecord = regexp.MustCompile(`^(?i)(?:(adr|decision|research)\s*)?(\d{1,4})(?:\s*§\s*(\d{1,3}))?$`) refIssue = regexp.MustCompile(`^(?i)issue\s*(\d{1,5})$`) refDesign = regexp.MustCompile(`^(?i)(?:design\s+)?(03-DESIGN/[0-9a-z/-]+\.md)$`) + // refMilestone is the tracker's name for a milestone: design/id, such as 47/G. + refMilestone = regexp.MustCompile(`^(?i)milestone\s+([0-9]{1,3}/[A-Za-z0-9][A-Za-z0-9_-]{0,31})$`) ) // ParseRef reads a link's target as an agent writes it: "0276", "ADR 0276 §4", "research 045", "issue 365", @@ -187,6 +196,9 @@ func ParseRef(s string) (Ref, error) { if m := refDesign.FindStringSubmatch(s); m != nil { return Ref{Kind: TargetDesign, Path: m[1]}, nil } + if m := refMilestone.FindStringSubmatch(s); m != nil { + return Ref{Kind: TargetMilestone, Path: m[1]}, nil + } if m := refRecord.FindStringSubmatch(s); m != nil { kind := KindDecision if strings.EqualFold(m[1], "research") { @@ -202,8 +214,8 @@ func ParseRef(s string) (Ref, error) { } return Ref{Kind: kind, Number: n, Clause: c}, nil } - return Ref{}, fmt.Errorf("%q is not a record (0276, ADR 0276 §4, research 045), an issue (issue 365) or a design "+ - "(03-DESIGN/01-to-be/55-the-knowledge-base.md)", s) + return Ref{}, fmt.Errorf("%q is not a record (0276, ADR 0276 §4, research 045), an issue (issue 365), a design "+ + "(03-DESIGN/01-to-be/55-the-knowledge-base.md) or a milestone (milestone 47/G)", s) } // Link is one typed link, held on the record it goes out from and read from both ends. From is the clause it diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/rules.go b/modules/mesh-knowledge/cmd/mesh-knowledge/rules.go index 0d93290cc..a893c47c2 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/rules.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/rules.go @@ -248,6 +248,10 @@ func JudgeRecordOn(r *Record, held *Entry, lookup Lookup, ref *Reference, day st if !decision || l.To.Kind != KindDecision || l.To.Clause > 0 { refuse("link-fits", "rests-on goes from a decision to a decision it stands on: not %s", l) } + case l.Kind == LinkBuiltIn: + if !decision || l.To.Kind != TargetMilestone { + refuse("link-fits", "built-in goes from a decision to the milestone that builds it, named as milestone 47/G: not %s", l) + } case l.Kind == LinkTouches: if r.Kind != KindResearch || l.To.Clause > 0 { refuse("link-fits", "touches goes from a research effort to a record, an issue or a design: not %s", l) diff --git a/modules/mesh-knowledge/cmd/mesh-knowledge/rules_test.go b/modules/mesh-knowledge/cmd/mesh-knowledge/rules_test.go index 5d39d5964..2e724cf38 100644 --- a/modules/mesh-knowledge/cmd/mesh-knowledge/rules_test.go +++ b/modules/mesh-knowledge/cmd/mesh-knowledge/rules_test.go @@ -54,6 +54,9 @@ func TestEveryPerRecordRuleRefusesAProposal(t *testing.T) { {"link-fits", func(p *ProposeAsk, w *world, t *testing.T) { p.Links = append(p.Links, LinkAsk{Kind: LinkBecame, To: "0276"}) }}, + {"link-fits", func(p *ProposeAsk, w *world, t *testing.T) { + p.Links = append(p.Links, LinkAsk{Kind: LinkBuiltIn, To: "0276"}) // built-in names a milestone, not a record + }}, {"self-link", func(p *ProposeAsk, w *world, t *testing.T) { p.Links = append(p.Links, LinkAsk{Kind: LinkCites, To: itoa4(p.Number)}) }}, @@ -165,11 +168,16 @@ func TestTheValidProposalIsTaken(t *testing.T) { eachStore(t, func(t *testing.T, s Store) { w := imported(t, s) n := w.reserved(t, "author@test") - got, err := w.k.Propose("author@test", valid(n)) + p := valid(n) + p.Links = append(p.Links, LinkAsk{Kind: LinkBuiltIn, To: "milestone 47/G"}) // a decision links its milestone + got, err := w.k.Propose("author@test", p) if err != nil { t.Fatal(err) } e := mustGet(t, s, Key{KindDecision, n}) + if l := e.Current.Links[len(e.Current.Links)-1]; l.Kind != LinkBuiltIn || l.To != (Ref{Kind: TargetMilestone, Path: "47/G"}) { + t.Fatalf("the milestone link was kept as %+v", l) + } if got.Status != StatusProposed || e.Current.Status != StatusProposed || len(e.Current.Clauses) != 2 || e.Current.Clauses[0].Status != ClauseProposed || e.Current.Author != "author@test" { t.Fatalf("%+v / %+v", got, e.Current) -- 2.54.0 From a7b83b27f07d4687bcea3acca2785823a4d04ac5 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:48:07 +0100 Subject: [PATCH 066/243] The issue tracker links a research effort by its number (milestone 47/G, W15.2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trace from research to the running build is links all the way (design 47). Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ (Joined from dd68c96d.) Builds: design 47 G §318 (ISS1) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-issues/cmd/mesh-issues/check.go | 14 ++++++++++---- modules/mesh-issues/cmd/mesh-issues/main.go | 2 +- .../mesh-issues/cmd/mesh-issues/tracker_test.go | 2 ++ 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/modules/mesh-issues/cmd/mesh-issues/check.go b/modules/mesh-issues/cmd/mesh-issues/check.go index c657d52b1..0df466e68 100644 --- a/modules/mesh-issues/cmd/mesh-issues/check.go +++ b/modules/mesh-issues/cmd/mesh-issues/check.go @@ -212,9 +212,12 @@ const ( LinkDependsOn = "depends-on" LinkPullRequest = "pull-request" LinkDecision = "decision" - LinkDesign = "design" - LinkCondition = "condition" - LinkDelivery = "delivery" + // LinkResearch is a research effort of the knowledge base, by its number (design 47: the trace from research to the + // running build is links all the way). + LinkResearch = "research" + LinkDesign = "design" + LinkCondition = "condition" + LinkDelivery = "delivery" // LinkMilestone is the one milestone an issue belongs to, design/id (ADR 0493 §1, §2). LinkMilestone = "milestone" // LinkCommit is a commit that fixes the issue, repository@full sha (ADR 0493 §4). @@ -222,7 +225,7 @@ const ( ) // LinkKinds is every kind, in the order they are told. -var LinkKinds = []string{LinkDuplicates, LinkRelatesTo, LinkDependsOn, LinkPullRequest, LinkDecision, LinkDesign, LinkCondition, LinkDelivery, +var LinkKinds = []string{LinkDuplicates, LinkRelatesTo, LinkDependsOn, LinkPullRequest, LinkDecision, LinkResearch, LinkDesign, LinkCondition, LinkDelivery, LinkMilestone, LinkCommit} // madeByPeople are the kinds only a person or a session links, never the triage (ADR 0493 §4). @@ -231,6 +234,7 @@ var madeByPeople = []string{LinkPullRequest, LinkMilestone, LinkCommit} var ( pullRequest = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,99}#[1-9][0-9]{0,6}$`) decision = regexp.MustCompile(`^[0-9]{4}$`) + research = regexp.MustCompile(`^[0-9]{3,4}$`) // deliveryID mirrors the delivery gate's `IDOf` (modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go): the forge's // repository as owner/name, lower-cased, `@` and at most twelve characters of its commit. The pattern this // replaced took neither the slash nor the `@`, so no delivery could be linked (novox/hq issue 475). @@ -280,6 +284,8 @@ func checkTarget(kind, target string) (string, error) { ok, shape = pullRequest.MatchString(t), "a repository and a pull request's number, such as mesh-host#246" case LinkDecision: ok, shape = decision.MatchString(t), "a decision record's four-digit number, such as 0275" + case LinkResearch: + ok, shape = research.MatchString(t), "a research effort's number, such as 085" case LinkDesign: ok, shape = designPath.MatchString(t), "a design document, such as 03-DESIGN/01-to-be/53-the-issue-tracker.md" case LinkCondition: diff --git a/modules/mesh-issues/cmd/mesh-issues/main.go b/modules/mesh-issues/cmd/mesh-issues/main.go index d211453d5..e4602f43e 100644 --- a/modules/mesh-issues/cmd/mesh-issues/main.go +++ b/modules/mesh-issues/cmd/mesh-issues/main.go @@ -616,7 +616,7 @@ func Tools(t *Tracker) []stdio.Tool { {Name: Seat + ".link", Description: "Link an issue to something, or with remove, take a link away. Kinds: duplicates or relates-to " + "another issue (its number), pull-request (such as mesh-host#246), decision (a decision record's " + - "number, such as 0275), design (a design document's path), condition (a condition's key) and delivery " + + "number, such as 0275), research (a research effort's number, such as 085), design (a design document's path), condition (a condition's key) and delivery " + "(a delivery's id, such as novox/mesh-host@d2e9dc61590f), milestone (the one milestone it belongs to, design/id such as " + "47/R6) and commit (a commit that fixes it, repository@full sha). Refused: a target not of its kind's shape, an issue that does not exist, a link it " + "already has, a second milestone (remove the first to move it), a milestone that does not exist or is closed.", diff --git a/modules/mesh-issues/cmd/mesh-issues/tracker_test.go b/modules/mesh-issues/cmd/mesh-issues/tracker_test.go index 55c29fc90..77155e53e 100644 --- a/modules/mesh-issues/cmd/mesh-issues/tracker_test.go +++ b/modules/mesh-issues/cmd/mesh-issues/tracker_test.go @@ -485,6 +485,7 @@ func TestLinksAreTypedAddedOnceAndRemoved(t *testing.T) { {Kind: LinkRelatesTo, Target: keyOf(b)}, {Kind: LinkPullRequest, Target: "mesh-controller#246"}, {Kind: LinkDecision, Target: "0275"}, + {Kind: LinkResearch, Target: "085"}, {Kind: LinkDesign, Target: "03-DESIGN/01-to-be/53-the-issue-tracker.md"}, {Kind: LinkCondition, Target: "mesh.hand-acts.backup.healer-wanted"}, {Kind: LinkDelivery, Target: "novox/mesh-controller@0f853e93fa11"}, @@ -505,6 +506,7 @@ func TestLinksAreTypedAddedOnceAndRemoved(t *testing.T) { {LinkAsk{Kind: LinkPullRequest, Target: "mesh-controller#246"}, "already has"}, {LinkAsk{Kind: LinkPullRequest, Target: "../x#1"}, "target is"}, {LinkAsk{Kind: LinkDecision, Target: "275"}, "four-digit"}, + {LinkAsk{Kind: LinkResearch, Target: "85"}, "research effort's number"}, {LinkAsk{Kind: LinkDesign, Target: "03-DESIGN/../../etc/passwd"}, "design document"}, {LinkAsk{Kind: LinkCondition, Target: "a key\nwith a line"}, "condition"}, {LinkAsk{Kind: LinkCondition, Target: "machine.ace.token=abcdefgh123.leaked"}, "secret"}, -- 2.54.0 From d3286ad0eb9252d63b722f7114054cd3573501b2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:53:43 +0100 Subject: [PATCH 067/243] BLD3, BLD4: the joined step test carries no names list, the lint's fixed-name rule being taken out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §7, §328, §329 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-build-go/buildstep/askrun_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/mesh-build-go/buildstep/askrun_test.go b/modules/mesh-build-go/buildstep/askrun_test.go index 693afbb83..173813ff0 100644 --- a/modules/mesh-build-go/buildstep/askrun_test.go +++ b/modules/mesh-build-go/buildstep/askrun_test.go @@ -379,8 +379,7 @@ func TestRunAskRunsAPipelineStepAndSaysItsEnd(t *testing.T) { stepOf := func(step string, kept ...sdkstep.KeptResult) sdkstep.StepAsk { return sdkstep.StepAsk{ID: sdkstep.NewID(), Kind: sdkstep.AskStep, Language: sdkstep.AskGo, Asker: "mesh-delivery-pipeline", At: time.Now(), Step: &sdkstep.StepRun{Pipeline: "main.12.m", Step: step, Module: "m", Artifact: "m:program", Commit: runCommit, Fingerprint: "fp-1", - Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: runCommit, Dir: "modules/m"}, Kept: kept, - Names: sdkstep.StepNames{Modules: []string{"m"}, Repositories: []string{"novox/x"}, Branches: []string{"main"}}}} + Fetch: sdkstep.Fetch{Repository: "novox/x", Commit: runCommit, Dir: "modules/m"}, Kept: kept}} } for _, row := range []struct { name string -- 2.54.0 From 579cd33cd540d2dccdeeb5aa2b78d38327cd4111 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:55:49 +0100 Subject: [PATCH 068/243] GATE2: the whole-module test counts the merges handed in the state merges, the merge as an owed act being left out of the join MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §160, tests 34, 39 (eb131beb left out, G §167, §542) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bus_test.go | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index bcbf1772a..13b2afa6b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -16,6 +16,7 @@ import ( "os" "os/exec" "reflect" + "strconv" "strings" "sync" "testing" @@ -697,7 +698,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB var d Delivery return err == nil && json.Unmarshal(e.Value(), &d) == nil && d.State == Merged }) - if n := len(rt2.mergesSaid()); n != 0 || len(onlyAskKeys(b)) > 1 { + if n := len(rt2.mergesSaid()); n != merges || len(onlyAskKeys(b)) > 1 { t.Fatalf("the restarted gate said the merge %d more times (%d before), or asked again: %v", n, merges, onlyAskKeys(b)) } // Quiet follows: nothing asks the forge or the controller again with no event to say so. @@ -908,3 +909,25 @@ func mustDecode[T any](t *testing.T, raw []byte) T { // (issue 1404). A failed first ask is asked again only after askAgainAfter (30 s, react.go), and 20 s deliberately does not // cover that: a failed first ask still fails the test. const eventWithin = 20 * time.Second + +// mergesSaid are the writes of the gate's state merges, the merges it handed to the forge bridge, in order (the old form of +// the merge: eb131beb, the merge as an owed act, is left out of the join). +func (r *runtime) mergesSaid() []string { + ctx := context.Background() + kv := r.b.kvs["merges"] + keys, err := kv.Keys(ctx) + if err != nil { + return nil + } + var out []string + for _, k := range keys { + hist, err := kv.History(ctx, k) + if err != nil { + continue + } + for _, e := range hist { + out = append(out, k+"@"+strconv.FormatUint(e.Revision(), 10)) + } + } + return out +} -- 2.54.0 From 9f62e0a2badd8d14b44de4bd74cda441e996895a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 14:59:16 +0100 Subject: [PATCH 069/243] BLD6: the real unit level's broken fixture fails in its tests alone, go vet being the lint step's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §328, §417 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-build-go/test_real_integration_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go index 369b38491..c76fef888 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/test_real_integration_test.go @@ -71,9 +71,9 @@ func TestUnitLevelRunsTheRealGoOnTheFixtureRepositories(t *testing.T) { "tagged": {passed: true, noTests: true}, "empty": {passed: true, noTests: true}, "slow": {passed: true}, - // go vet, before the tests since R4g (ADR 0529 §1), names the directory and the compiler's line; the tests still - // run after it (ADR 0591), and the package that does not build is their failure. - "broken": {failed: []string{".", "fixture/broken"}}, + // go vet is the lint step's, not the unit level's (design 47 G §328, §417): the package that does not build is the + // tests' failure alone. + "broken": {failed: []string{"fixture/broken"}}, } for name, w := range want { home := t.TempDir() -- 2.54.0 From bea1c446ea4f0247434c41b295d894fa905ce7a3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:45:03 +0100 Subject: [PATCH 070/243] NATS10: the bus module serves no restore point tool and keeps no state `kept` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverts bbd9f638, c2e017a4 and b6884796; cmd/mesh-nats-snapshot/restore.go is as before them. The manifest test asserts the claim serves only the reading verbs and the keeper's state is asked, requests, spent. Builds: design 47 G §554, §555 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/nats/README.md | 13 - .../nats/cmd/mesh-nats-snapshot/bussnap.go | 2 - .../nats/cmd/mesh-nats-snapshot/restore.go | 114 +++++ modules/nats/cmd/nats-tools/keep.go | 6 +- .../cmd/nats-tools/kept_integration_test.go | 153 ------- modules/nats/cmd/nats-tools/keptools.go | 112 ----- modules/nats/cmd/nats-tools/keptools_test.go | 82 ---- modules/nats/cmd/nats-tools/main.go | 7 +- modules/nats/cmd/nats-tools/manifest_test.go | 14 +- modules/nats/internal/bussnap/restore.go | 126 ------ modules/nats/internal/keeper/keeper.go | 3 - modules/nats/internal/keeper/kept.go | 405 ------------------ .../nats/internal/keeper/kept_jetstream.go | 128 ------ modules/nats/internal/keeper/kept_test.go | 311 -------------- modules/nats/internal/keeper/requests.go | 34 -- modules/nats/module.json | 11 +- 16 files changed, 126 insertions(+), 1395 deletions(-) delete mode 100644 modules/nats/cmd/nats-tools/kept_integration_test.go delete mode 100644 modules/nats/cmd/nats-tools/keptools.go delete mode 100644 modules/nats/cmd/nats-tools/keptools_test.go delete mode 100644 modules/nats/internal/bussnap/restore.go delete mode 100644 modules/nats/internal/keeper/kept.go delete mode 100644 modules/nats/internal/keeper/kept_jetstream.go delete mode 100644 modules/nats/internal/keeper/kept_test.go diff --git a/modules/nats/README.md b/modules/nats/README.md index 28e5b9a51..104e79e20 100644 --- a/modules/nats/README.md +++ b/modules/nats/README.md @@ -136,19 +136,6 @@ a put of the assigner (novox/hq ADR 0546, ADR 0565 clauses 8 and 9). In this ord tested) and the manifest's shape (programs mounted read-only, named in `restart-on`, no image built from a Dockerfile). -## A module's kept data, backed up and restored with its delivery (milestone G) - -The keeper takes, on its own account, a restore point of one module's kept data on the bus before the module moves across a -change of that kept data's `form`, and restores it beside the live data when a want takes the module back (design 47, *A -module's kept data on the bus*). Its seat verbs, served by the tools and handed to the keeper as requests: -`mesh-broker.snapshot-kept` (the data as the server's snapshot, the module's consumers' positions and the catch-up tool, in -one directory under the keeper's state), `mesh-broker.restore-kept` (the live data is snapshotted into a set-aside -directory under a name of its own and only then removed; the restore point is put in its place and the consumers on other -streams are set back; with `aside-only` it sets the live data aside and restores nothing, with `discard-aside`, naming data -set aside, it removes that data on a call that says why, recorded first, and restores nothing). Both are called by the -holder of `node-backup` on the same node alone. The backup holder -keeps the restore points; it calls these verbs through the seat. - ## Files it does not know Each directory or archive this module declares says how it treats a file it did not write (ADR 0422, ADR 0416). diff --git a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go index 444dd213f..e4d8a8f59 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/bussnap.go +++ b/modules/nats/cmd/mesh-nats-snapshot/bussnap.go @@ -14,8 +14,6 @@ var ( StreamNames = bussnap.StreamNames request = bussnap.Request truncate = bussnap.Truncate - RestoreAll = bussnap.RestoreAll - heldTo = bussnap.HeldTo ) const ( diff --git a/modules/nats/cmd/mesh-nats-snapshot/restore.go b/modules/nats/cmd/mesh-nats-snapshot/restore.go index f98bbc536..5d5866094 100644 --- a/modules/nats/cmd/mesh-nats-snapshot/restore.go +++ b/modules/nats/cmd/mesh-nats-snapshot/restore.go @@ -4,8 +4,10 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "errors" "fmt" + "io" "net" "os" "os/exec" @@ -18,6 +20,118 @@ import ( "nats-tools/internal/busarchive" ) +// restoreChunk is what is sent at a time; the server answers each before the next goes. +const restoreChunk = 64 * 1024 + +// RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds +// each restored stream's state to the manifest. **Nothing is restored over anything**: a server that +// already holds any of the streams is refused before the first is sent. +func RestoreAll(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, log func(string, ...any)) error { + var present []string + for _, s := range a.Manifest.Streams { + err := request(ctx, nc, apiStreamInfo+s.Name, nil, nil) + var api *apiError + switch { + case err == nil: + present = append(present, s.Name) + case errors.As(err, &api) && api.Code == 404: + default: + return err + } + } + if len(present) > 0 { + return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+ + "store (--into) and swap it in with the bus stopped", strings.Join(present, ", ")) + } + for _, s := range a.Manifest.Streams { + started := time.Now() + if err := restoreStream(ctx, nc, a, s); err != nil { + return fmt.Errorf("stream %s: %w", s.Name, err) + } + log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds()) + } + return nil +} + +func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error { + meta, archive, err := a.Open(s) + if err != nil { + return err + } + defer archive.Close() + var req map[string]json.RawMessage + if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { + return errors.New("its meta is not a stream's configuration and state") + } + var resp struct { + DeliverSubject string `json:"deliver_subject"` + } + if err := request(ctx, nc, apiStreamRestore+s.Name, req, &resp); err != nil { + return err + } + if resp.DeliverSubject == "" { + return errors.New("the server gave nowhere to send the archive") + } + buf := make([]byte, restoreChunk) + for { + n, err := archive.Read(buf) + if n > 0 { + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n]) + cancel() + if rerr != nil { + return fmt.Errorf("sending the archive: %w", rerr) + } + if len(reply.Data) > 0 { + return fmt.Errorf("the server refused the archive: %s", truncate(reply.Data)) + } + } + if err == io.EOF { + break + } + if err != nil { + return err + } + } + // The end: an empty message, answered once the server has rebuilt the stream from what it was + // sent — which for a large stream takes a while. + fctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil) + if err != nil { + return fmt.Errorf("finishing the restore: %w", err) + } + var created struct { + Error *apiError `json:"error"` + State streamState `json:"state"` + } + if err := json.Unmarshal(final.Data, &created); err != nil { + return fmt.Errorf("the server's last answer is not JSON: %q", truncate(final.Data)) + } + if created.Error != nil { + return created.Error + } + return heldTo(created.State, s) +} + +// heldTo is whether a restored stream holds at least what its snapshot said it held. +// +// **At least, not exactly — found by the restore test, not assumed.** The state the server answers a +// snapshot request with is taken when the snapshot starts; the stream goes on taking messages while +// its blocks are read out, and a block read a moment later carries some of what was appended in that +// moment. So a stream written during its snapshot restores with a later last sequence than the +// manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of +// the messages in that tail some may be there and some not. Every message up to the manifest's last +// sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message +// by message. What must never happen is a restored stream that ends before the manifest said it did. +func heldTo(restored streamState, s busarchive.StreamEntry) error { + if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) { + return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", + restored.Messages, restored.LastSeq, s.Messages, s.LastSeq) + } + return nil +} + // RestoreOffline fills a new store directory with the archive's streams, by a server of this // program's own: nats-server, the same binary the bus runs, on loopback, with the bus's one account // and a user that exists only for this run. It is stopped when the streams are in. What it leaves diff --git a/modules/nats/cmd/nats-tools/keep.go b/modules/nats/cmd/nats-tools/keep.go index 57c852c18..6576c1284 100644 --- a/modules/nats/cmd/nats-tools/keep.go +++ b/modules/nats/cmd/nats-tools/keep.go @@ -236,10 +236,8 @@ func runKeeper(ctx context.Context, getenv func(string) string, logOut io.Writer defer nc.Close() recorder := meshhandactsclient.Recorder{Ask: conn.Call} k := &keeper.Keeper{ - Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))}, - Kept: keeper.KeptJetStream{NC: nc, JS: js, Logf: logger.Printf}, - KeptDir: getenv("MESH_NATS_SNAPSHOTS_DIR"), - Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), + Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, getenv("MESH_NATS_SNAPSHOTS_DIR"))}, + Store: processStore{conn}, Emit: streamEmitter(js, account), Say: sayer(js), View: firstView(conn), Act: func(what, why, cause string) error { _, err := recorder.Record(meshhandactsclient.Request{What: what, Why: why, Cause: cause}) return err diff --git a/modules/nats/cmd/nats-tools/kept_integration_test.go b/modules/nats/cmd/nats-tools/kept_integration_test.go deleted file mode 100644 index 8f1c0523c..000000000 --- a/modules/nats/cmd/nats-tools/kept_integration_test.go +++ /dev/null @@ -1,153 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "os" - "path/filepath" - "sync" - "testing" - "time" - - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "nats-tools/internal/bussnap" - "nats-tools/internal/keeper" -) - -// mapStore is the keeper's own state in memory. -type mapStore struct { - mu sync.Mutex - kv map[string][]byte -} - -func (s *mapStore) Get(state, key string) ([]byte, uint64, bool, error) { - s.mu.Lock() - defer s.mu.Unlock() - v, ok := s.kv[state+"/"+key] - return v, 1, ok, nil -} -func (s *mapStore) Put(state, key string, v []byte) (uint64, error) { - s.mu.Lock() - defer s.mu.Unlock() - s.kv[state+"/"+key] = v - return 1, nil -} -func (s *mapStore) Delete(state, key string) error { - s.mu.Lock() - defer s.mu.Unlock() - delete(s.kv, state+"/"+key) - return nil -} -func (s *mapStore) Keys(string) ([]string, error) { return nil, nil } -func (s *mapStore) Create(state, key string, v []byte) (uint64, error) { - return 0, errors.New("not used") -} - -// A module's kept data on the real bus is backed up and restored beside the live data (design 47, "A module's kept data on -// the bus, backed up and restored with its delivery"): the bucket as the server's own snapshot, the module's consumer on -// another stream at its position; after a change, a restore brings the bucket and the position back, and the live bucket -// is kept whole in the data set aside. -func TestAKeptDataIsRestoredBesideTheLiveDataOnTheRealBus(t *testing.T) { - s, err := server.NewServer(&server.Options{Port: -1, JetStream: true, StoreDir: t.TempDir(), NoLog: true, NoSigs: true}) - if err != nil { - t.Fatal(err) - } - go s.Start() - if !s.ReadyForConnections(5 * time.Second) { - t.Fatal("the server did not start") - } - defer s.Shutdown() - nc, err := nats.Connect(s.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer nc.Close() - js, err := jetstream.New(nc) - if err != nil { - t.Fatal(err) - } - ctx := context.Background() - kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "idx_modules"}) - if err != nil { - t.Fatal(err) - } - forge, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "forge", Subjects: []string{"forge.>"}}) - if err != nil { - t.Fatal(err) - } - for i := 0; i < 10; i++ { - if _, err := js.Publish(ctx, "forge.merged", []byte("m")); err != nil { - t.Fatal(err) - } - } - cons, err := forge.CreateConsumer(ctx, jetstream.ConsumerConfig{Durable: "ace_idx", AckPolicy: jetstream.AckExplicitPolicy}) - if err != nil { - t.Fatal(err) - } - batch, err := cons.Fetch(4) - if err != nil { - t.Fatal(err) - } - for m := range batch.Messages() { - _ = m.DoubleAck(ctx) - } - if _, err := kv.PutString(ctx, "ace", "fingerprint-1"); err != nil { - t.Fatal(err) - } - - dir := t.TempDir() - var acts []string - k := &keeper.Keeper{Bus: keeper.JetStream{JS: js, Snap: bussnap.Into(nc, dir)}, Store: &mapStore{kv: map[string][]byte{}}, - Kept: keeper.KeptJetStream{NC: nc, JS: js}, KeptDir: dir, - Act: func(what, why, cause string) error { acts = append(acts, what); return nil }} - take := &keeper.Request{Verb: keeper.VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Why: "before form 1"} - if _, err := k.SnapshotKept(ctx, take); err != nil { - t.Fatal(err) - } - // The form-1 build rewrites the kept data, and its consumer moves on. - if _, err := kv.PutString(ctx, "ace", "fingerprint-2-form-1"); err != nil { - t.Fatal(err) - } - batch, err = cons.Fetch(5) - if err != nil { - t.Fatal(err) - } - for m := range batch.Messages() { - _ = m.DoubleAck(ctx) - } - - restore := &keeper.Request{Verb: keeper.VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: take.Dir, Why: "revert"} - if _, err := k.RestoreKept(ctx, restore); err != nil { - t.Fatal(err) - } - kv, err = js.KeyValue(ctx, "idx_modules") - if err != nil { - t.Fatal(err) - } - if e, err := kv.Get(ctx, "ace"); err != nil || string(e.Value()) != "fingerprint-1" { - t.Errorf("the restored bucket holds %v (%v), want fingerprint-1", e, err) - } - if restore.CatchUp != "indexer_take_in" { - t.Errorf("catch-up %q", restore.CatchUp) - } - forge, _ = js.Stream(ctx, "forge") - ci, err := forge.Consumer(ctx, "ace_idx") - if err != nil || ci.CachedInfo().Config.OptStartSeq != 5 { - t.Errorf("the consumer is %+v (%v): it starts after the 4 messages it had acknowledged", ci.CachedInfo().Config, err) - } - kept, err := os.ReadDir(filepath.Join(dir, "aside")) - if err != nil || len(kept) != 1 { - t.Fatalf("aside holds %v (%v)", kept, err) - } - if _, err := os.Stat(filepath.Join(dir, "aside", kept[0].Name(), keeper.ArchiveFile)); err != nil { - t.Error("the live data set aside is not kept whole") - } - if len(acts) != 2 { - t.Errorf("hand acts %v: the restore point and the restore", acts) - } -} diff --git a/modules/nats/cmd/nats-tools/keptools.go b/modules/nats/cmd/nats-tools/keptools.go deleted file mode 100644 index 87545381a..000000000 --- a/modules/nats/cmd/nats-tools/keptools.go +++ /dev/null @@ -1,112 +0,0 @@ -package main - -import ( - "fmt" - "strings" - "time" - - stdio "git.novox.be/novox/mesh-sdk/go" - - "nats-tools/internal/keeper" -) - -// keptWait is how long a call to take or restore a kept data waits for the keeper's end on the watch of its requests: under -// the runtime's own bound on a call, so a long snapshot is answered as "handed, not ended" and read again with its request. -const keptWait = 50 * time.Second - -func formOf(a map[string]any) int { - f, _ := a["form"].(float64) - return int(f) -} - -// keptTools are the seat mesh-broker's verbs for a module's kept data (design 47 G §405): -// each verb hands its request to the keeper, which acts on its own account, and waits for the end on the watch of its -// requests (never a poll). Asked again with `request`, a verb reads the end of the request it made. -func keptTools(rs requestStore) []stdio.Tool { - entry := func(extra map[string]any) map[string]any { - in := map[string]any{"module": str("the module whose kept data it is, e.g. mesh-delivery-indexer"), "local-name": str("the kept data's local name, e.g. modules"), - "form": num("the kept data's form (a manifest field; absent reads 0)"), "fingerprint": str("the fingerprint the restore point was taken before"), - "why": str("why, in plain words: kept with the act"), "request": str("a request this verb made before: its end is read, nothing new is asked")} - for k, v := range extra { - in[k] = v - } - return in - } - run := func(verb string, build func(a map[string]any, r *keeper.Request) error) func(a map[string]any) (any, error) { - return func(a map[string]any) (any, error) { - name := "mesh-broker." + verb - if id := arg(a, "request"); id != "" { - r, err := rs.Get(id) - if err != nil || r == nil { - return nil, fmt.Errorf("no request %s is held (%v)", id, err) - } - if r.Done.IsZero() && rs.Ended != nil { - if got := rs.Ended(id, keptWait); got != nil { - return got, nil - } - if again, err := rs.Get(id); err == nil && again != nil { - r = again - } - } - return r, nil - } - r := keeper.Request{Verb: verb, Why: strings.TrimSpace(arg(a, "why")), Caller: stdio.CallerOf(a), At: rs.Now()} - if err := build(a, &r); err != nil { - return nil, err - } - if r.Why == "" { - return nil, fmt.Errorf("%s says why", name) - } - r.ID = requestID(r.At) - if err := rs.Put(r.ID, r); err != nil { - return nil, fmt.Errorf("the request cannot be handed to the keeper: %w", err) - } - if rs.Ended != nil { - if got := rs.Ended(r.ID, keptWait); got != nil { - return got, nil - } - } - return map[string]any{"request": r.ID, "state": "handed to the keeper, not ended yet", "read": fmt.Sprintf("%s with request %s", name, r.ID)}, nil - } - } - point := func(a map[string]any, r *keeper.Request) error { - r.Module, r.Local, r.Form, r.Fingerprint = arg(a, "module"), arg(a, "local-name"), formOf(a), arg(a, "fingerprint") - if r.Module == "" || r.Local == "" || r.Fingerprint == "" { - return fmt.Errorf("a restore point is of one kept data: a module, its local name and the fingerprint it is taken before") - } - return nil - } - return []stdio.Tool{ - {Name: "mesh-broker.snapshot-kept", - Description: "Take the restore point of one module's kept data on the bus (milestone G, design 47): the data whole as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool the newer manifest declares, kept by the bus module's keeper. Answers the directory the restore point is in. Handed to the keeper, recorded as a hand act.", - Input: entry(map[string]any{"catch-up": str("the tool a restore is followed by, from the newer manifest, kept in the restore point")}), - Run: run(keeper.VerbSnapshotKept, func(a map[string]any, r *keeper.Request) error { - r.CatchUp = arg(a, "catch-up") - return point(a, r) - })}, - {Name: "mesh-broker.restore-kept", - Description: "Restore one module's kept data on the bus from a restore point beside the live data, never over it (design 47 G §405): the live data is snapshotted into a set-aside directory under a name of its own and then removed from the bus, the restore point's streams are put in its place and the module's other consumers are set back to their positions. With `aside-only` it sets the live data aside and restores nothing; with `discard-aside`, naming data set aside, it removes that data and restores nothing. Answers what was set aside and the catch-up tool the restore point carries. Handed to the keeper, recorded as a hand act.", - Input: entry(map[string]any{"from": str("the directory the restore point was restored to, an absolute path"), - "aside-only": map[string]any{"type": "boolean", "description": "set the live data aside and restore nothing"}, - "discard-aside": str("the id of data set aside: remove it and restore nothing")}), - Run: run(keeper.VerbRestoreKept, func(a map[string]any, r *keeper.Request) error { - if id := arg(a, "discard-aside"); id != "" { - r.Verb, r.From = keeper.VerbDiscardAside, id - return nil - } - if b, _ := a["aside-only"].(bool); b { - r.AsideOnly = true - r.Module, r.Local = arg(a, "module"), arg(a, "local-name") - if r.Module == "" || r.Local == "" { - return fmt.Errorf("aside-only is of one kept data: a module and its local name") - } - return nil - } - r.From = arg(a, "from") - if r.From == "" { - return fmt.Errorf("a restore names the directory the restore point is in") - } - return point(a, r) - })}, - } -} diff --git a/modules/nats/cmd/nats-tools/keptools_test.go b/modules/nats/cmd/nats-tools/keptools_test.go deleted file mode 100644 index a088cd704..000000000 --- a/modules/nats/cmd/nats-tools/keptools_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - stdio "git.novox.be/novox/mesh-sdk/go" - - "nats-tools/internal/keeper" -) - -// The seat mesh-broker's verbs for a module's kept data hand their request to the keeper as the proved caller, wait for -// its end on the watch of the requests, and say what a restore set aside (design 47, "A module's kept data on the bus"). -func TestTheKeptDataVerbsAreHandedToTheKeeperAndReadBack(t *testing.T) { - held := map[string]keeper.Request{} - rs := requestStore{ - Put: func(key string, r keeper.Request) error { held[key] = r; return nil }, - Get: func(key string) (*keeper.Request, error) { - r, ok := held[key] - if !ok { - return nil, nil - } - return &r, nil - }, - Now: func() time.Time { return time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) }, - // The keeper's end, as its watch hands it. - Ended: func(id string, within time.Duration) *keeper.Request { - r := held[id] - r.Done, r.Outcome, r.Dir = time.Unix(1, 0), "done", "/keeper/points/idx/modules/f0-abc" - held[id] = r - return &r - }, - } - tools := map[string]stdio.Tool{} - for _, tl := range keptTools(rs) { - tools[tl.Name] = tl - } - for _, verb := range KeptVerbs { - if _, ok := tools["mesh-broker."+verb]; !ok { - t.Errorf("the verb mesh-broker.%s is not served", verb) - } - } - out, err := tools["mesh-broker.snapshot-kept"].Run(map[string]any{"module": "idx", "local-name": "modules", "form": float64(1), - "fingerprint": "abc", "catch-up": "indexer_take_in", "why": "before form 1", "caller": "person.mallory"}) - if err != nil { - t.Fatal(err) - } - if r := out.(*keeper.Request); r.Verb != keeper.VerbSnapshotKept || r.Form != 1 || r.Fingerprint != "abc" || r.CatchUp != "indexer_take_in" || - r.Caller != stdio.Unproven || r.Dir == "" { - t.Errorf("answered %+v: the request for form 1 before abc, for the caller the runtime proved, with the directory the keeper answered", r) - } - for name, args := range map[string]map[string]any{ - "mesh-broker.snapshot-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc"}, - "mesh-broker.restore-kept": {"module": "idx", "local-name": "modules", "fingerprint": "abc", "why": "revert"}, - "mesh-broker.restore-kept#aside-only": {"aside-only": true, "why": "x"}, - } { - name, _, _ = strings.Cut(name, "#") - if _, err := tools[name].Run(args); err == nil { - t.Errorf("%s with %v was handed to the keeper", name, args) - } - } - if out, err := tools["mesh-broker.restore-kept"].Run(map[string]any{"module": "idx", "local-name": "modules", "fingerprint": "abc", - "from": "/restored/f0-abc", "why": "revert"}); err != nil || out.(*keeper.Request).From != "/restored/f0-abc" { - t.Errorf("restore answered %+v (%v)", out, err) - } - // `aside-only` sets the live data aside and restores nothing; `discard-aside` removes the data set aside (§405). - if out, err := tools["mesh-broker.restore-kept"].Run(map[string]any{"module": "idx", "local-name": "modules", "aside-only": true, "why": "revert"}); err != nil || - !out.(*keeper.Request).AsideOnly || out.(*keeper.Request).Verb != keeper.VerbRestoreKept { - t.Errorf("aside-only answered %+v (%v)", out, err) - } - if out, err := tools["mesh-broker.restore-kept"].Run(map[string]any{"discard-aside": "idx.modules.1", "why": "operator's call"}); err != nil || - out.(*keeper.Request).Verb != keeper.VerbDiscardAside || out.(*keeper.Request).From != "idx.modules.1" { - t.Errorf("discard-aside answered %+v (%v)", out, err) - } -} - -func toJSON(v any) string { - b, _ := json.Marshal(v) - return string(b) -} diff --git a/modules/nats/cmd/nats-tools/main.go b/modules/nats/cmd/nats-tools/main.go index d9f614485..034c5be3b 100644 --- a/modules/nats/cmd/nats-tools/main.go +++ b/modules/nats/cmd/nats-tools/main.go @@ -168,9 +168,6 @@ func main() { // them. var ReadingVerbs = []string{"list", "show", "check", "orphans"} -// KeptVerbs are the seat mesh-broker's verbs for a module's kept data (keptools.go): each handed to the keeper. -var KeptVerbs = []string{"snapshot-kept", "restore-kept"} - // Events are the seat mesh-broker's events (ADR 0595 §6, ADR 0620 §2): said on the seat's subjects on the shared events stream // and kept in full on the bus module's own stream broker-events (BROKER_EVENTS), which `history` reads (keep.go). var Events = []string{"created", "refused", "drifted", "drift-cleared", "full", "full-cleared", "orphaned", "recreated", "dropped"} @@ -193,7 +190,5 @@ func allTools() []stdio.Tool { out = append(out, brokerTools(HeldInState, jsz, StreamHistory(LiveBus()))...) out = append(out, keeperHealthTool(StatusInState, HeldInState, jsz)) out = append(out, readTools(jsz, LiveMsgGet(LiveBus()))...) - requests := RequestsInState() - out = append(out, requestTools(requests)...) - return append(out, keptTools(requests)...) + return append(out, requestTools(RequestsInState())...) } diff --git a/modules/nats/cmd/nats-tools/manifest_test.go b/modules/nats/cmd/nats-tools/manifest_test.go index d6639fdf3..2394a9eb8 100644 --- a/modules/nats/cmd/nats-tools/manifest_test.go +++ b/modules/nats/cmd/nats-tools/manifest_test.go @@ -189,8 +189,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { if err := json.Unmarshal(raw, &nm); err != nil { t.Fatal(err) } - if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, append(slices.Clone(ReadingVerbs), KeptVerbs...)) { - t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v and the keeper-handed verbs %v alone", nm.Claims, ReadingVerbs, KeptVerbs) + if len(nm.Claims) != 1 || nm.Claims[0].Name != "mesh-broker" || !slices.Equal(nm.Claims[0].Serves, ReadingVerbs) { + t.Errorf("claims are %+v: mesh-broker, serving its reading verbs %v alone", nm.Claims, ReadingVerbs) } if !slices.Equal(nm.Consumes, []string{"mesh-catalog.manifest-accepted"}) { t.Errorf("consumes %v: the seat mesh-catalog's manifest-accepted alone (ADR 0595 §4)", nm.Consumes) @@ -224,8 +224,8 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { } // spent, the warrants the keeper spent, is written by the keeper alone and only read by the tools (review round 2, // security F2: the tool runner's account, which writes requests, may never delete a spend record). - if !slices.Equal(states, []string{"asked", "requests", "spent", "kept"}) || !slices.Equal(readOnly, []string{"spent"}) { - t.Errorf("state is %v, read-only %v: asked and requests, old sections (ADR 0595 §10), spent, read-only, and kept, the restore points taken", states, readOnly) + if !slices.Equal(states, []string{"asked", "requests", "spent"}) || !slices.Equal(readOnly, []string{"spent"}) { + t.Errorf("state is %v, read-only %v: asked and requests, old sections (ADR 0595 §10), and spent, read-only", states, readOnly) } if _, ok := nm.OwnSecrets["broker-keeper"]; !ok { t.Error("own-secrets lack broker-keeper, the keeper's own account (ADR 0436 §1)") @@ -234,9 +234,9 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { bus, _ := keeper["bus"].(map[string]any) health, _ := keeper["health"].(map[string]any) if keeper["type"] != "process" || keeper["artifact"] != "tools" || fmt.Sprint(keeper["run"]) != "[./nats-tools keep]" || - fmt.Sprint(bus["state"]) != "[asked requests spent kept]" || + fmt.Sprint(bus["state"]) != "[asked requests spent]" || health["kind"] != "tool" || health["tool"] != "nats_keeper_health" { - t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked, requests, spent and kept, judged by nats_keeper_health", keeper) + t.Errorf("the keeper is %v: a process of the tools bundle, run as keep, keeping asked, requests and spent, judged by nats_keeper_health", keeper) } // Its bus declares keeps, the one key more ADR 0626 §1 allows it, read through the SDK as the grant composition reads it. var whole catalogue.Manifest @@ -263,7 +263,7 @@ func TestTheBussManifestMountsItsProgramsIntoTheUpstreamImage(t *testing.T) { t.Errorf("the manifest lists the tool %s, which the bundle does not serve", name) } } - for _, verb := range append(slices.Clone(ReadingVerbs), KeptVerbs...) { + for _, verb := range ReadingVerbs { if !served["mesh-broker."+verb] { t.Errorf("the bundle does not serve the seat verb mesh-broker.%s", verb) } diff --git a/modules/nats/internal/bussnap/restore.go b/modules/nats/internal/bussnap/restore.go deleted file mode 100644 index 484ddf22e..000000000 --- a/modules/nats/internal/bussnap/restore.go +++ /dev/null @@ -1,126 +0,0 @@ -package bussnap - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "strings" - "time" - - "github.com/nats-io/nats.go" - "nats-tools/internal/busarchive" -) - -// restoreChunk is what is sent at a time; the server answers each before the next goes. -const restoreChunk = 64 * 1024 - -// RestoreAll puts every stream of the archive into the server nc reaches, one at a time, and holds -// each restored stream's state to the manifest. **Nothing is restored over anything**: a server that -// already holds any of the streams is refused before the first is sent. -func RestoreAll(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, log func(string, ...any)) error { - var present []string - for _, s := range a.Manifest.Streams { - err := Request(ctx, nc, APIStreamInfo+s.Name, nil, nil) - var api *APIError - switch { - case err == nil: - present = append(present, s.Name) - case errors.As(err, &api) && api.Code == 404: - default: - return err - } - } - if len(present) > 0 { - return fmt.Errorf("the server already holds %s; nothing is restored over a live stream — restore into a new "+ - "store (--into) and swap it in with the bus stopped", strings.Join(present, ", ")) - } - for _, s := range a.Manifest.Streams { - started := time.Now() - if err := restoreStream(ctx, nc, a, s); err != nil { - return fmt.Errorf("stream %s: %w", s.Name, err) - } - log("%s: restored, at least %d message(s) to sequence %d, in %.2fs", s.Name, s.Messages, s.LastSeq, time.Since(started).Seconds()) - } - return nil -} - -func restoreStream(ctx context.Context, nc *nats.Conn, a *busarchive.Archive, s busarchive.StreamEntry) error { - meta, archive, err := a.Open(s) - if err != nil { - return err - } - defer archive.Close() - var req map[string]json.RawMessage - if err := json.Unmarshal(meta, &req); err != nil || req["config"] == nil { - return errors.New("its meta is not a stream's configuration and state") - } - var resp struct { - DeliverSubject string `json:"deliver_subject"` - } - if err := Request(ctx, nc, APIStreamRestore+s.Name, req, &resp); err != nil { - return err - } - if resp.DeliverSubject == "" { - return errors.New("the server gave nowhere to send the archive") - } - buf := make([]byte, restoreChunk) - for { - n, err := archive.Read(buf) - if n > 0 { - cctx, cancel := context.WithTimeout(ctx, 30*time.Second) - reply, rerr := nc.RequestWithContext(cctx, resp.DeliverSubject, buf[:n]) - cancel() - if rerr != nil { - return fmt.Errorf("sending the archive: %w", rerr) - } - if len(reply.Data) > 0 { - return fmt.Errorf("the server refused the archive: %s", Truncate(reply.Data)) - } - } - if err == io.EOF { - break - } - if err != nil { - return err - } - } - // The end: an empty message, answered once the server has rebuilt the stream from what it was - // sent — which for a large stream takes a while. - fctx, cancel := context.WithTimeout(ctx, 30*time.Minute) - defer cancel() - final, err := nc.RequestWithContext(fctx, resp.DeliverSubject, nil) - if err != nil { - return fmt.Errorf("finishing the restore: %w", err) - } - var created struct { - Error *APIError `json:"error"` - State StreamState `json:"state"` - } - if err := json.Unmarshal(final.Data, &created); err != nil { - return fmt.Errorf("the server's last answer is not JSON: %q", Truncate(final.Data)) - } - if created.Error != nil { - return created.Error - } - return HeldTo(created.State, s) -} - -// HeldTo is whether a restored stream holds at least what its snapshot said it held. -// -// **At least, not exactly — found by the restore test, not assumed.** The state the server answers a -// snapshot request with is taken when the snapshot starts; the stream goes on taking messages while -// its blocks are read out, and a block read a moment later carries some of what was appended in that -// moment. So a stream written during its snapshot restores with a later last sequence than the -// manifest says (90075 where it said 90024, with a writer publishing every 2ms through it), and of -// the messages in that tail some may be there and some not. Every message up to the manifest's last -// sequence is there, exactly; that is the snapshot's promise, and what the test holds it to message -// by message. What must never happen is a restored stream that ends before the manifest said it did. -func HeldTo(restored StreamState, s busarchive.StreamEntry) error { - if restored.LastSeq < s.LastSeq || (s.Messages > 0 && restored.Messages == 0) { - return fmt.Errorf("restored with %d message(s) up to sequence %d; the snapshot held %d up to %d", - restored.Messages, restored.LastSeq, s.Messages, s.LastSeq) - } - return nil -} diff --git a/modules/nats/internal/keeper/keeper.go b/modules/nats/internal/keeper/keeper.go index bde7cd12e..073671dd4 100644 --- a/modules/nats/internal/keeper/keeper.go +++ b/modules/nats/internal/keeper/keeper.go @@ -219,9 +219,6 @@ type Keeper struct { Warrant func(askID string) (*oc.Ask, *oc.Warrant, error) Now func() time.Time Logf func(string, ...any) - // Kept takes and restores a module's kept data, in KeptDir (kept.go); nil where the process has no such bus. - Kept KeptBus - KeptDir string // Raise and Clear are the fill marks in percent (ADR 0595 §6): settings of the bus module, 75 and 70 by default. Raise, Clear int64 diff --git a/modules/nats/internal/keeper/kept.go b/modules/nats/internal/keeper/kept.go deleted file mode 100644 index 945f71ae4..000000000 --- a/modules/nats/internal/keeper/kept.go +++ /dev/null @@ -1,405 +0,0 @@ -package keeper - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "slices" - "strings" - "time" - - "nats-tools/internal/busarchive" -) - -// A module's kept data on the bus, backed up before the module moves and restored beside the live data when a want takes -// it back (milestone G, design 47 "A module's kept data on the bus, backed up and restored with its delivery"). The -// keeper takes it on its own account, as it takes a snapshot before a recreate (ADR 0235 §5): the streams of one kept -// data as the server's own snapshot, the position of each durable consumer the module reads with, and the catch-up tool -// the newer manifest declares, in one directory the backup holder keeps as a restore point. A restore never goes over -// the live data: the live data is first snapshotted into a set-aside directory under a name of its own, and only then -// removed from the bus; only discard-aside removes the set-aside copy, on a person's call. - -const ( - // VerbSnapshotKept takes a restore point of one kept data; VerbRestoreKept restores one beside the live data. - VerbSnapshotKept = "snapshot-kept" - VerbRestoreKept = "restore-kept" - // VerbDiscardAside removes the data a restore set aside, by its id. - VerbDiscardAside = "discard-aside" -) - -// StateKept is the keeper's own state of the restore points it took and the data it set aside (never their bytes, which -// are files in the keeper's directory): `point....` and `aside.`. Read by the -// bus module's tool nats_kept and by the seat mesh-broker's `aside`. -const StateKept = "kept" - -// PointFile and ArchiveFile are the two files of a restore point's directory; AsideFile of a set-aside directory. -const ( - PointFile = "point.json" - ArchiveFile = "bus.tar" - AsideFile = "aside.json" -) - -// Position is where one durable consumer of the module stood on a stream: the stream sequence it had acknowledged up to, -// and its configuration as the server held it, so a restore can make it again at that place. -type Position struct { - Stream string `json:"stream"` - Consumer string `json:"consumer"` - AckFloor uint64 `json:"ack-floor"` - Config json.RawMessage `json:"config"` -} - -// KeptBus is what the keeper needs of the bus beyond Bus to take and restore a kept data. -type KeptBus interface { - // Existing answers which of the stream names the bus holds. - Existing(ctx context.Context, names []string) ([]string, error) - // Positions are the module's durable consumers on every stream (a consumer is the module's when it is named for it). - Positions(ctx context.Context, module string) ([]Position, error) - // SetPositions makes each consumer again from the place it stood. - SetPositions(ctx context.Context, ps []Position) error - // RestoreArchive puts every stream of an archive into the bus, which must hold none of them. - RestoreArchive(ctx context.Context, a *busarchive.Archive) error - // DeleteStreams removes streams by name. - DeleteStreams(ctx context.Context, names []string) error -} - -// Point is a restore point's description, its directory's point.json. -type Point struct { - Module string `json:"module"` - Local string `json:"local"` - Form int `json:"form"` - Fingerprint string `json:"fingerprint"` - CatchUp string `json:"catch-up,omitempty"` - Streams []string `json:"streams"` - Positions []Position `json:"positions"` - At time.Time `json:"at"` - Bytes int64 `json:"bytes"` -} - -// Aside is data a restore set aside: the live kept data as it was when the restore removed it. -type Aside struct { - ID string `json:"id"` - Module string `json:"module"` - Local string `json:"local"` - Streams []string `json:"streams"` - At time.Time `json:"at"` - Bytes int64 `json:"bytes"` - By string `json:"by"` - Why string `json:"why"` - Dir string `json:"dir"` -} - -// PointKey is a restore point's key in StateKept. -func PointKey(module, local string, form int, fingerprint string) string { - return fmt.Sprintf("point.%s.%s.%d.%s", module, local, form, fingerprint) -} - -// safeName is one path component made of what a module, a kept data and a fingerprint are made of. -func safeName(s string) bool { - return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\\\x00 *>.") -} - -func (k *Keeper) pointDir(module, local string, form int, fingerprint string) string { - return filepath.Join(k.KeptDir, "points", module, local, fmt.Sprintf("f%d-%s", form, fingerprint)) -} - -// streamsOf is the streams one kept data is: its plan's objects where the keeper made it, else the streams the bus holds -// under its bus name (a foundation part's own section, made by the grants part). -func (k *Keeper) streamsOf(ctx context.Context, module, local string) ([]string, error) { - if held, _, found := k.asked(module); found { - if m, made := held.Made[local]; made && len(m.Plan.Objects) > 0 { - var out []string - for _, o := range m.Plan.Objects { - out = append(out, o.Stream()) - } - return out, nil - } - } - name := BusName(module, local) - got, err := k.Kept.Existing(ctx, []string{"KV_" + name, "OBJ_" + name, name}) - if err != nil { - return nil, err - } - if len(got) == 0 { - return nil, fmt.Errorf("the bus holds no kept data %s of %s (looked for KV_%s, OBJ_%s and %s)", local, module, name, name, name) - } - return got, nil -} - -func (k *Keeper) keptReady(r *Request) error { - if k.Kept == nil || k.KeptDir == "" { - return errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR), so no kept data is taken or restored") - } - if !safeName(r.Module) || !safeName(r.Local) || (!r.AsideOnly && !safeName(r.Fingerprint)) { - return fmt.Errorf("%q, %q and %q are not a module, a kept data and a fingerprint: each is one plain name", r.Module, r.Local, r.Fingerprint) - } - return nil -} - -// SnapshotKept takes the restore point of one kept data: the file pair in a directory named for its form and fingerprint, -// written whole before its name is given, and its record in StateKept. It answers the directory. -func (k *Keeper) SnapshotKept(ctx context.Context, r *Request) (string, error) { - if err := k.keptReady(r); err != nil { - return "", err - } - streams, err := k.streamsOf(ctx, r.Module, r.Local) - if err != nil { - return "", err - } - // The positions first: a consumer that moves while the snapshot is taken is then a little behind its archive, and a - // restored consumer that is behind hears a message twice, never misses one. - positions, err := k.Kept.Positions(ctx, r.Module) - if err != nil { - return "", fmt.Errorf("the consumers' positions of %s cannot be read, so no restore point is taken: %w", r.Module, err) - } - tar, err := k.Bus.Snapshot(ctx, streams) - if err != nil { - return "", fmt.Errorf("the snapshot of %s's %s failed, so no restore point is taken: %w", r.Module, r.Local, err) - } - dir := k.pointDir(r.Module, r.Local, r.Form, r.Fingerprint) - if err := os.MkdirAll(filepath.Dir(dir), 0o700); err != nil { - return "", err - } - partial := dir + ".partial" - _ = os.RemoveAll(partial) - if err := os.Mkdir(partial, 0o700); err != nil { - return "", err - } - if err := os.Rename(tar, filepath.Join(partial, ArchiveFile)); err != nil { - _ = os.RemoveAll(partial) - return "", fmt.Errorf("the snapshot cannot be kept as a restore point: %w", err) - } - st, _ := os.Stat(filepath.Join(partial, ArchiveFile)) - p := Point{Module: r.Module, Local: r.Local, Form: r.Form, Fingerprint: r.Fingerprint, CatchUp: r.CatchUp, - Streams: streams, Positions: positions, At: k.now()} - if st != nil { - p.Bytes = st.Size() - } - raw, _ := json.MarshalIndent(p, "", " ") - if err := os.WriteFile(filepath.Join(partial, PointFile), append(raw, '\n'), 0o600); err != nil { - _ = os.RemoveAll(partial) - return "", err - } - // A restore point taken again at the same form and fingerprint replaces the earlier one, whole. - _ = os.RemoveAll(dir) - if err := os.Rename(partial, dir); err != nil { - return "", err - } - val, _ := json.Marshal(p) - if _, err := k.Store.Put(StateKept, PointKey(r.Module, r.Local, r.Form, r.Fingerprint), val); err != nil { - k.logf("the record of the restore point %s cannot be written to its own %s: %v", dir, StateKept, err) - } - r.Dir = dir - k.record(r, fmt.Sprintf("took the restore point of %s's %s at form %d before %s (%d stream(s), %d consumer position(s), %d bytes)", - r.Module, r.Local, r.Form, r.Fingerprint, len(streams), len(positions), p.Bytes)) - return fmt.Sprintf("restore point of %s's %s at form %d before %s in %s", r.Module, r.Local, r.Form, r.Fingerprint, dir), nil -} - -// readPoint reads and checks a restore point's directory: its point.json names what the request names, its files are -// plain files (never a link), and its archive matches its own manifest. -func readPoint(dir string, r *Request) (*Point, *busarchive.Archive, error) { - for _, name := range []string{PointFile, ArchiveFile} { - fi, err := os.Lstat(filepath.Join(dir, name)) - if err != nil { - return nil, nil, fmt.Errorf("the restore point %s holds no %s: %w", dir, name, err) - } - if !fi.Mode().IsRegular() { - return nil, nil, fmt.Errorf("%s of the restore point %s is not a plain file", name, dir) - } - } - raw, err := os.ReadFile(filepath.Join(dir, PointFile)) - if err != nil { - return nil, nil, err - } - var p Point - if err := json.Unmarshal(raw, &p); err != nil { - return nil, nil, fmt.Errorf("%s of %s does not read: %w", PointFile, dir, err) - } - if p.Module != r.Module || p.Local != r.Local || p.Form != r.Form || p.Fingerprint != r.Fingerprint { - return nil, nil, fmt.Errorf("the directory %s holds the restore point of %s's %s at form %d before %s, not of %s's %s at form %d before %s", - dir, p.Module, p.Local, p.Form, p.Fingerprint, r.Module, r.Local, r.Form, r.Fingerprint) - } - f, err := os.Open(filepath.Join(dir, ArchiveFile)) - if err != nil { - return nil, nil, err - } - defer f.Close() - a, err := busarchive.ReadArchive(f, "") - if err != nil { - return nil, nil, fmt.Errorf("the archive of the restore point %s does not read: %w", dir, err) - } - if err := a.Verify(); err != nil { - a.Close() - return nil, nil, fmt.Errorf("the archive of the restore point %s does not match its own manifest, so nothing is restored: %w", dir, err) - } - return &p, a, nil -} - -// RestoreKept restores one kept data beside the live data: the live data is snapshotted into a set-aside directory first -// and only then removed from the bus, and the restore point's streams are put in its place; the consumers the module -// reads with on other streams are set back to their positions. It answers what was set aside, where. -func (k *Keeper) RestoreKept(ctx context.Context, r *Request) (string, error) { - if err := k.keptReady(r); err != nil { - return "", err - } - var p *Point - var a *busarchive.Archive - var live []string - if r.AsideOnly { - // `aside-only` (design 47 G §405): the live data is set aside and nothing is restored. - streams, err := k.streamsOf(ctx, r.Module, r.Local) - if err != nil { - return "", err - } - live = streams - } else { - if !filepath.IsAbs(r.From) { - return "", fmt.Errorf("the restore point's directory is given as an absolute path, not %q", r.From) - } - var err error - p, a, err = readPoint(filepath.Clean(r.From), r) - if err != nil { - return "", err - } - defer a.Close() - live, err = k.Kept.Existing(ctx, p.Streams) - if err != nil { - return "", err - } - } - id := fmt.Sprintf("%s.%s.%s", r.Module, r.Local, k.now().Format("20060102t150405")) - var aside *Aside - if len(live) > 0 { - // Aside before anything is removed: a snapshot that fails leaves the live data where it is. - tar, err := k.Bus.Snapshot(ctx, live) - if err != nil { - return "", fmt.Errorf("the live %s of %s could not be set aside, so nothing was removed: %w", r.Local, r.Module, err) - } - dir := filepath.Join(k.KeptDir, "aside", id) - if err := os.MkdirAll(dir, 0o700); err != nil { - return "", err - } - if err := os.Rename(tar, filepath.Join(dir, ArchiveFile)); err != nil { - return "", fmt.Errorf("the live data's snapshot cannot be kept: %w", err) - } - st, _ := os.Stat(filepath.Join(dir, ArchiveFile)) - aside = &Aside{ID: id, Module: r.Module, Local: r.Local, Streams: live, At: k.now(), By: r.Caller, Why: r.Why, Dir: dir} - if st != nil { - aside.Bytes = st.Size() - } - raw, _ := json.MarshalIndent(aside, "", " ") - if err := os.WriteFile(filepath.Join(dir, AsideFile), append(raw, '\n'), 0o600); err != nil { - return "", err - } - val, _ := json.Marshal(aside) - if _, err := k.Store.Put(StateKept, "aside."+id, val); err != nil { - return "", fmt.Errorf("what is set aside cannot be recorded, so nothing was removed: %w", err) - } - if err := k.Kept.DeleteStreams(ctx, live); err != nil { - return "", fmt.Errorf("the live data was set aside in %s and could not be removed from the bus: %w", dir, err) - } - } - if r.AsideOnly { - r.Dir = "" - where := "nothing was live, so nothing was set aside" - if aside != nil { - r.Dir = aside.Dir - where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir) - } - k.record(r, fmt.Sprintf("set aside %s's %s and restored nothing; %s", r.Module, r.Local, where)) - return fmt.Sprintf("set aside %s's %s; %s", r.Module, r.Local, where), nil - } - if err := k.Kept.RestoreArchive(ctx, a); err != nil { - msg := fmt.Sprintf("the restore point could not be put on the bus: %v", err) - if aside != nil { - if back, rerr := k.putBack(ctx, aside); rerr != nil { - msg += fmt.Sprintf("; the live data, set aside in %s, could not be put back either: %v", aside.Dir, rerr) - } else { - msg += fmt.Sprintf("; the live data is back as it was (%s)", back) - } - } - return "", errors.New(msg) - } - // The consumers on streams the restore point did not carry (the snapshot of a stream carries its own consumers). - var apply []Position - for _, pos := range p.Positions { - if !slices.Contains(p.Streams, pos.Stream) { - apply = append(apply, pos) - } - } - if len(apply) > 0 { - if err := k.Kept.SetPositions(ctx, apply); err != nil { - return "", fmt.Errorf("the kept data is restored, but the consumers' positions could not be set back: %w", err) - } - } - r.Dir, r.CatchUp = "", p.CatchUp - where := "nothing was live, so nothing was set aside" - if aside != nil { - r.Dir = aside.Dir - where = fmt.Sprintf("the live data is set aside as %s (%d bytes) in %s", aside.ID, aside.Bytes, aside.Dir) - } - k.record(r, fmt.Sprintf("restored %s's %s from the restore point at form %d before %s beside the live data; %s", r.Module, r.Local, r.Form, r.Fingerprint, where)) - return fmt.Sprintf("restored %s's %s from its restore point; %s; %d consumer position(s) set back; catch-up tool %q", r.Module, r.Local, where, len(apply), p.CatchUp), nil -} - -// putBack restores set-aside data to the bus, after a restore failed. -func (k *Keeper) putBack(ctx context.Context, a *Aside) (string, error) { - f, err := os.Open(filepath.Join(a.Dir, ArchiveFile)) - if err != nil { - return "", err - } - defer f.Close() - arch, err := busarchive.ReadArchive(f, "") - if err != nil { - return "", err - } - defer arch.Close() - if err := k.Kept.DeleteStreams(ctx, a.Streams); err != nil { - return "", err - } - if err := k.Kept.RestoreArchive(ctx, arch); err != nil { - return "", err - } - return "from " + a.Dir, nil -} - -// DiscardAside removes the data a restore set aside, by its id (Request.From), on a person's call: the request says who -// and why, and is recorded as a hand act before the data goes. -func (k *Keeper) DiscardAside(r *Request) (string, error) { - if k.KeptDir == "" { - return "", errors.New("no directory for restore points is set (MESH_NATS_SNAPSHOTS_DIR)") - } - id := strings.TrimSpace(r.From) - if !safeAsideID(id) { - return "", fmt.Errorf("%q is not the id of data set aside", id) - } - if strings.TrimSpace(r.Why) == "" { - return "", errors.New("discarding set-aside data says why") - } - raw, _, found, err := k.Store.Get(StateKept, "aside."+id) - if err != nil { - return "", err - } - if !found { - return "nothing done", fmt.Errorf("no data is set aside as %s", id) - } - var a Aside - _ = json.Unmarshal(raw, &a) - k.record(r, fmt.Sprintf("discarded the data set aside as %s (%s's %s, %d bytes)", id, a.Module, a.Local, a.Bytes)) - if r.Unrecorded != "" { - return "nothing done", fmt.Errorf("the hand act cannot be recorded, so the set-aside data stays: %s", r.Unrecorded) - } - if err := os.RemoveAll(filepath.Join(k.KeptDir, "aside", id)); err != nil { - return "partly done", err - } - if err := k.Store.Delete(StateKept, "aside."+id); err != nil { - return "partly done", err - } - return fmt.Sprintf("discarded the data set aside as %s", id), nil -} - -func safeAsideID(id string) bool { - return id != "" && !strings.ContainsAny(id, "/\\\x00 *>") && !strings.HasPrefix(id, ".") -} diff --git a/modules/nats/internal/keeper/kept_jetstream.go b/modules/nats/internal/keeper/kept_jetstream.go deleted file mode 100644 index 8a3042f0e..000000000 --- a/modules/nats/internal/keeper/kept_jetstream.go +++ /dev/null @@ -1,128 +0,0 @@ -package keeper - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "nats-tools/internal/busarchive" - "nats-tools/internal/bussnap" -) - -// KeptJetStream is KeptBus on the keeper's own connection. -type KeptJetStream struct { - NC *nats.Conn - JS jetstream.JetStream - Logf func(string, ...any) -} - -// Existing answers which of the names are streams the bus holds. -func (b KeptJetStream) Existing(ctx context.Context, names []string) ([]string, error) { - var out []string - for _, n := range names { - _, err := b.JS.Stream(ctx, n) - switch { - case err == nil: - out = append(out, n) - case errors.Is(err, jetstream.ErrStreamNotFound): - default: - return nil, err - } - } - return out, nil -} - -// ConsumerOf is whether a durable consumer is the module's: named for it, alone or after the machine that runs it -// (`_`, process.Account.QueueConsumerName). -func ConsumerOf(consumer, module string) bool { - return consumer == module || strings.HasSuffix(consumer, "_"+module) -} - -// Positions are the module's durable consumers on every stream, each with the stream sequence it has acknowledged up to. -func (b KeptJetStream) Positions(ctx context.Context, module string) ([]Position, error) { - var out []Position - streams := b.JS.ListStreams(ctx) - var names []string - for s := range streams.Info() { - names = append(names, s.Config.Name) - } - if err := streams.Err(); err != nil { - return nil, err - } - for _, name := range names { - st, err := b.JS.Stream(ctx, name) - if err != nil { - return nil, err - } - list := st.ListConsumers(ctx) - for ci := range list.Info() { - if !ConsumerOf(ci.Name, module) { - continue - } - cfg, err := json.Marshal(ci.Config) - if err != nil { - return nil, err - } - out = append(out, Position{Stream: name, Consumer: ci.Name, AckFloor: ci.AckFloor.Stream, Config: cfg}) - } - if err := list.Err(); err != nil { - return nil, err - } - } - return out, nil -} - -// SetPositions makes each consumer again, from the sequence after the one it had acknowledged. A consumer's place cannot -// be edited, so it is removed and made with its own configuration and a new start. -func (b KeptJetStream) SetPositions(ctx context.Context, ps []Position) error { - for _, p := range ps { - st, err := b.JS.Stream(ctx, p.Stream) - if err != nil { - return fmt.Errorf("the stream %s of %s's position cannot be read: %w", p.Stream, p.Consumer, err) - } - var cfg jetstream.ConsumerConfig - if err := json.Unmarshal(p.Config, &cfg); err != nil { - return fmt.Errorf("the configuration of %s on %s does not read: %w", p.Consumer, p.Stream, err) - } - info, err := st.Info(ctx) - if err != nil { - return err - } - start := p.AckFloor + 1 - if start > info.State.LastSeq+1 { - start = info.State.LastSeq + 1 - } - cfg.DeliverPolicy, cfg.OptStartSeq, cfg.OptStartTime = jetstream.DeliverByStartSequencePolicy, start, nil - if err := st.DeleteConsumer(ctx, p.Consumer); err != nil && !errors.Is(err, jetstream.ErrConsumerNotFound) { - return fmt.Errorf("%s on %s cannot be removed to be made again: %w", p.Consumer, p.Stream, err) - } - if _, err := st.CreateConsumer(ctx, cfg); err != nil { - return fmt.Errorf("%s on %s cannot be made again at %d: %w", p.Consumer, p.Stream, start, err) - } - } - return nil -} - -// RestoreArchive puts every stream of the archive into the bus, which must hold none of them. -func (b KeptJetStream) RestoreArchive(ctx context.Context, a *busarchive.Archive) error { - log := b.Logf - if log == nil { - log = func(string, ...any) {} - } - return bussnap.RestoreAll(ctx, b.NC, a, log) -} - -// DeleteStreams removes streams by name. -func (b KeptJetStream) DeleteStreams(ctx context.Context, names []string) error { - for _, n := range names { - if err := b.JS.DeleteStream(ctx, n); err != nil && !errors.Is(err, jetstream.ErrStreamNotFound) { - return fmt.Errorf("%s: %w", n, err) - } - } - return nil -} diff --git a/modules/nats/internal/keeper/kept_test.go b/modules/nats/internal/keeper/kept_test.go deleted file mode 100644 index 95e0d2b12..000000000 --- a/modules/nats/internal/keeper/kept_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package keeper - -import ( - "archive/tar" - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/klauspost/compress/s2" - - "nats-tools/internal/busarchive" -) - -// tarBus is the fake bus whose snapshot is a real archive of the streams asked for, as the server's is. -type tarBus struct { - *fakeBus - dir string - n int -} - -func (b *tarBus) Snapshot(ctx context.Context, streams []string) (string, error) { - if _, err := b.fakeBus.Snapshot(ctx, streams); err != nil { - return "", err - } - b.n++ - path := filepath.Join(b.dir, fmt.Sprintf(".entry-%d.tar", b.n)) - return path, os.WriteFile(path, archiveOf(streams), 0o600) -} - -func sha(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } - -// archiveOf is a snapshot archive of the streams, each holding one block. -func archiveOf(streams []string) []byte { - var out bytes.Buffer - tw := tar.NewWriter(&out) - put := func(name string, b []byte) { - _ = tw.WriteHeader(&tar.Header{Name: name, Mode: 0o600, Size: int64(len(b))}) - _, _ = tw.Write(b) - } - m := busarchive.Manifest{Format: 1, Finished: time.Date(2026, 11, 9, 3, 0, 0, 0, time.UTC)} - files := map[string][]byte{} - for _, name := range streams { - var raw bytes.Buffer - itw := tar.NewWriter(&raw) - _ = itw.WriteHeader(&tar.Header{Name: "msgs.blk", Mode: 0o600, Size: 3}) - _, _ = itw.Write([]byte("abc")) - itw.Close() - var arc bytes.Buffer - w := s2.NewWriter(&arc) - _, _ = w.Write(raw.Bytes()) - w.Close() - meta := []byte(`{"config":{"name":"` + name + `"}}`) - e := busarchive.StreamEntry{Name: name, Meta: "streams/" + name + "/backup.json", MetaSHA256: sha(meta), - Archive: "streams/" + name + "/stream.tar.s2", ArchiveBytes: int64(arc.Len()), ArchiveSHA256: sha(arc.Bytes())} - m.Streams = append(m.Streams, e) - files[e.Meta], files[e.Archive] = meta, arc.Bytes() - } - raw, _ := json.Marshal(m) - put(busarchive.ManifestName, raw) - for _, s := range m.Streams { - put(s.Meta, files[s.Meta]) - put(s.Archive, files[s.Archive]) - } - tw.Close() - return out.Bytes() -} - -// keptBus is the fake bus's streams and consumers, as the server holds them. -type keptBus struct { - streams map[string]bool - positions []Position - set []Position - restored [][]string - refuse error -} - -func (b *keptBus) Existing(_ context.Context, names []string) ([]string, error) { - var out []string - for _, n := range names { - if b.streams[n] { - out = append(out, n) - } - } - return out, nil -} -func (b *keptBus) Positions(context.Context, string) ([]Position, error) { return b.positions, nil } -func (b *keptBus) SetPositions(_ context.Context, ps []Position) error { - b.set = append(b.set, ps...) - return nil -} -func (b *keptBus) RestoreArchive(_ context.Context, a *busarchive.Archive) error { - if err := b.refuse; err != nil { - b.refuse = nil // refused once, as a server that was busy - return err - } - var names []string - for _, s := range a.Manifest.Streams { - if b.streams[s.Name] { - return fmt.Errorf("the server already holds %s", s.Name) - } - names = append(names, s.Name) - } - for _, n := range names { - b.streams[n] = true - } - b.restored = append(b.restored, names) - return nil -} -func (b *keptBus) DeleteStreams(_ context.Context, names []string) error { - for _, n := range names { - delete(b.streams, n) - } - return nil -} - -func keptKeeper(t *testing.T) (*Keeper, *keptBus, *memStore, *[]string) { - t.Helper() - k, fb, store, _ := newKeeper(FirstView{}) - dir := t.TempDir() - kb := &keptBus{streams: map[string]bool{"KV_idx_modules": true}, - positions: []Position{{Stream: "forge", Consumer: "ace_idx", AckFloor: 41, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}, - {Stream: "KV_idx_modules", Consumer: "ace_idx", AckFloor: 7, Config: json.RawMessage(`{"durable_name":"ace_idx"}`)}}} - acts := &[]string{} - k.Bus, k.Kept, k.KeptDir = &tarBus{fakeBus: fb, dir: dir}, kb, dir - k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil } - return k, kb, store, acts -} - -func take(t *testing.T, k *Keeper) *Request { - t.Helper() - r := &Request{Verb: VerbSnapshotKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", CatchUp: "indexer_take_in", Caller: "ace.restic", Why: "before form 1"} - if _, err := k.SnapshotKept(context.Background(), r); err != nil { - t.Fatal(err) - } - return r -} - -// A module's kept data on the bus has a restore point before its module moves, and a restore never goes over the live -// data (design 47 "A module's kept data on the bus, backed up and restored with its delivery"). -func TestAKeptDataIsBackedUpAndRestoredBesideTheLiveData(t *testing.T) { - for _, row := range []struct { - name string - run func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) - }{ - {"a restore point holds the archive, the positions and the catch-up tool, keyed by form and fingerprint", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - r := take(t, k) - want := filepath.Join(k.KeptDir, "points", "idx", "modules", "f0-abc123") - if r.Dir != want { - t.Fatalf("restore point in %s, want %s", r.Dir, want) - } - raw, err := os.ReadFile(filepath.Join(want, PointFile)) - if err != nil { - t.Fatal(err) - } - var p Point - if err := json.Unmarshal(raw, &p); err != nil || p.CatchUp != "indexer_take_in" || len(p.Positions) != 2 || p.Streams[0] != "KV_idx_modules" || p.Bytes == 0 { - t.Errorf("point.json = %s (%v)", raw, err) - } - if _, err := os.Stat(filepath.Join(want, ArchiveFile)); err != nil { - t.Error(err) - } - if _, _, found, _ := store.Get(StateKept, PointKey("idx", "modules", 0, "abc123")); !found { - t.Error("the restore point is not recorded in the state kept") - } - if len(*acts) != 1 || !strings.Contains((*acts)[0], "took the restore point") { - t.Errorf("hand acts %v", *acts) - } - }}, - {"aside-only sets the live data aside whole and restores nothing", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", AsideOnly: true, Caller: "ace.restic", Why: "newer build stopped"} - if _, err := k.RestoreKept(context.Background(), r); err != nil { - t.Fatal(err) - } - if kb.streams["KV_idx_modules"] || len(kb.restored) != 0 { - t.Errorf("streams %v restored %v: the live data is removed and nothing restored", kb.streams, kb.restored) - } - aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside")) - if err != nil || len(aside) != 1 || r.Dir == "" { - t.Fatalf("aside: %v %v dir %q", aside, err, r.Dir) - } - }}, - {"a kept data the bus does not hold is refused naming what was looked for, and nothing is written", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - r := &Request{Module: "idx", Local: "gone", Fingerprint: "abc123"} - _, err := k.SnapshotKept(context.Background(), r) - if err == nil || !strings.Contains(err.Error(), "KV_idx_gone") { - t.Fatalf("err = %v", err) - } - if _, err := os.Stat(filepath.Join(k.KeptDir, "points")); err == nil { - t.Error("a directory was made for a snapshot that was not taken") - } - }}, - {"a name that is not one plain name is refused", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - for _, r := range []*Request{{Module: "../x", Local: "modules", Fingerprint: "a"}, {Module: "idx", Local: "mod/ules", Fingerprint: "a"}, {Module: "idx", Local: "modules", Fingerprint: ""}} { - if _, err := k.SnapshotKept(context.Background(), r); err == nil { - t.Errorf("%+v was taken", r) - } - } - }}, - {"a restore sets the live data aside whole, removes it, restores the archive and sets back the other consumers", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - p := take(t, k) - r := &Request{Verb: VerbRestoreKept, Module: "idx", Local: "modules", Form: 0, Fingerprint: "abc123", From: p.Dir, Caller: "ace.restic", Why: "revert"} - if _, err := k.RestoreKept(context.Background(), r); err != nil { - t.Fatal(err) - } - if !kb.streams["KV_idx_modules"] || len(kb.restored) != 1 { - t.Errorf("streams %v restored %v", kb.streams, kb.restored) - } - if r.CatchUp != "indexer_take_in" { - t.Errorf("the catch-up tool is %q", r.CatchUp) - } - aside, err := os.ReadDir(filepath.Join(k.KeptDir, "aside")) - if err != nil || len(aside) != 1 { - t.Fatalf("aside: %v %v", aside, err) - } - if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", aside[0].Name(), ArchiveFile)); err != nil { - t.Error("the live data's archive is not kept") - } - if _, _, found, _ := store.Get(StateKept, "aside."+aside[0].Name()); !found { - t.Error("what is set aside is not recorded") - } - // Only the consumer on a stream the point did not carry is made again; a stream's own comes with its snapshot. - if len(kb.set) != 1 || kb.set[0].Stream != "forge" || kb.set[0].AckFloor != 41 { - t.Errorf("positions set back: %+v", kb.set) - } - }}, - {"a restore point of another fingerprint or a linked file is refused before anything is removed", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - p := take(t, k) - bad := *p - bad.Fingerprint = "other" - if _, err := k.RestoreKept(context.Background(), &bad); err == nil || !kb.streams["KV_idx_modules"] { - t.Errorf("err = %v, live %v", err, kb.streams) - } - link := filepath.Join(p.Dir, ArchiveFile) - real := filepath.Join(p.Dir, "real.tar") - if err := os.Rename(link, real); err != nil { - t.Fatal(err) - } - if err := os.Symlink(real, link); err != nil { - t.Fatal(err) - } - r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} - if _, err := k.RestoreKept(context.Background(), r); err == nil || !strings.Contains(err.Error(), "plain file") || !kb.streams["KV_idx_modules"] { - t.Errorf("err = %v, live %v", err, kb.streams) - } - }}, - {"a live data that cannot be set aside stays where it is", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - p := take(t, k) - k.Bus.(*tarBus).fakeBus.refuse["snapshot"] = errors.New("the server refused") - r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} - if _, err := k.RestoreKept(context.Background(), r); err == nil || !kb.streams["KV_idx_modules"] || !strings.Contains(err.Error(), "nothing was removed") { - t.Errorf("err = %v, live %v", err, kb.streams) - } - }}, - {"a restore that fails puts the live data back and says so", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - p := take(t, k) - kb.refuse = errors.New("the server refused the archive") - r := &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir} - _, err := k.RestoreKept(context.Background(), r) - if err == nil || !strings.Contains(err.Error(), "the live data is back as it was") { - t.Fatalf("err = %v", err) - } - }}, - {"set-aside data goes only on a call that says why and is recorded first", func(t *testing.T, k *Keeper, kb *keptBus, store *memStore, acts *[]string) { - p := take(t, k) - if _, err := k.RestoreKept(context.Background(), &Request{Module: "idx", Local: "modules", Fingerprint: "abc123", From: p.Dir}); err != nil { - t.Fatal(err) - } - entries, _ := os.ReadDir(filepath.Join(k.KeptDir, "aside")) - id := entries[0].Name() - if _, err := k.DiscardAside(&Request{From: id}); err == nil { - t.Error("discarded with no why") - } - if _, err := k.DiscardAside(&Request{From: "../points", Why: "x"}); err == nil { - t.Error("discarded a path") - } - k.Act = func(string, string, string) error { return errors.New("the log is down") } - r := &Request{From: id, Why: "the revert held", Caller: "ace.operator"} - k.record(r, "x") // sets Unrecorded as a failing log does - if _, err := k.DiscardAside(r); err == nil { - t.Error("discarded with the act unrecorded") - } - if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err != nil { - t.Error("the data went with its act unrecorded") - } - k.Act = func(what, why, cause string) error { *acts = append(*acts, what); return nil } - if _, err := k.DiscardAside(&Request{From: id, Why: "the revert held", Caller: "ace.operator"}); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(filepath.Join(k.KeptDir, "aside", id)); err == nil { - t.Error("still there") - } - if _, _, found, _ := store.Get(StateKept, "aside."+id); found { - t.Error("its record stays") - } - }}, - } { - t.Run(row.name, func(t *testing.T) { - k, kb, store, acts := keptKeeper(t) - row.run(t, k, kb, store, acts) - }) - } -} diff --git a/modules/nats/internal/keeper/requests.go b/modules/nats/internal/keeper/requests.go index b7993c83d..d956a239e 100644 --- a/modules/nats/internal/keeper/requests.go +++ b/modules/nats/internal/keeper/requests.go @@ -46,15 +46,6 @@ type Request struct { Snapshot string `json:"snapshot,omitempty"` // Waiting is why it waits, while it waits: the operator's warrant, or the keeper's first view. Waiting string `json:"waiting,omitempty"` - // Form, Fingerprint and CatchUp name a restore point (kept.go); From is the directory a restore reads, or the id of the - // data a discard removes; Dir is what the keeper answers: the restore point's directory, or the data set aside. - Form int `json:"form,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` - CatchUp string `json:"catch-up,omitempty"` - // AsideOnly is the argument `aside-only` of restore-kept: the live data is set aside and nothing is restored. - AsideOnly bool `json:"aside-only,omitempty"` - From string `json:"from,omitempty"` - Dir string `json:"dir,omitempty"` // Unrecorded is why the hand act of what it did could not be recorded, when it could not: said, never only logged. Unrecorded string `json:"unrecorded,omitempty"` } @@ -145,10 +136,6 @@ func (k *Keeper) act(r *Request) (string, error) { } return fmt.Sprintf("the first view answered at revision %d", s.Revision), nil } - switch r.Verb { - case VerbSnapshotKept, VerbRestoreKept, VerbDiscardAside: - return k.actKept(r) - } if !k.Viewed() { return "", fmt.Errorf("%w: the keeper holds no first view yet, so it acts on nothing", ErrNoWarrant) } @@ -308,24 +295,3 @@ func (k *Keeper) record(r *Request, what string) { k.logf("the hand act %q could not be recorded: %v", what, err) } } - -// actKept acts on a request about a kept data's restore point. These need no first view or warrant: a snapshot changes -// nothing, a restore sets the live data aside whole before it removes it, and a discard names who and why (kept.go). -func (k *Keeper) actKept(r *Request) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute) - defer cancel() - var out string - var err error - switch r.Verb { - case VerbSnapshotKept: - out, err = k.SnapshotKept(ctx, r) - case VerbRestoreKept: - out, err = k.RestoreKept(ctx, r) - default: - out, err = k.DiscardAside(r) - } - if err != nil { - return "nothing done", err - } - return out, nil -} diff --git a/modules/nats/module.json b/modules/nats/module.json index f239c2e49..ad9d08019 100644 --- a/modules/nats/module.json +++ b/modules/nats/module.json @@ -32,9 +32,7 @@ "list", "show", "check", - "orphans", - "snapshot-kept", - "restore-kept" + "orphans" ] } ], @@ -100,10 +98,6 @@ "history": 1, "ttl-seconds": 86400, "read-only": true - }, - { - "name": "kept", - "history": 1 } ], "listens": [ @@ -263,8 +257,7 @@ "state": [ "asked", "requests", - "spent", - "kept" + "spent" ], "asks": true, "keeps": true, -- 2.54.0 From 9e2311029a493a8cfd8d79eff6ca9825c6afdefc Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:46:35 +0100 Subject: [PATCH 071/243] COND2: the keeper keeps its sources in the state findings-views and hears them before its first view of the catalog half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §388, §365 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/asks.go | 1 + .../cmd/mesh-conditions/catalog.go | 105 +++++++++++++++++- .../cmd/mesh-conditions/events.go | 4 + .../cmd/mesh-conditions/keeper_test.go | 27 +++++ .../cmd/mesh-conditions/main.go | 2 + .../cmd/mesh-conditions/manifest_test.go | 2 +- modules/mesh-conditions/module.json | 1 + 7 files changed, 138 insertions(+), 4 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/asks.go b/modules/mesh-conditions/cmd/mesh-conditions/asks.go index a380abd87..f9868baa5 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/asks.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/asks.go @@ -79,6 +79,7 @@ func (k *Keeper) Ask(by, module string) (string, error) { } var problems []string for _, w := range wait { + k.saveView(w[0], w[1]) if err := k.Store.CreateTTL(BoundsState, askedKey(w[0], w[1], ask), map[string]any{"observer": w[0], "node": w[1], "ask": ask, "until": now.Add(AskedWithin)}, AskedWithin); err != nil && !errors.Is(err, stdio.ErrChanged) { problems = append(problems, err.Error()) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go index 2b176aa2d..b513677b5 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go @@ -7,12 +7,17 @@ package main import ( "encoding/json" + "errors" + "maps" "slices" "sort" "strings" + "time" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" wants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" + stdio "git.novox.be/novox/mesh-sdk/go" engine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" ) @@ -48,14 +53,23 @@ func (k *Keeper) CatalogHeld(branch string, entries map[string]catseat.CatalogEn k.init() k.halves[branch] = entries var gone []string - for observer := range k.findings { - if observer == Module { - continue + seen := map[string]bool{} + consider := func(observer string) { + if observer == Module || observer == "" || seen[observer] { + return } + seen[observer] = true if _, ok := k.entry(observer); !ok { gone = append(gone, observer) } } + for observer := range k.findings { + consider(observer) + } + // A kept source (design 47 G §388) no revision lists any more leaves the list with its conditions. + for _, v := range k.views { + consider(v.observer) + } k.mu.Unlock() sort.Strings(gone) var first error @@ -85,6 +99,12 @@ func (k *Keeper) isSource(observer string) bool { k.mu.Lock() defer k.mu.Unlock() k.init() + // A source the keeper kept in `findings-views` is heard before its first view of the catalog half (design 47 G §388). + for _, v := range k.views { + if v.observer == observer && v.kept { + return true + } + } branches := make([]string, 0, len(k.halves)) for b := range k.halves { branches = append(branches, b) @@ -198,6 +218,7 @@ func (k *Keeper) sweepStopped() error { func (k *Keeper) clearObserver(observer, node, why string, first *error) { k.mu.Lock() var keys []string + var dropped [][2]string for key := range k.findings[observer] { if node == "" || k.keyNode[observer][key] == node { keys = append(keys, key) @@ -212,9 +233,13 @@ func (k *Keeper) clearObserver(observer, node, why string, first *error) { _ = k.Store.Delete(BoundsState, askedKey(observer, v.node, ask)) } delete(k.views, name) + dropped = append(dropped, [2]string{v.observer, v.node}) } } k.mu.Unlock() + for _, d := range dropped { + _ = k.Store.Delete(ViewsState, viewKey(d[0], d[1])) + } sort.Strings(keys) for _, key := range keys { k.mu.Lock() @@ -229,3 +254,77 @@ func (k *Keeper) clearObserver(observer, node, why string, first *error) { k.mu.Unlock() } } + +// ViewsState is the keeper's list of sources (design 47 G §388), in the form of its `conditions`: one key per observer and +// node, `.`, its value what the keeper heard there: the last stated set and the open asks. +const ViewsState = "findings-views" + +func viewKey(observer, node string) string { return observer + "." + node } + +// keptView is the value of one key of `findings-views`. +type keptView struct { + Observer string `json:"observer"` + Node string `json:"node"` + At time.Time `json:"at"` + Pending map[string]time.Time `json:"pending,omitempty"` + Asks map[string]time.Time `json:"asks,omitempty"` + Findings []seat.Finding `json:"findings,omitempty"` +} + +// saveView keeps what the keeper heard of the observer on the node, by compare-and-set; a source is kept once it is heard. +func (k *Keeper) saveView(observer, node string) { + k.mu.Lock() + k.init() + v := k.views[observer+"."+node] + if v == nil { + k.mu.Unlock() + return + } + v.kept = true + value := keptView{Observer: observer, Node: node, At: v.At, Pending: maps.Clone(v.Pending), Asks: maps.Clone(v.Asks), + Findings: slices.Clone(v.Findings)} + k.mu.Unlock() + key := viewKey(observer, node) + for range ConditionWrites { + _, rev, found, err := k.Store.Get(ViewsState, key) + if err == nil && !found { + _, err = k.Store.Create(ViewsState, key, value) + } else if err == nil { + _, err = k.Store.Update(ViewsState, key, value, rev) + } + if err == nil || !errors.Is(err, stdio.ErrChanged) { + if err != nil { + k.Logf("the source %s on %s is not kept: %v", observer, node, err) + } + return + } + } +} + +// RestoreViews reads the kept sources at the start, before the first view of the catalog half: their events are applied +// at once from the durable consumers, and the catalog half only adds and removes sources (design 47 G §388). +func (k *Keeper) RestoreViews() { + keys, err := k.Store.Keys(ViewsState) + if err != nil { + k.Logf("the kept sources are not read: %v", err) + return + } + for _, key := range keys { + raw, _, found, err := k.Store.Get(ViewsState, key) + if err != nil || !found { + continue + } + var kv keptView + if json.Unmarshal(raw, &kv) != nil || kv.Observer == "" || kv.Node == "" { + k.Logf("the kept source %s is not readable: left out", key) + continue + } + k.mu.Lock() + k.init() + v := k.view(kv.Observer, kv.Node) + v.At, v.Findings, v.kept = kv.At, kv.Findings, true + maps.Copy(v.Pending, kv.Pending) + maps.Copy(v.Asks, kv.Asks) + k.mu.Unlock() + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index ec7abe5d8..e0eb2c7b1 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -107,6 +107,7 @@ func (k *Keeper) take(observer, node string, f seat.Finding) error { if !named.IsZero() { _ = k.Store.Delete(BoundsState, pendingKey(observer, node, v.Key)) } + k.saveView(observer, node) return k.Finding(observer, v.Key, value, false) } @@ -236,6 +237,8 @@ type sourceView struct { Findings []seat.Finding // Asks are the asks it has not answered, with the end of their bound (asks.go). Asks map[string]time.Time + // kept says the source is in the state `findings-views` (catalog.go). + kept bool } // view is the observer's view on the node, made when there is none. Called with mu held. @@ -341,6 +344,7 @@ func (k *Keeper) FindingsStated(observer, node string, body []byte) error { _ = k.Store.Delete(BoundsState, askedKey(observer, node, ask)) } k.clearUnstated(observer, node) + k.saveView(observer, node) sort.Strings(drop) for key := range oldPending { if _, still := v.Pending[key]; !still { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 446b155fd..1f1836648 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -40,6 +40,12 @@ func (f *fakeStore) Get(state, key string) ([]byte, uint64, bool, error) { return v, f.revs[state+"/"+key], ok, nil } +// Get2 says whether the key is held. +func (f *fakeStore) Get2(state, key string) ([]byte, bool) { + v, _, ok, _ := f.Get(state, key) + return v, ok +} + func (f *fakeStore) put(state, key string, value any) uint64 { raw, _ := json.Marshal(value) f.next++ @@ -580,6 +586,27 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { } return r.k.CatalogHeld("main", map[string]catseat.CatalogEntry{watchmanModule: {}, selfCheckModule: {}}) }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftCatalog}, + // Design 47 G §388: a restarted keeper hears its kept sources before its first view of the catalog half. + {"a restarted keeper with no view of the catalog half hears its kept sources, and the view takes one away", func() error { + if _, ok := r.st.Get2(ViewsState, viewKey(watchman, "ace")); !ok { + t.Errorf("the source %s on ace was not kept in %s", watchman, ViewsState) + } + k2 := &Keeper{Store: r.st, Now: func() time.Time { return r.now }, Say: func(e Event) error { r.said = append(r.said, e); return nil }} + k2.RestoreViews() + if err := k2.FindingWritten(watchman, "ace", mustJSON(t, f(watchman, "k"))); err != nil { + return err + } + if _, ok := r.open("machine.k.silent"); !ok { + t.Errorf("a kept source's finding was not raised with no view of the catalog half") + } + if err := k2.FindingWritten("fixture", "ace", mustJSON(t, f("fixture", "q"))); err != nil { + return err + } + if _, ok := r.open("machine.q.silent"); ok { + t.Errorf("a module that was never a source raised a condition") + } + return k2.CatalogHeld("main", map[string]catseat.CatalogEntry{selfCheckModule: {}}) + }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftCatalog}, } for _, s := range steps { if err := s.do(); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index 9b5a81ddc..92c9ed5b3 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -109,6 +109,8 @@ func Keep(k *Keeper, v *views.Reader) { } } } + // The kept sources are heard at once, before the first view of the catalog half (design 47 G §388). + k.RestoreViews() said := func(err error) error { if err != nil { logf("%v", err) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 6dc3c9aa5..6b2d320ea 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -63,7 +63,7 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { } // Clause 12: each state with the tool that reads it, each act with the tool that starts it. readBy := map[string]string{ConditionsState: Seat + ".conditions", HistoryState: HistoryTool, - HeldState: HeldTool, BoundsState: HeldTool, "a resync": Seat + ".resync", "what it holds of observers": FindingsViewsTool, "what it reads": InputsTool} + HeldState: HeldTool, BoundsState: HeldTool, ViewsState: FindingsViewsTool, "a resync": Seat + ".resync", "what it holds of observers": FindingsViewsTool, "what it reads": InputsTool} tools := map[string]bool{} for _, tool := range Tools(&Keeper{}) { tools[tool.Name] = true diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index b0431616d..f9aed5620 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -41,6 +41,7 @@ "ttl-seconds": 7776000 }, "held", + "findings-views", { "name": "bounds", "key-ttl": true -- 2.54.0 From 76be864bff12f1119d8d8184eb895089fe5e180f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:48:35 +0100 Subject: [PATCH 072/243] HEAL2 (manifest only): mesh-healers asks findings and finding-events in requires; the writer's use waits on the contracts' finding subjects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §529, §384, §461 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-healers/module.json | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/modules/mesh-healers/module.json b/modules/mesh-healers/module.json index 7fe853339..4bd6f46ea 100644 --- a/modules/mesh-healers/module.json +++ b/modules/mesh-healers/module.json @@ -35,7 +35,6 @@ "name": "would", "key-ttl": true }, - "findings", { "name": "wakes", "key-ttl": true @@ -62,14 +61,6 @@ "unknown": "program" } ], - "streams": [ - { - "name": "mesh-healers-findings", - "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], - "max-age-days": 7, - "max-mib": 16 - } - ], "build": { "artifacts": [ { @@ -84,5 +75,26 @@ ] } ] - } + }, + "requires": [ + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ] } -- 2.54.0 From 26c82a507a7719cc263e2decce8b8be80b7b0b39 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:48:35 +0100 Subject: [PATCH 073/243] OBS4: artifact-store-tools, mesh-watcher, minio and nfs-server ask findings and finding-events in requires MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §529, §387 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/artifact-store-tools/store_test.go | 9 +++-- modules/artifact-store-tools/module.json | 34 ++++++++++++------- modules/mesh-watcher/module.json | 34 ++++++++++++------- modules/minio/module.json | 32 ++++++++++------- modules/nfs-server/module.json | 32 +++++++++++------ 5 files changed, 93 insertions(+), 48 deletions(-) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go index 8156df52c..e7e183262 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/store_test.go @@ -480,8 +480,13 @@ func TestTheToolsServedAreTheToolsTheManifestNames(t *testing.T) { } // The operator, 2026-11-04 (issue 1966): it reaches the store through the registry seats' verbs, and requires no // provision artifact-store. - if len(m.Requires) != 0 { - t.Errorf("requires %v, want nothing: the store is reached through the registry seats", m.Requires) + // What it requires is its finding state and stream alone, in their template form (design 47 G §529). + var required []string + for _, r := range m.Requires { + required = append(required, fmt.Sprint(r.(map[string]any)["name"])) + } + if !reflect.DeepEqual(required, []string{"findings", "finding-events"}) { + t.Errorf("requires %v, want its findings and finding-events alone: the store is reached through the registry seats", m.Requires) } // Its address file is filled from the image seat holder's listen (ADR 0632 §1), never by the retired provision. address, err := os.ReadFile("../../files/store-address") diff --git a/modules/artifact-store-tools/module.json b/modules/artifact-store-tools/module.json index 8ae3fe620..b0e2e25a8 100644 --- a/modules/artifact-store-tools/module.json +++ b/modules/artifact-store-tools/module.json @@ -42,9 +42,6 @@ "mesh-delivery-indexer.views.manifests", "mesh-conditions.findings-asked" ], - "state": [ - "findings" - ], "tools": [ "artifact_store_repositories", "artifact_store_usage", @@ -86,14 +83,6 @@ "template": "files/store-address" } ], - "streams": [ - { - "name": "artifact-store-tools-findings", - "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], - "max-age-days": 7, - "max-mib": 16 - } - ], "build": { "artifacts": [ { @@ -117,5 +106,26 @@ "from": "files" } ] - } + }, + "requires": [ + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ] } diff --git a/modules/mesh-watcher/module.json b/modules/mesh-watcher/module.json index 5a855c5fb..c4d8d9f98 100644 --- a/modules/mesh-watcher/module.json +++ b/modules/mesh-watcher/module.json @@ -12,9 +12,6 @@ "mesh-watcher.watcher_ping", "seat:mesh-hand-acts.hand-act" ], - "state": [ - "findings" - ], "own-secrets": { "telegram-token": "${dir:state}/telegram-token" }, @@ -54,14 +51,6 @@ "template": "files/settings.json" } ], - "streams": [ - { - "name": "mesh-watcher-findings", - "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], - "max-age-days": 7, - "max-mib": 16 - } - ], "build": { "artifacts": [ { @@ -85,5 +74,26 @@ "from": "files" } ] - } + }, + "requires": [ + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ] } diff --git a/modules/minio/module.json b/modules/minio/module.json index 47f19cdf6..60800260d 100644 --- a/modules/minio/module.json +++ b/modules/minio/module.json @@ -19,7 +19,26 @@ } ], "requires": [ - "route" + "route", + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } ], "seats": [ { @@ -47,9 +66,6 @@ "mesh-delivery-assigner.views.seat-holders", "mesh-conditions.findings-asked" ], - "state": [ - "findings" - ], "contributes": { "route": { "api": { @@ -193,14 +209,6 @@ "package": "minio-client" } ], - "streams": [ - { - "name": "minio-findings", - "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], - "max-age-days": 7, - "max-mib": 16 - } - ], "build": { "artifacts": [ { diff --git a/modules/nfs-server/module.json b/modules/nfs-server/module.json index 6f77dba2c..9e7d61b69 100644 --- a/modules/nfs-server/module.json +++ b/modules/nfs-server/module.json @@ -48,8 +48,7 @@ "name": "exports", "ttl-seconds": 120, "per-machine": true - }, - "findings" + } ], "reads": [ "disk-load.readings", @@ -172,14 +171,6 @@ "template": "files/shares.conf" } ], - "streams": [ - { - "name": "nfs-server-findings", - "events": ["finding-written.*", "finding-cleared.*", "findings-stated.*"], - "max-age-days": 7, - "max-mib": 16 - } - ], "build": { "artifacts": [ { @@ -207,5 +198,26 @@ "consumes": [ "mesh-delivery-assigner.views.seat-holders", "mesh-conditions.findings-asked" + ], + "requires": [ + { + "name": "findings", + "storage": "key-value", + "template": "state" + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } ] } -- 2.54.0 From ef60e5d09eea3dd757d7e604d60a702d117c5263 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 15:49:13 +0100 Subject: [PATCH 074/243] COND2 (whole): the keeper's whole test starts with a kept source and no view of the catalog half MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §388 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/whole_integration_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go index 36292d779..48af4aaf2 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/whole_integration_test.go @@ -294,7 +294,7 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { } buckets := []string{"mesh-controller_conditions", "mesh-delivery-assigner_wants", "node-engine_have", "mesh-delivery-assigner_assignments", "mesh-healers_attempts"} - for _, s := range []string{ConditionsState, HistoryState, HeldState, BoundsState} { + for _, s := range []string{ConditionsState, HistoryState, HeldState, ViewsState, BoundsState} { buckets = append(buckets, Module+"_"+s) } for _, b := range buckets { @@ -302,6 +302,12 @@ func TestTheKeeperWholeRaisesAndClearsOnTheBus(t *testing.T) { t.Fatal(err) } } + // Design 47 G §388: the catalog's holder is stopped, and the keeper hears the source it kept at an earlier run. + views, _ := js.KeyValue(ctx, Module+"_"+ViewsState) + kept, _ := json.Marshal(keptView{Observer: watchmanModule, Node: "ace"}) + if _, err := views.Create(ctx, viewKey(watchmanModule, "ace"), kept); err != nil { + t.Fatal(err) + } events, err := nc.SubscribeSync("mesh.seat." + Seat + ".event.>") if err != nil { t.Fatal(err) -- 2.54.0 From 8f4f70f08875b88b955183e3b7d8cd2105ec49c1 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:26:40 +0100 Subject: [PATCH 075/243] COND6: the keeper follows one catalog view per watched branch and adopts a branch registered while it runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §365, §479, §521, §532 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/branches.go | 128 ++++++++++++++++++ .../cmd/mesh-conditions/branches_test.go | 43 ++++++ .../cmd/mesh-conditions/catalog.go | 14 ++ .../cmd/mesh-conditions/keeper_test.go | 10 ++ .../cmd/mesh-conditions/main.go | 10 ++ .../cmd/mesh-conditions/manifest_test.go | 5 +- modules/mesh-conditions/module.json | 6 +- 7 files changed, 213 insertions(+), 3 deletions(-) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/branches.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/branches_test.go diff --git a/modules/mesh-conditions/cmd/mesh-conditions/branches.go b/modules/mesh-conditions/cmd/mesh-conditions/branches.go new file mode 100644 index 000000000..f65d63ec8 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/branches.go @@ -0,0 +1,128 @@ +package main + +// The keeper follows one view of the catalog half per watched branch, the branches the registrations name, and adopts a +// branch registered while it runs with no restart (design 47 G §532, §365, §479). No branch is named here (§7). + +import ( + "encoding/json" + "slices" + "sort" + "sync" + "time" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// Branches keeps the keeper's views of the catalog half, one per watched branch. +type Branches struct { + // Open adopts the view of the branch's catalog half and answers its name in the reader; Drop ends it, and Dropped + // forgets what the keeper held of the branch. + Open func(branch string) (string, error) + Drop func(name string) error + Dropped func(branch string) error + Logf func(string, ...any) + + mu sync.Mutex + names map[string]string +} + +// Watched sets the watched branches to the ones the registrations name: a branch new to them is adopted, a branch no +// registration names any more is dropped. +func (b *Branches) Watched(regs []catseat.Registration) { + want := map[string]bool{} + for _, r := range regs { + for _, branch := range r.Branches { + want[keys.EscapeName(branch)] = true + } + } + b.mu.Lock() + if b.names == nil { + b.names = map[string]string{} + } + var adopt, drop []string + for branch := range want { + if _, ok := b.names[branch]; !ok { + adopt = append(adopt, branch) + } + } + for branch := range b.names { + if !want[branch] { + drop = append(drop, branch) + } + } + b.mu.Unlock() + sort.Strings(adopt) + sort.Strings(drop) + for _, branch := range adopt { + name, err := b.Open(branch) + if err != nil { + b.Logf("the catalog half of %s is not followed: %v", branch, err) + continue + } + b.mu.Lock() + b.names[branch] = name + b.mu.Unlock() + } + for _, branch := range drop { + b.mu.Lock() + name := b.names[branch] + delete(b.names, branch) + b.mu.Unlock() + if err := b.Drop(name); err != nil { + b.Logf("the view %s is not dropped: %v", name, err) + } + if err := b.Dropped(branch); err != nil { + b.Logf("the conditions of the dropped branch %s are not cleared: %v", branch, err) + } + } +} + +// Followed is the branches followed, in name order. +func (b *Branches) Followed() []string { + b.mu.Lock() + defer b.mu.Unlock() + out := make([]string, 0, len(b.names)) + for branch := range b.names { + out = append(out, branch) + } + slices.Sort(out) + return out +} + +// followCatalog is Branches over the keeper's reader: each branch's view is the catalog client's, whose changes hand the +// branch's whole half to the keeper. +func followCatalog(k *Keeper, v *views.Reader, call meshcatalogclient.Call) *Branches { + return &Branches{ + Logf: logf, + Open: func(branch string) (string, error) { + var mu sync.Mutex + var c *meshcatalogclient.Catalog + changed := func(string, json.RawMessage, bool, time.Time) { + mu.Lock() + view := c + mu.Unlock() + if view == nil { + return + } + if half, ok := view.View(); ok { + if err := k.CatalogHeld(branch, half.Entries); err != nil { + logf("%v", err) + } + } + } + mu.Lock() + defer mu.Unlock() + made, err := meshcatalogclient.WireCatalog(v, call, branch, changed) + if err != nil { + return "", err + } + c = made + return made.Follower.Owner + "." + made.Follower.State, nil + }, + Drop: v.Drop, + Dropped: k.CatalogDropped, + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/branches_test.go b/modules/mesh-conditions/cmd/mesh-conditions/branches_test.go new file mode 100644 index 000000000..f7bbd1d8f --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/branches_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "slices" + "testing" + + catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" +) + +// Design 47 G §532: the keeper adopts a branch a registration adds while it runs, and drops one no registration names. +func TestTheKeeperFollowsTheBranchesTheRegistrationsName(t *testing.T) { + var opened, dropped, forgotten []string + b := &Branches{ + Logf: func(string, ...any) {}, + Open: func(branch string) (string, error) { + opened = append(opened, branch) + return "mesh-catalog." + branch, nil + }, + Drop: func(name string) error { dropped = append(dropped, name); return nil }, + Dropped: func(branch string) error { forgotten = append(forgotten, branch); return nil }, + } + reg := func(module string, branches ...string) catseat.Registration { + return catseat.Registration{Module: module, Branches: branches} + } + steps := []struct { + name string + regs []catseat.Registration + followed, opened, dropped, forgotten []string + }{ + {"the first registrations", []catseat.Registration{reg("a", "trunk")}, []string{"trunk"}, []string{"trunk"}, nil, nil}, + {"a branch registered while it runs", []catseat.Registration{reg("a", "trunk"), reg("b", "feature/x")}, []string{"feature/x", "trunk"}, []string{"trunk", "feature/x"}, nil, nil}, + {"a branch nobody names is dropped", []catseat.Registration{reg("a", "trunk")}, []string{"trunk"}, []string{"trunk", "feature/x"}, []string{"mesh-catalog.feature/x"}, []string{"feature/x"}}, + } + for _, s := range steps { + b.Watched(s.regs) + if got := b.Followed(); !slices.Equal(got, s.followed) { + t.Errorf("%s: followed %v, want %v", s.name, got, s.followed) + } + if !slices.Equal(opened, s.opened) || !slices.Equal(dropped, s.dropped) || !slices.Equal(forgotten, s.forgotten) { + t.Errorf("%s: opened %v dropped %v forgotten %v", s.name, opened, dropped, forgotten) + } + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go index b513677b5..f7f89381c 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go @@ -52,6 +52,20 @@ func (k *Keeper) CatalogHeld(branch string, entries map[string]catseat.CatalogEn k.mu.Lock() k.init() k.halves[branch] = entries + return k.sweepCatalog() +} + +// CatalogDropped forgets a branch no registration watches any more (design 47 G §532): the modules only it listed leave +// the sources with their conditions. +func (k *Keeper) CatalogDropped(branch string) error { + k.mu.Lock() + k.init() + delete(k.halves, branch) + return k.sweepCatalog() +} + +// sweepCatalog clears the conditions of every observer no held revision lists. Called with mu held; it releases it. +func (k *Keeper) sweepCatalog() error { var gone []string seen := map[string]bool{} consider := func(observer string) { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 1f1836648..31e11071f 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -607,6 +607,16 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { } return k2.CatalogHeld("main", map[string]catseat.CatalogEntry{selfCheckModule: {}}) }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftCatalog}, + // Design 47 G §532: a module only a second watched branch lists is a source, and the branch dropped takes it away. + {"a module listed only by a second branch is a source", func() error { + if err := r.k.CatalogHeld("feature", map[string]catseat.CatalogEntry{"fixture": {Streams: []string{EventFindingWritten}}}); err != nil { + return err + } + return r.k.FindingWritten("fixture", "ace", mustJSON(t, f("fixture", "z"))) + }, []string{"machine.d.silent", "machine.g.silent", "machine.z.silent"}, ""}, + {"the second branch dropped takes its source away", func() error { + return r.k.CatalogDropped("feature") + }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftCatalog}, } for _, s := range steps { if err := s.do(); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index 92c9ed5b3..55f780583 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -18,7 +18,9 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" + catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" meshwants "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -156,6 +158,14 @@ func NewViews(k *Keeper) *views.Reader { if _, err := meshwantsclient.WireAssignment(v, stdio.Ask, func(rev meshwants.AssignmentRevision) { _ = k.AssignmentHeld(rev) }); err != nil { logf("the assignment half is not followed: %v", err) } + // One view of the catalog half per watched branch, the branches the registrations name, adopted and dropped while the + // keeper runs (design 47 G §365, §532). + branches := followCatalog(k, v, stdio.Ask) + var regs *meshcatalogclient.Registrations + regs, err := meshcatalogclient.WireRegistrations(v, stdio.Ask, func(catseat.Registration, bool) { branches.Watched(regs.All()) }) + if err != nil { + logf("the registrations are not followed: %v", err) + } return v } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 6b2d320ea..273a924a0 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -36,9 +36,10 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { // The assigner's wants and assignments are asked once of their read tools and followed by their events (ADR 0615 §1); // the observers' findings are heard as events and no `findings` is read (design 47 G §383). reads := []string{HavesRead, AttemptsRead} - if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.revision"}) || + if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.revision", + "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision"}) || !slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments", - "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-wants.assignment-revision", "node-engine.running"}) { + "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-wants.assignment-revision", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "node-engine.running"}) { t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes) } for _, r := range m.Reads { diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index f9aed5620..66cced877 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -21,7 +21,9 @@ "mesh-delivery-assigner.assigner_read_wants", "mesh-delivery-assigner.assigner_read_assignments", "seat:mesh-hand-acts.hand-act", - "seat:mesh-wants.revision" + "seat:mesh-wants.revision", + "seat:mesh-catalog.registrations", + "seat:mesh-catalog.revision" ], "consumes": [ "mesh-delivery-assigner.views.wants", @@ -32,6 +34,8 @@ "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-wants.assignment-revision", + "mesh-catalog.registration-changed", + "mesh-catalog.catalog-revision", "node-engine.running" ], "state": [ -- 2.54.0 From 811b59b4540dfc7e13e5d9068b193f758f03af50 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:27:47 +0100 Subject: [PATCH 076/243] COND9: the keeper takes a part whose finding stream the bus module made as a source (the moved-stream half stops on a question) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §515, §663 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/bound.go | 73 +++++++++++++++++++ .../cmd/mesh-conditions/catalog.go | 4 + .../cmd/mesh-conditions/events.go | 6 +- .../cmd/mesh-conditions/keeper.go | 2 + .../cmd/mesh-conditions/keeper_test.go | 18 +++++ .../cmd/mesh-conditions/main.go | 1 + .../cmd/mesh-conditions/manifest_test.go | 4 +- modules/mesh-conditions/go.mod | 1 + modules/mesh-conditions/module.json | 4 +- 9 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/bound.go diff --git a/modules/mesh-conditions/cmd/mesh-conditions/bound.go b/modules/mesh-conditions/cmd/mesh-conditions/bound.go new file mode 100644 index 000000000..edff8dac4 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/bound.go @@ -0,0 +1,73 @@ +package main + +// The keeper's sources are the parts whose finding stream the bus module made (design 47 G §515, §663): it takes a source +// when it hears the seat mesh-broker's `created` of a part's `finding-events` entry, and at its start reads the entries +// the bus module holds by `list`. No part is named here (§7). + +import ( + "encoding/json" + "fmt" + "time" + + broker "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" +) + +// FindingEventsTemplate is the template of a part's finding stream in its `requires` (design 47 G §529). +const FindingEventsTemplate = "finding-events" + +// BrokerCreated takes one `created` of the bus module: a `finding-events` entry makes its module a source, asked once with +// `findings-asked` naming it. An entry of another template, or a module already a source, changes nothing. +func (k *Keeper) BrokerCreated(body []byte) error { + var said broker.Said + if err := json.Unmarshal(body, &said); err != nil || said.Module == "" { + k.Logf("a created of the bus module is not readable: %v", err) + return nil + } + if said.Template != FindingEventsTemplate { + return nil + } + return k.takeSource(said.Module) +} + +// takeSource makes the module a source the bus module made a stream for, and asks it once. +func (k *Keeper) takeSource(module string) error { + k.mu.Lock() + k.init() + if k.created == nil { + k.created = map[string]bool{} + } + if k.created[module] { + k.mu.Unlock() + return nil + } + k.created[module] = true + k.mu.Unlock() + if _, err := k.Ask(Module, module); err != nil { + return fmt.Errorf("asking %s for its findings: %w", module, err) + } + return nil +} + +// ListBound reads the entries the bus module holds at the keeper's start: every module with a `finding-events` entry is a +// source. A call that fails is asked again with a growing wait, the bus module being delivered after the keeper at times. +func ListBound(k *Keeper, call func(key string, body any) (json.RawMessage, error)) { + for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { + raw, err := call("seat:"+broker.Seat+"."+broker.VerbList, broker.ListInput{}) + var answer broker.ListAnswer + if err == nil { + err = json.Unmarshal(raw, &answer) + } + if err == nil { + for _, e := range answer.Entries { + if e.Template == FindingEventsTemplate { + if err := k.takeSource(e.Module); err != nil { + k.Logf("%v", err) + } + } + } + return + } + k.Logf("the entries of the bus module are not read yet (%v); asked again in %s", err, wait) + time.Sleep(wait) + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go index f7f89381c..19e16487c 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go @@ -113,6 +113,10 @@ func (k *Keeper) isSource(observer string) bool { k.mu.Lock() defer k.mu.Unlock() k.init() + // A part whose finding stream the bus module made is a source (design 47 G §663). + if k.created[observer] { + return true + } // A source the keeper kept in `findings-views` is heard before its first view of the catalog half (design 47 G §388). for _, v := range k.views { if v.observer == observer && v.kept { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index e0eb2c7b1..f2be01310 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -14,6 +14,7 @@ import ( "strings" "time" + broker "git.novox.be/novox/mesh-contracts/go/seats/meshbroker" engine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" @@ -51,6 +52,9 @@ func (k *Keeper) FindingEvent(e stdio.Envelope) error { if e.Key == engine.Seat+"."+engine.EventRunning { return k.EngineRunning(e.Node, e.Body) } + if e.Key == broker.Seat+"."+broker.EventCreated { + return k.BrokerCreated(e.Body) + } if observer, ok := emitterOf(e.Key, EventFindingsStated); ok { return k.FindingsStated(observer, e.Node, e.Body) } @@ -202,7 +206,7 @@ func (k *Keeper) MachinePresence(body []byte, away bool) error { func hear(k *Keeper) { patterns := []string{"*." + EventFindingWritten, "*." + EventFindingCleared, "*." + EventFindingsStated, watchdogs.Seat + "." + watchdogs.EventMachineAway, watchdogs.Seat + "." + watchdogs.EventMachineBack, - engine.Seat + "." + engine.EventRunning} + engine.Seat + "." + engine.EventRunning, broker.Seat + "." + broker.EventCreated} for _, pattern := range patterns { go func() { <-stdio.Served() diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 6753dffc2..a8598441d 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -147,6 +147,8 @@ type Keeper struct { placed map[string]bool placedHeld bool stopped map[string]bool + // created are the modules whose `finding-events` entry the bus module made (bound.go). + created map[string]bool } func (k *Keeper) init() { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 31e11071f..3d47e4f62 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -617,6 +617,24 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { {"the second branch dropped takes its source away", func() error { return r.k.CatalogDropped("feature") }, []string{"machine.d.silent", "machine.g.silent"}, whyLeftCatalog}, + // Design 47 G §663: a part whose finding-events entry the bus module made is a source, asked once. + {"a created finding-events entry makes its part a source, asked once", func() error { + before := len(asked) + created := mustJSON(t, map[string]any{"module": "newpart", "template": FindingEventsTemplate, "bus-name": "newpart_finding-events", "local-name": "finding-events"}) + other := mustJSON(t, map[string]any{"module": "otherpart", "template": "state", "bus-name": "otherpart_x", "local-name": "x"}) + for _, body := range [][]byte{created, other, created} { + if err := r.k.FindingEvent(stdio.Envelope{Key: "mesh-broker.created", Body: body}); err != nil { + return err + } + } + if len(asked) != before+1 || asked[before]["module"] != "newpart" { + t.Errorf("asked %v, want newpart once", asked[before:]) + } + if err := r.k.FindingWritten("otherpart", "ace", mustJSON(t, f("otherpart", "o"))); err != nil { + return err + } + return r.k.FindingWritten("newpart", "ace", mustJSON(t, f("newpart", "w"))) + }, []string{"machine.d.silent", "machine.g.silent", "machine.w.silent"}, ""}, } for _, s := range steps { if err := s.do(); err != nil { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index 55f780583..a185af451 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -130,6 +130,7 @@ func Keep(k *Keeper, v *views.Reader) { return said(k.Attempt(c.Key, c.Value, c.Op != "put")) }) hear(k) + go func() { <-stdio.Served(); ListBound(k, stdio.Ask) }() watch(k, v, BoundsState, "", func(c stdio.StateChange) error { if c.Op == stdio.OpExpire { return said(k.BoundPassed(c.Key)) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go index 273a924a0..73d3b0920 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/manifest_test.go @@ -37,9 +37,9 @@ func TestTheManifestSaysWhatTheCodeDoes(t *testing.T) { // the observers' findings are heard as events and no `findings` is read (design 47 G §383). reads := []string{HavesRead, AttemptsRead} if !slices.Equal(m.Invokes, []string{WantsTool, AssignmentsTool, "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.revision", - "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision"}) || + "seat:mesh-catalog.registrations", "seat:mesh-catalog.revision", "seat:mesh-broker.list"}) || !slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.wants", "mesh-delivery-assigner.views.assignments", - "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-wants.assignment-revision", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "node-engine.running"}) { + "*.finding-written", "*.finding-cleared", "*.findings-stated", "mesh-watchdogs.machine-away", "mesh-watchdogs.machine-back", "mesh-wants.assignment-revision", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", "mesh-broker.created", "node-engine.running"}) { t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes) } for _, r := range m.Reads { diff --git a/modules/mesh-conditions/go.mod b/modules/mesh-conditions/go.mod index 7ce3ace27..13d6fdc7e 100644 --- a/modules/mesh-conditions/go.mod +++ b/modules/mesh-conditions/go.mod @@ -5,6 +5,7 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-contracts/go/seats/meshbroker v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index 66cced877..569f6071a 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -23,7 +23,8 @@ "seat:mesh-hand-acts.hand-act", "seat:mesh-wants.revision", "seat:mesh-catalog.registrations", - "seat:mesh-catalog.revision" + "seat:mesh-catalog.revision", + "seat:mesh-broker.list" ], "consumes": [ "mesh-delivery-assigner.views.wants", @@ -36,6 +37,7 @@ "mesh-wants.assignment-revision", "mesh-catalog.registration-changed", "mesh-catalog.catalog-revision", + "mesh-broker.created", "node-engine.running" ], "state": [ -- 2.54.0 From ecf597adce36b757d9e3656670d39525235145e6 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:29:24 +0100 Subject: [PATCH 077/243] COND11 (in part): the keeper's ask bound and first-view bound are settings read at its start; the two hours wait on COND7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §557 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/asks.go | 16 ++++++--- .../cmd/mesh-conditions/keeper.go | 2 ++ .../cmd/mesh-conditions/main.go | 2 ++ .../cmd/mesh-conditions/settings.go | 36 +++++++++++++++++++ .../cmd/mesh-conditions/settings_test.go | 32 +++++++++++++++++ modules/mesh-conditions/module.json | 14 ++++++++ 6 files changed, 98 insertions(+), 4 deletions(-) create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/settings.go create mode 100644 modules/mesh-conditions/cmd/mesh-conditions/settings_test.go diff --git a/modules/mesh-conditions/cmd/mesh-conditions/asks.go b/modules/mesh-conditions/cmd/mesh-conditions/asks.go index f9868baa5..b9bff6d23 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/asks.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/asks.go @@ -18,9 +18,17 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" ) -// AskedWithin is how long an observer has to state its findings after an ask. +// AskedWithin is how long an observer has to state its findings after an ask, when the setting says none (design 47 G §557). const AskedWithin = 5 * time.Minute +// askedWithin is the keeper's bound for an ask: its setting, AskedWithin when none. +func (k *Keeper) askedWithin() time.Duration { + if k.AskBound > 0 { + return k.AskBound + } + return AskedWithin +} + // The kind of the keeper's finding of an observer that stated nothing, and the prefix of its ask's key in `bounds`. const ( KindFindingsUnstated = "findings-unstated" @@ -62,7 +70,7 @@ func (k *Keeper) Ask(by, module string) (string, error) { if v.observer == "" || module != "" && v.observer != module || k.away[v.node] { continue } - v.Asks[ask] = now.Add(AskedWithin) + v.Asks[ask] = now.Add(k.askedWithin()) wait = append(wait, [2]string{v.observer, v.node}) } k.mu.Unlock() @@ -81,7 +89,7 @@ func (k *Keeper) Ask(by, module string) (string, error) { for _, w := range wait { k.saveView(w[0], w[1]) if err := k.Store.CreateTTL(BoundsState, askedKey(w[0], w[1], ask), map[string]any{"observer": w[0], "node": w[1], "ask": ask, - "until": now.Add(AskedWithin)}, AskedWithin); err != nil && !errors.Is(err, stdio.ErrChanged) { + "until": now.Add(k.askedWithin())}, k.askedWithin()); err != nil && !errors.Is(err, stdio.ErrChanged) { problems = append(problems, err.Error()) } } @@ -138,7 +146,7 @@ func (k *Keeper) AskPassed(bound string) error { now := k.Now().UTC() f := seat.FindingsByKeyValue{V: cc.FindingVersion, Key: unstatedKey(observer, node), Kind: KindFindingsUnstated, Subject: seat.Subject{Scope: cc.ScopeModule, ID: observer + "." + node, Machine: node}, - Severity: cc.Warning, Summary: fmt.Sprintf("%s on %s stated no findings within %s of the ask", observer, node, AskedWithin), + Severity: cc.Warning, Summary: fmt.Sprintf("%s on %s stated no findings within %s of the ask", observer, node, k.askedWithin()), Headline: observer + " on " + node + " did not state its findings", Explanation: "The keeper asked " + observer + " on " + node + " to state its findings and heard nothing; its conditions stand as they were.", Source: "findings-asked", Observer: Module, At: now, Evidence: []seat.Evidence{{At: now, Said: "ask " + ask + " unanswered"}}} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index a8598441d..154773e4d 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -107,6 +107,8 @@ type Keeper struct { Homes map[string]Home // Bound is a delivery's hold bound; motion.DefaultDeliveryBound when zero (hold.go). Bound time.Duration + // AskBound is how long an observer has to state its findings after an ask (the setting ask-bound-seconds, §557); AskedWithin when zero. + AskBound time.Duration mu sync.Mutex findings map[string]map[string]seat.FindingsByKeyValue diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index a185af451..90fd771f6 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -193,6 +193,8 @@ func main() { v := NewViews(k) go func() { <-stdio.Served() + // The bounds are settings, read at the start once the runtime serves (design 47 G §557). + v.Bound = ReadSettings(k, stdio.Config) v.Start(context.Background()) Keep(k, v) }() diff --git a/modules/mesh-conditions/cmd/mesh-conditions/settings.go b/modules/mesh-conditions/cmd/mesh-conditions/settings.go new file mode 100644 index 000000000..b29fe7596 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/settings.go @@ -0,0 +1,36 @@ +package main + +// The bounds the keeper times are settings read at its start, their defaults the design's values, so a whole test sets +// them in seconds and never waits on real time (design 47 G §557). + +import ( + "encoding/json" + "time" +) + +// The settings, in seconds. +const ( + SettingAskBound = "ask-bound-seconds" + SettingViewWaits = "view-waits-seconds" + defaultViewWaits = 5 * time.Minute +) + +// seconds reads one setting as a duration; def when the setting is unreadable, absent or not above zero. +func seconds(read func(key string) (json.RawMessage, error), key string, def time.Duration) time.Duration { + raw, err := read(key) + if err != nil { + logf("the setting %s cannot be read (%v); its default %s stands", key, err, def) + return def + } + var n float64 + if json.Unmarshal(raw, &n) != nil || n <= 0 { + return def + } + return time.Duration(n * float64(time.Second)) +} + +// ReadSettings sets the keeper's bounds and the first views' wait from the settings. +func ReadSettings(k *Keeper, read func(key string) (json.RawMessage, error)) (viewWaits time.Duration) { + k.AskBound = seconds(read, SettingAskBound, AskedWithin) + return seconds(read, SettingViewWaits, defaultViewWaits) +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/settings_test.go b/modules/mesh-conditions/cmd/mesh-conditions/settings_test.go new file mode 100644 index 000000000..9d9f48e82 --- /dev/null +++ b/modules/mesh-conditions/cmd/mesh-conditions/settings_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "encoding/json" + "errors" + "testing" + "time" +) + +// Design 47 G §557: the bounds the keeper times are settings, their defaults the design's values. +func TestTheKeepersBoundsAreSettings(t *testing.T) { + for _, c := range []struct { + name string + read func(string) (json.RawMessage, error) + ask, view time.Duration + }{ + {"unset", func(string) (json.RawMessage, error) { return nil, errors.New("none") }, AskedWithin, defaultViewWaits}, + {"set in seconds", func(key string) (json.RawMessage, error) { + if key == SettingAskBound { + return json.RawMessage("2"), nil + } + return json.RawMessage("3"), nil + }, 2 * time.Second, 3 * time.Second}, + {"not above zero", func(string) (json.RawMessage, error) { return json.RawMessage("0"), nil }, AskedWithin, defaultViewWaits}, + } { + k := &Keeper{} + view := ReadSettings(k, c.read) + if k.askedWithin() != c.ask || view != c.view { + t.Errorf("%s: ask bound %s, view bound %s; want %s, %s", c.name, k.askedWithin(), view, c.ask, c.view) + } + } +} diff --git a/modules/mesh-conditions/module.json b/modules/mesh-conditions/module.json index 569f6071a..a59a68797 100644 --- a/modules/mesh-conditions/module.json +++ b/modules/mesh-conditions/module.json @@ -53,6 +53,20 @@ "key-ttl": true } ], + "config": { + "ask-bound-seconds": { + "type": "number", + "description": "How long an observer has to state its findings after an ask, in seconds, before the keeper raises findings-unstated for it: 300 is five minutes.", + "minimum": 1, + "default": 300 + }, + "view-waits-seconds": { + "type": "number", + "description": "How long a first view may stay unanswered, in seconds, before the keeper raises view-waits for it: 300 is five minutes.", + "minimum": 1, + "default": 300 + } + }, "own-secrets": { "broker": "${dir:state}/broker" }, -- 2.54.0 From 1af839fb727a0a80e6deef0dc604cd49bf067c0d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 20:44:44 +0100 Subject: [PATCH 078/243] COND9 (moved-stream half): the keeper keeps a moved part's conditions, drops the old source and asks once on the new stream MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §663, §690 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/catalog.go | 66 ++++++++++++++++++- .../cmd/mesh-conditions/keeper.go | 2 + .../cmd/mesh-conditions/keeper_test.go | 41 ++++++++++++ 3 files changed, 106 insertions(+), 3 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go index 19e16487c..4b9cb39dc 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/catalog.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/catalog.go @@ -46,13 +46,60 @@ func declaresFindings(e catseat.CatalogEntry) bool { return false } +// findingStreams names the streams the entry declares its finding events in, one string, in name order; none is "". +func findingStreams(e catseat.CatalogEntry) string { + var names []string + for _, s := range e.Streams { + if strings.Contains(s, EventFindingWritten) { + names = append(names, s) + } + } + streams, _ := e.Manifest["streams"].([]any) + for _, s := range streams { + m, _ := s.(map[string]any) + name, _ := m["name"].(string) + events, _ := m["events"].([]any) + for _, ev := range events { + if n, _ := ev.(string); strings.HasPrefix(n, EventFindingWritten) { + names = append(names, name) + break + } + } + } + sort.Strings(names) + return strings.Join(names, ",") +} + // CatalogHeld takes the newest catalog revision of one watched branch: the sources are the modules declaring the finding -// events in any branch, and the conditions of a module in no branch are cleared. +// events in any branch, and the conditions of a module in no branch are cleared. A module whose entry names another +// stream than the one the keeper followed for it has moved its findings: the keeper keeps its open conditions and asks +// once on the new stream (design 47 G §663, §690). func (k *Keeper) CatalogHeld(branch string, entries map[string]catseat.CatalogEntry) error { k.mu.Lock() k.init() k.halves[branch] = entries - return k.sweepCatalog() + if k.followed == nil { + k.followed = map[string]string{} + } + var moved []string + for module, e := range entries { + now := findingStreams(e) + was := k.followed[module] + if now != "" { + k.followed[module] = now + } + if was != "" && now != "" && now != was { + moved = append(moved, module) + } + } + sort.Strings(moved) + err := k.sweepCatalog() + for _, module := range moved { + if _, askErr := k.Ask(Module, module); askErr != nil && err == nil { + err = askErr + } + } + return err } // CatalogDropped forgets a branch no registration watches any more (design 47 G §532): the modules only it listed leave @@ -118,8 +165,11 @@ func (k *Keeper) isSource(observer string) bool { return true } // A source the keeper kept in `findings-views` is heard before its first view of the catalog half (design 47 G §388). + // Once a revision lists the part and no longer declares its finding stream, a kept source of the old stream is dropped, + // its open conditions kept (§690); a part no revision lists is kept until the first view takes it away. + _, listed := k.entry(observer) for _, v := range k.views { - if v.observer == observer && v.kept { + if v.observer == observer && v.kept && (!listed || k.declares(observer)) { return true } } @@ -135,6 +185,16 @@ func (k *Keeper) isSource(observer string) bool { return false } +// declares says whether a held revision lists the observer with its finding events. Called with mu held. +func (k *Keeper) declares(observer string) bool { + for _, half := range k.halves { + if e, ok := half[observer]; ok && declaresFindings(e) { + return true + } + } + return false +} + // catalogHolder is Keeper.Holder: an observer whose catalog entry claims only seats held once for the mesh is held on the // node the assignments name as the holder of one of them (design 47 G §384); unknown, it is taken from every node. func (k *Keeper) catalogHolder(observer string) (string, bool) { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 154773e4d..4b600e93c 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -151,6 +151,8 @@ type Keeper struct { stopped map[string]bool // created are the modules whose `finding-events` entry the bus module made (bound.go). created map[string]bool + // followed are the finding streams the catalog half declared for each module, by module (catalog.go). + followed map[string]string } func (k *Keeper) init() { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 3d47e4f62..0578a1dd9 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -635,6 +635,47 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { } return r.k.FindingWritten("newpart", "ace", mustJSON(t, f("newpart", "w"))) }, []string{"machine.d.silent", "machine.g.silent", "machine.w.silent"}, ""}, + // Design 47 G §663, §690: a part whose findings move keeps its condition, is asked once on the new stream, and its + // old source is dropped once its entry declares no finding stream. + {"a part that moves its finding stream is asked once and keeps its condition", func() error { + entry := func(stream string) map[string]catseat.CatalogEntry { + e := catseat.CatalogEntry{Module: "movedpart", Manifest: map[string]any{}} + if stream != "" { + e.Manifest["streams"] = []any{map[string]any{"name": stream, "events": []any{"finding-written.*"}}} + } + return map[string]catseat.CatalogEntry{"movedpart": e} + } + if err := r.k.CatalogHeld("moves", entry("old-findings")); err != nil { + return err + } + if err := r.k.FindingWritten("movedpart", "ace", mustJSON(t, f("movedpart", "m"))); err != nil { + return err + } + before := len(asked) + if err := r.k.CatalogHeld("moves", entry("old-findings")); err != nil { + return err + } + if len(asked) != before { + t.Errorf("the same entry asked %v", asked[before:]) + } + if err := r.k.CatalogHeld("moves", entry("new-findings")); err != nil { + return err + } + if len(asked) != before+1 || asked[before]["module"] != "movedpart" { + t.Errorf("asked %v, want movedpart once on the new stream", asked[before:]) + } + // Its entry declares no finding stream now (the findings moved to a template): the old source is dropped. + if err := r.k.CatalogHeld("moves", entry("")); err != nil { + return err + } + if err := r.k.FindingWritten("movedpart", "ace", mustJSON(t, f("movedpart", "late"))); err != nil { + return err + } + if _, ok := r.open("machine.late.silent"); ok { + t.Errorf("the dropped source raised a condition") + } + return nil + }, []string{"machine.d.silent", "machine.g.silent", "machine.m.silent"}, ""}, // newpart, listed by no revision, leaves with the first view } for _, s := range steps { if err := s.do(); err != nil { -- 2.54.0 From fa0f851f81df9fdc93a476045f4dc36ffcfc4199 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:16:18 +0100 Subject: [PATCH 079/243] mesh-build-go, mesh-build-image and mesh-build-node read a step key's node as SD4 has it: ParseStepKey answers asker, node and id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §484, §649 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-build-go/cmd/mesh-build-go/read.go | 2 +- modules/mesh-build-image/cmd/mesh-build-image/read.go | 2 +- modules/mesh-build-node/cmd/mesh-build-node/read.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/read.go b/modules/mesh-build-go/cmd/mesh-build-go/read.go index 2682604f7..1938479bb 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/read.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/read.go @@ -150,7 +150,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error) own := keys[:0:0] for _, k := range keys { if strings.HasPrefix(k, m.Node+".") { - if node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { + if _, node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { own = append(own, k) } } diff --git a/modules/mesh-build-image/cmd/mesh-build-image/read.go b/modules/mesh-build-image/cmd/mesh-build-image/read.go index d0ef53194..bea8a7ac5 100644 --- a/modules/mesh-build-image/cmd/mesh-build-image/read.go +++ b/modules/mesh-build-image/cmd/mesh-build-image/read.go @@ -149,7 +149,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error) own := keys[:0:0] for _, k := range keys { if strings.HasPrefix(k, m.Node+".") { - if node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { + if _, node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { own = append(own, k) } } diff --git a/modules/mesh-build-node/cmd/mesh-build-node/read.go b/modules/mesh-build-node/cmd/mesh-build-node/read.go index d28fe95e9..102b28082 100644 --- a/modules/mesh-build-node/cmd/mesh-build-node/read.go +++ b/modules/mesh-build-node/cmd/mesh-build-node/read.go @@ -150,7 +150,7 @@ func (m *Module) RecentSteps(limit int) (lines []StepLine, cut bool, err error) own := keys[:0:0] for _, k := range keys { if strings.HasPrefix(k, m.Node+".") { - if node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { + if _, node, _, err := sdkstep.ParseStepKey(k); err == nil && node == m.Node { own = append(own, k) } } -- 2.54.0 From 9c1cb63c3b4d5b2735e3e0a8201c3e863dd99b1f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:18:40 +0100 Subject: [PATCH 080/243] mesh-provisioner uses the catalog key's codec meshcatalogkey, as meshcatalogclient's wrappers go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §670 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-provisioner/cmd/mesh-provisioner/composed_test.go | 3 ++- modules/mesh-provisioner/cmd/mesh-provisioner/holder.go | 3 ++- modules/mesh-provisioner/go.mod | 1 + 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/composed_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/composed_test.go index e1958d7a1..f6de5f2d1 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/composed_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/composed_test.go @@ -16,6 +16,7 @@ import ( engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" meshseat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" "os" "slices" @@ -274,7 +275,7 @@ func valueOf(e *Entry) string { // composedKey is a module's key in the node-engine's composed state, from the fingerprint the test names it by. func composedKey(node, module, fp string) (string, error) { - part, err := catalogclient.KeyPart(fp) + part, err := meshcatalogkey.KeyPart(fp) if err != nil { return "", err } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go index bd7519586..3ab411822 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/holder.go @@ -35,6 +35,7 @@ import ( engineclient "git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient" meshseat "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey" catalog "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" builds "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds" "slices" @@ -1624,7 +1625,7 @@ type publishedPlan struct { // composedAt is a module's key in its node's compositions, or "" for a fingerprint no key can name. func composedAt(node, module, fp string) string { - part, err := catalogclient.KeyPart(fp) + part, err := meshcatalogkey.KeyPart(fp) if err != nil { return "" } diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index d4c7efcf3..00b8dd54e 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -12,6 +12,7 @@ require ( git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient v0.0.0-00010101000000-000000000000 + git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogkey v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/seats/meshwants v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/builds v0.0.0-00010101000000-000000000000 -- 2.54.0 From b90d3a4556085b134c030838d0c2999f3f5ce601 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:26:20 +0100 Subject: [PATCH 081/243] The delivery gate names the ledger's fill answer as the sdk has it: buscalls.Fill MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sdk (02bea66) answers the ledger's fill call as one buscalls.Fill; the gate's LedgerFilled is a name that is gone. Builds: design 47 G §462 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/record_test.go | 10 +++++----- .../cmd/mesh-delivery-gate/recordfill.go | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 9d72a2777..a5d091560 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -31,12 +31,12 @@ type ledgerFake struct { history []string failOn string // fill is what Fill answers; nil, the ledger says no fill. - fill *buscalls.LedgerFilled + fill *buscalls.Fill } -func (f *ledgerFake) Fill() (buscalls.LedgerFilled, error) { +func (f *ledgerFake) Fill() (buscalls.Fill, error) { if f.fill == nil { - return buscalls.LedgerFilled{}, errors.New("the runtime answers no mesh/ledger.fill") + return buscalls.Fill{}, errors.New("the runtime answers no mesh/ledger.fill") } return *f.fill, nil } @@ -809,8 +809,8 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { // says room and hands the merge once the ledger is below 80 % again; the read tool says the fill. func TestTheRecordsLedgerSaysFillingFullAndRoom(t *testing.T) { const capBytes = 10 << 20 - filled := func(bytes uint64) *buscalls.LedgerFilled { - return &buscalls.LedgerFilled{History: buscalls.Fill{Count: 1, Bytes: bytes, MaxBytes: capBytes}, + filled := func(bytes uint64) *buscalls.Fill { + return &buscalls.Fill{History: buscalls.Fill{Count: 1, Bytes: bytes, MaxBytes: capBytes}, State: buscalls.Fill{Bytes: 1, MaxBytes: 64 << 20}} } heads := map[string]string{"novox/app@111111111111": strings.Repeat("1", 40)} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go index 33d856012..0154b4317 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go @@ -29,15 +29,15 @@ var ErrRecordsFull = errors.New("the delivery records' ledger is full") // fillReader is a ledger that says its fill against its cap. type fillReader interface { - Fill() (buscalls.LedgerFilled, error) + Fill() (buscalls.Fill, error) } -func (sdkLedger) Fill() (buscalls.LedgerFilled, error) { +func (sdkLedger) Fill() (buscalls.Fill, error) { out, err := ledgerCall(buscalls.LedgerFill, buscalls.LedgerRequest{Ledger: RecordsLedger}) if err != nil { - return buscalls.LedgerFilled{}, err + return buscalls.Fill{}, err } - var got buscalls.LedgerFilled + var got buscalls.Fill if err := json.Unmarshal(out, &got); err != nil { return got, fmt.Errorf("the ledger's fill is not readable: %w", err) } -- 2.54.0 From d786477a0d8d1508adb015186af761d85c946065 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:35:26 +0100 Subject: [PATCH 082/243] =?UTF-8?q?GATE7=20(source=20part,=20=C2=A7690):?= =?UTF-8?q?=20the=20gate's=20source=20answers=20a=20branch's=20source=20ke?= =?UTF-8?q?y,=20or=20the=20branches=20it=20keeps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gate's source value is the contract's SourceByBranchValue and its key SourceKey(branch); the verb takes an optional branch and answers branches alone when given none. The whole-module package does not compile on the join for GATE11 (settle) and GATE17 (ledger fill and remove), so the unit suite was run with those five sites stubbed in a scratch edit, never committed: green apart from the stubbed removal case. Builds: design 47 G §690, §614, §15 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bridge_test.go | 16 ++++- .../cmd/mesh-delivery-gate/bus_test.go | 4 +- .../cmd/mesh-delivery-gate/issue888_test.go | 10 ++- .../cmd/mesh-delivery-gate/main.go | 5 +- .../cmd/mesh-delivery-gate/merges.go | 70 ++++++++++++++----- .../cmd/mesh-delivery-gate/values.go | 2 +- 6 files changed, 80 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go index 13770632f..c31a424ac 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go @@ -178,17 +178,27 @@ func (s *fakeSource) Move(update func(prev *Source) (Source, bool)) error { return nil } -// Read is source.main as held, and its revision: the number of writes. -func (s *fakeSource) Read() (*Source, uint64, error) { +// Read is a branch's source key as held, and its revision: the number of writes. The fake holds the key of MergeBranch only. +func (s *fakeSource) Read(branch string) (*Source, uint64, error) { s.mu.Lock() defer s.mu.Unlock() - if s.main == nil { + if s.main == nil || branch != MergeBranch { return nil, 0, nil } held := *s.main return &held, uint64(len(s.moves)), nil } +// Branches are the branches whose source key the fake holds. +func (s *fakeSource) Branches() ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.main == nil { + return nil, nil + } + return []string{MergeBranch}, nil +} + func sortStrings(s []string) { sort.Strings(s) } func itoa(n int) string { return strconv.Itoa(n) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 13b2afa6b..29f225922 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -639,7 +639,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB // The forge's own word that the pull request merged follows the bridge's (its webhook): the delivery ends merged. b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged("novox/app", 7, head, strings.Repeat("b", 40)))) rt.until("source.main to hold the merge", eventWithin, func() bool { - e, err := b.kvs["source"].Get(ctx, mdc.SourceMainKey) + e, err := b.kvs["source"].Get(ctx, mdc.SourceKey(MergeBranch)) if err != nil { return false } @@ -864,7 +864,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe b.forgeSays(t, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: two.Repository, Number: two.Number, Head: two.Head.Head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"x": "t"}}) rt.until("source.main to hold both merges", eventWithin, func() bool { - e, err := b.kvs["source"].Get(ctx, mdc.SourceMainKey) + e, err := b.kvs["source"].Get(ctx, mdc.SourceKey(MergeBranch)) if err != nil { return false } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go index d28d6d1a1..86a09ffcc 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go @@ -1,6 +1,7 @@ package main import ( + "reflect" "testing" "time" @@ -41,11 +42,18 @@ func TestAMergeHeardOnlyFromTheForgeMovesSourceMain(t *testing.T) { t.Fatalf("source.main is %+v; wanted %s at %s and %s kept", w.source.main, i888App, i888Merge1, i888Other) } // The seat's verb source answers each repository's trunk branch and head, and the revision, from the same key. - if a := w.h.SourceRevision(); a.Error != "" || a.Revision != 1 || len(a.Repositories) != 2 || + if a := w.h.SourceRevision(MergeBranch); a.Error != "" || a.Revision != 1 || len(a.Repositories) != 2 || a.Repositories[0] != (seat.SourceHead{Repository: i888App, Trunk: MergeBranch, Head: i888Merge1}) || a.Repositories[1].Repository != i888Other || a.Repositories[1].Head != i888OtherH { t.Fatalf("source answers %+v", a) } + // §690: given no branch it answers the branches of the keys it keeps, and no repository; a branch it keeps no key of has none. + if a := w.h.SourceRevision(""); a.Error != "" || !reflect.DeepEqual(a.Branches, []string{MergeBranch}) || len(a.Repositories) != 0 || a.Revision != 0 { + t.Fatalf("source with no branch answers %+v, want the branches %v alone", a, []string{MergeBranch}) + } + if a := w.h.SourceRevision("beta"); a.Error != "" || a.Branch != "beta" || a.Revision != 0 || len(a.Repositories) != 0 { + t.Fatalf("source of a branch with no key answers %+v", a) + } n := len(w.source.moves) w.settleAll() w.h.PullMerged(i888MergedAt(i888App, 3, i888Head1, i888Merge1, w.now.Add(-time.Minute))) // heard again diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index fdb7beceb..6430fa230 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -345,8 +345,9 @@ func tools(h *Holder, l *listening) []stdio.Tool { Description: "The source revision the gate holds: every repository it names, its trunk branch and that trunk's head, the " + "group that moved it last and its revision. Read, never written: a node-engine reads the repositories here to call a " + "reverted build's catch-up tool once per repository, never the gate's state.", - Input: map[string]any{"type": "object", "properties": map[string]any{}}, - Run: func(map[string]any) (any, error) { return h.SourceRevision(), nil }}, + Input: map[string]any{"type": "object", "properties": map[string]any{ + "branch": str("the watched branch, escaped as a key name is; absent: the list of the branches the gate keeps a source revision of")}}, + Run: func(a map[string]any) (any, error) { return h.SourceRevision(strArg(a, "branch")), nil }}, {Name: seat + "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each of the commit's " + "statuses (mesh/check, mesh/delivery, …) with its state, description and when it " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go index e9701a0af..150b7f9e2 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/merges.go @@ -14,6 +14,7 @@ import ( seat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" ) // The gate merges through the forge bridge, never itself (novox/hq ADR 0437 §8, ADR 0441 §5, ADR 0450 §3). It writes into @@ -48,13 +49,15 @@ func (kvOld) Get(state, key string) ([]byte, uint64, error) { return e.Value, e.Revision, nil } -// SourceStore is the gate's bucket source, one key, moved by compare-and-set. +// SourceStore is the gate's bucket source, one key per watched branch (design 47 G §15), moved by compare-and-set. type SourceStore interface { - // Move reads source.main and writes what update makes of it, unless update says nothing is to be written; a write - // another writer won is read again and tried again. + // Move reads the source key of the gate's merge branch (MergeBranch) and writes what update makes of it, unless update + // says nothing is to be written; a write another writer won is read again and tried again. Move(update func(prev *Source) (Source, bool)) error - // Read is source.main as held and its revision on the bus; nil and 0 before it is first written. - Read() (*Source, uint64, error) + // Read is a branch's source key as held and its revision on the bus; nil and 0 before it is first written. + Read(branch string) (*Source, uint64, error) + // Branches are the branch names of the source keys the gate keeps (§614, §690). + Branches() ([]string, error) } // kvMerges is merges over the runtime. @@ -95,7 +98,7 @@ type kvSource struct{} func (kvSource) Move(update func(prev *Source) (Source, bool)) error { state := stdio.State("source") for range changeAttempts { - e, err := state.Get(mdc.SourceMainKey) + e, err := state.Get(mdc.SourceKey(MergeBranch)) if err != nil { return err } @@ -116,9 +119,9 @@ func (kvSource) Move(update func(prev *Source) (Source, bool)) error { return err } if e == nil { - _, err = state.Create(mdc.SourceMainKey, json.RawMessage(raw)) + _, err = state.Create(mdc.SourceKey(MergeBranch), json.RawMessage(raw)) } else { - _, err = state.Update(mdc.SourceMainKey, json.RawMessage(raw), e.Revision) + _, err = state.Update(mdc.SourceKey(MergeBranch), json.RawMessage(raw), e.Revision) } if err == nil { return nil @@ -130,33 +133,64 @@ func (kvSource) Move(update func(prev *Source) (Source, bool)) error { return fmt.Errorf("source.main was written by another writer first %d times: %w", changeAttempts, stdio.ErrChanged) } -func (kvSource) Read() (*Source, uint64, error) { - e, err := stdio.State("source").Get(mdc.SourceMainKey) +func (kvSource) Read(branch string) (*Source, uint64, error) { + e, err := stdio.State("source").Get(mdc.SourceKey(branch)) if err != nil || e == nil { return nil, 0, err } held, err := mdc.DecodeSource(e.Value) if err != nil { - return nil, 0, fmt.Errorf("source.main is not readable: %w", err) + return nil, 0, fmt.Errorf("source.%s is not readable: %w", branch, err) } return &held, e.Revision, nil } -// SourceRevision is the seat's verb source: every repository source.main names, its trunk branch and head, the group that -// moved it last and its revision, or why it cannot be read. source.main names each repository's main only (MergeBranch). -// Outside the lock. -func (h *Holder) SourceRevision() seat.SourceAnswer { +func (kvSource) Branches() ([]string, error) { + names, err := stdio.State("source").Keys() + if err != nil { + return nil, err + } + var out []string + for _, key := range names { + escaped, ok := seat.ParseSourceByBranchKey(key) + if !ok { + continue + } + branch, err := keys.UnescapeName(escaped) + if err != nil { + return nil, fmt.Errorf("the source key %q is not a branch's: %w", key, err) + } + out = append(out, branch) + } + sort.Strings(out) + return out, nil +} + +// SourceRevision is the seat's verb source (§690): given a branch, every repository that branch's source key names, its +// trunk branch and head, the group that moved it last and its revision, or why it cannot be read; given none, `branches`, +// the branch of each source key the gate keeps. A key names each repository's merge branch only (MergeBranch). Outside the lock. +func (h *Holder) SourceRevision(branch string) seat.SourceAnswer { if h.Source == nil { return seat.SourceAnswer{Error: "no source bucket"} } - held, revision, err := h.Source.Read() + if branch == "" { + branches, err := h.Source.Branches() + if err != nil { + return seat.SourceAnswer{Error: err.Error()} + } + if branches == nil { + branches = []string{} + } + return seat.SourceAnswer{Branches: branches} + } + held, revision, err := h.Source.Read(branch) if err != nil { return seat.SourceAnswer{Error: err.Error()} } if held == nil { - return seat.SourceAnswer{} + return seat.SourceAnswer{Branch: branch} } - out := seat.SourceAnswer{Group: held.Group, Revision: int64(revision)} + out := seat.SourceAnswer{Branch: branch, Group: held.Group, Revision: int64(revision)} for repository, head := range held.Heads { out.Repositories = append(out.Repositories, seat.SourceHead{Repository: repository, Trunk: MergeBranch, Head: head}) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go index 20b00411e..7b3b2247d 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/values.go @@ -22,7 +22,7 @@ const ( // are converted at the edge, so a stored value decodes unchanged (novox/hq ADR 0634 §2). type ( // Source is the state source's value. - Source = seat.SourceByMainValue + Source = seat.SourceByBranchValue // Owed is the state owed's value: what the forge bridge performs. Owed = seat.OwedByRepositoryNumberHeadActValue // Answer is the operator's answer as the gate acts on it. -- 2.54.0 From 846291069b6db042715913e9185e76ceacd6280d Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:44:05 +0100 Subject: [PATCH 083/243] GATE17, GATE11: the gate removes a record by the runtime's ledger removal and reads the plain fill; settle and its ask machinery go MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GATE17: record.go Remove asks buscalls.LedgerRemove, recordfill.go reads buscalls.Fill, the manifest names ledger-upkeep-calls. GATE11: the verb settle, its ask, EvSettled and the Settle field leave with the merge states. Builds: design 47 G §587, §431 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/approval.go | 10 +- .../cmd/mesh-delivery-gate/main.go | 24 -- .../cmd/mesh-delivery-gate/manifest_test.go | 19 +- .../cmd/mesh-delivery-gate/model.go | 4 - .../cmd/mesh-delivery-gate/record.go | 13 +- .../cmd/mesh-delivery-gate/record_test.go | 33 ++- .../cmd/mesh-delivery-gate/recordfill.go | 4 +- .../cmd/mesh-delivery-gate/settle.go | 277 ------------------ .../cmd/mesh-delivery-gate/settle_test.go | 127 -------- .../cmd/mesh-delivery-gate/table.go | 15 - .../cmd/mesh-delivery-gate/table_test.go | 12 +- .../cmd/mesh-delivery-gate/verbs.go | 3 - modules/mesh-delivery-gate/module.json | 5 +- 13 files changed, 48 insertions(+), 498 deletions(-) delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go index 32e42916f..9b4c5718f 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go @@ -313,7 +313,6 @@ func (h *Holder) headsHold(d *Delivery, c *ReadyRun) bool { // A Decline stands, and is not ended. An ask none of whose deliveries can still merge by it is taken away after // (endAsksOfFinished). Under the lock; the cancels run with the look's asks. func (h *Holder) endStaleAsks() { - h.endStaleSettles() ending := map[string]bool{} for _, d := range h.deliveries { r := d.Ready @@ -674,7 +673,7 @@ func (h *Holder) hearAsks() { return } h.mu.Lock() - keys := append(h.pendingAsks(), h.settleKeys()...) + keys := h.pendingAsks() h.mu.Unlock() for _, key := range keys { answer, err := h.Operator.Where(key) @@ -686,12 +685,7 @@ func (h *Holder) hearAsks() { continue } h.mu.Lock() - var said []func() - if h.isSettleKey(key) { - h.settled(key, *answer) // a person's settle ask (settle.go, novox/hq ADR 0556 clause 5) - } else { - said = h.answered(key, *answer) - } + said := h.answered(key, *answer) h.mu.Unlock() for _, s := range said { s() diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 6430fa230..5b73dbf71 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -397,30 +397,6 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return h.Close(strArg(a, "id"), strArg(a, "why")) }}, - {Name: seat + "settle", - Description: "Ask the operator to settle deliveries held because their merge reached the trunk without a passing check: " + - "one ask on the operator's channel for all of them, as an Approve is asked. The verb moves nothing; on the operator's " + - "answer each goes to merged, and whether it is on the machines is derived as for any merge (novox/hq ADR 0556). " + - "Refused, nothing asked, when any delivery named is not held or is held for another reason, naming why.", - Input: map[string]any{"type": "object", "additionalProperties": false, "required": []string{"ids", "why"}, - "properties": map[string]any{ - "ids": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, "minItems": 1, - "description": "the held deliveries' ids (owner/repository@commit)"}, - "why": str("why they may be settled, shown to the operator")}}, - Run: func(a map[string]any) (any, error) { - var ids []string - switch v := a["ids"].(type) { - case []any: - for _, x := range v { - if s, ok := x.(string); ok { - ids = append(ids, s) - } - } - case []string: - ids = v - } - return h.Settle(ids, strArg(a, "why"), aPerson) - }}, {Name: "delivery_merge_again", Description: "On the operator's word, ask again at once the merge of an approved delivery the forge refused, or of an approved " + "group held or half merged, at the heads the Approve named (novox/hq issue 1275): every member is read first and " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 141ad05c0..2a340989e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -11,7 +11,6 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" "git.novox.be/novox/mesh-sdk/go/buscalls" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/seatlist/seatlisttest" @@ -76,9 +75,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if len(m.Seats) != 0 { t.Fatal("the manifest declares a seat; mesh-delivery is the mesh's own") } - // The eight verbs the seat keeps (ADR 0539 §17): times, what-if, release, stop and retire-history went with the plans; and - // settle, a person's ask to settle merges held without a passing check (ADR 0556 clause 5). - if want := []string{"deliveries", "show", "groups", "source", "checks", "table", "stalled", "recheck", "close", "settle"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { + // The verbs the seat keeps: times, what-if, release, stop and retire-history went with the plans (ADR 0539 §17), and + // settle with the merge states (design 47 G §431). + if want := []string{"deliveries", "show", "groups", "source", "checks", "table", "stalled", "recheck", "close"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want) } // The forge messages' layout is the bridge's published package forgemsg, taken by its series (ADR 0634 §3, design 32). @@ -138,9 +137,9 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { t.Fatalf("the manifest still names check-gates-merge (%v)", err) } // W11.20: the gate keeps its records in a ledger and reads its fill before it opens one, so it is held off a machine whose - // tool runner answers neither (the fill a capability of its own, the lead's ruling). - if !reflect.DeepEqual(m.Capabilities, []string{buscalls.LedgerCalls, buscalls.LedgerFillCalls}) { - t.Fatalf("capabilities %v, want %s and %s", m.Capabilities, buscalls.LedgerCalls, buscalls.LedgerFillCalls) + // tool runner answers neither (the fill and the removal one capability, ledger-upkeep-calls, design 47 G §587). + if !reflect.DeepEqual(m.Capabilities, []string{buscalls.LedgerCalls, buscalls.LedgerUpkeepCalls}) { + t.Fatalf("capabilities %v, want %s and %s", m.Capabilities, buscalls.LedgerCalls, buscalls.LedgerUpkeepCalls) } // What it reads: the checker's verdicts alone (issue 1302; they go with the checker). The teller's answers and the delivery // view's reach, which `deliveries` and `show` derive whether a merge is on the machines from (ADR 0556 clause 4), are no @@ -286,12 +285,12 @@ func TestTheGateHoldsTheSeatAsTheSDKsTableOfRelease6HasIt(t *testing.T) { served[verb] = true } } - for removed := range meshdeliveryclient.RemovedVerbs() { + for _, removed := range []string{"settle"} { // design 47 G §431 if served[removed] { - t.Errorf("the gate serves %s, which release 6 removed: the SDK answers it by name", removed) + t.Errorf("the gate serves %s, which G removed with the merge states", removed) } if slices.Contains(manifest.Claims[0].Serves, removed) { - t.Errorf("the manifest serves %s, which release 6 removed", removed) + t.Errorf("the manifest serves %s, which G removed with the merge states", removed) } } for _, verb := range []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"} { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go index 340bd7dac..6f275fb34 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/model.go @@ -106,10 +106,6 @@ type Delivery struct { // trunk's head, before the operator is asked (novox/hq ADR 0421 §3, issue 1022). Cleared when it is asked. FreshWanted bool `json:"fresh_wanted,omitempty"` - // Settle is the settle ask a person had written for it while it is held for a merge without a passing check (settle.go, - // novox/hq ADR 0556 clause 5): its key in the gate's asks, why, who asked and when, and the operator's answer once read. - Settle *SettleAsk `json:"settle,omitempty"` - Transitions []Transition `json:"transitions,omitempty"` // Owed are the forge's and the bus's effects of its transitions not yet done: said only after the // delivery is kept, retried until done. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index f99abaa03..1d5123b51 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -275,7 +275,7 @@ type RecordLedger interface { Apply(instance, operation string, body any) (uint64, error) // Get is the instance's state and the sequence of its last operation; a nil state when it has none. Get(instance string) (json.RawMessage, uint64, error) - // Remove removes a record with its operations and its snapshot (OpRemove); an error when any of them stays. + // Remove removes a record with its operations and its snapshot through the tool runner's ledger removal. Remove(instance string) error } @@ -312,15 +312,16 @@ func applyCall(instance, operation string, body any) (buscalls.Applied, error) { return got, nil } -// Remove applies OpRemove and reads the ledger's answer: a removal that left the record's operations, its snapshot or its -// state is an error, so the retention says it rather than leaving a record's history to grow unseen. +// Remove asks the tool runner to remove the record's operations, its snapshot and its keys (buscalls.LedgerRemove, design +// 47 G §587); an instance the ledger does not hold answers none removed, so a removal said twice changes nothing. func (sdkLedger) Remove(instance string) error { - got, err := applyCall(instance, OpRemove, map[string]any{}) + out, err := ledgerCall(buscalls.LedgerRemove, buscalls.LedgerRemoveRequest{Ledger: RecordsLedger, Instance: instance}) if err != nil { return err } - if !got.Removed { - return fmt.Errorf("the ledger removed %s but kept its operations or its snapshot (removed false at sequence %d)", instance, got.Sequence) + var got buscalls.LedgerRemoved + if err := json.Unmarshal(out, &got); err != nil { + return fmt.Errorf("the ledger's answer to the removal of %s is not readable: %w", instance, err) } return nil } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index a5d091560..d1a7f09e4 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -787,19 +787,33 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { t.Fatalf("a second repair did something: %+v", lines) } }) - t.Run("a removal that keeps the record's operations is an error", func(t *testing.T) { + t.Run("a removal asks the runtime's ledger removal of the instance, and one said twice changes nothing", func(t *testing.T) { was := ledgerCall t.Cleanup(func() { ledgerCall = was }) - removed := false + var asked []buscalls.LedgerRemoveRequest ledgerCall = func(method string, body any) (json.RawMessage, error) { - return json.Marshal(buscalls.Applied{Taken: true, Sequence: 4, Removed: removed}) + in, ok := body.(buscalls.LedgerRemoveRequest) + if method != buscalls.LedgerRemove || !ok { + t.Fatalf("the removal called %s with %T", method, body) + } + asked = append(asked, in) + if len(asked) == 1 { + return json.Marshal(buscalls.LedgerRemoved{Instance: in.Instance, Operations: 3, Bytes: 900}) + } + return json.Marshal(buscalls.LedgerRemoved{Instance: in.Instance}) } - if err := (sdkLedger{}).Remove("record/47_G/0011223344556677"); err == nil || !strings.Contains(err.Error(), "kept its operations") { - t.Fatalf("a removal that kept its operations: %v", err) + const inst = "record/47_G/0011223344556677" + for range 2 { + if err := (sdkLedger{}).Remove(inst); err != nil { + t.Fatal(err) + } } - removed = true - if err := (sdkLedger{}).Remove("record/47_G/0011223344556677"); err != nil { - t.Fatal(err) + if len(asked) != 2 || asked[0].Ledger != RecordsLedger || asked[0].Instance != inst { + t.Fatalf("asked %+v", asked) + } + ledgerCall = func(string, any) (json.RawMessage, error) { return nil, errors.New("the runtime is away") } + if err := (sdkLedger{}).Remove(inst); err == nil { + t.Fatal("a removal the runtime did not answer was taken as done") } }) } @@ -810,8 +824,7 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { func TestTheRecordsLedgerSaysFillingFullAndRoom(t *testing.T) { const capBytes = 10 << 20 filled := func(bytes uint64) *buscalls.Fill { - return &buscalls.Fill{History: buscalls.Fill{Count: 1, Bytes: bytes, MaxBytes: capBytes}, - State: buscalls.Fill{Bytes: 1, MaxBytes: 64 << 20}} + return &buscalls.Fill{Count: 1, Bytes: bytes, MaxBytes: capBytes} } heads := map[string]string{"novox/app@111111111111": strings.Repeat("1", 40)} at := time.Date(2026, 11, 9, 20, 0, 0, 0, time.UTC) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go index 0154b4317..22565c04e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recordfill.go @@ -44,7 +44,7 @@ func (sdkLedger) Fill() (buscalls.Fill, error) { return got, nil } -// Fill is the records' ledger's store nearest its cap. +// Fill is the records' ledger's history against its cap (design 47 G §587). func (rs *Records) Fill() (buscalls.Fill, error) { fr, ok := rs.Ledger.(fillReader) if !ok { @@ -54,7 +54,7 @@ func (rs *Records) Fill() (buscalls.Fill, error) { if err != nil { return buscalls.Fill{}, fmt.Errorf("the records' ledger cannot say its fill: %w", err) } - return f.Fullest(), nil + return f, nil } // pastFilling is a fill at or past FillingPercent of its cap. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go deleted file mode 100644 index b43aa67c5..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle.go +++ /dev/null @@ -1,277 +0,0 @@ -package main - -import ( - "fmt" - "sort" - "strings" - "time" - - occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" - oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" - mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" -) - -// A person settles a delivery held because its merge reached the trunk without a passing check (novox/hq ADR 0556 clause 5). -// The verb `settle` moves nothing: it writes one ask, for every delivery named, into the gate's asks, which the teller puts to -// the operator as it puts an Approve, its options binding mesh-delivery.settle (mdc.AskVerb). On the answer settle each -// delivery takes the table's asked row held → merged with the operator's words; on leave they stay held. Whether a settled -// merge is on the machines is then derived as for any merge (reached.go). A hold of another kind, or a delivery not held, is -// refused, naming why. - -// The settle ask's two answers, both at the level approve. -const ( - OptionSettle = "settle" - OptionLeave = "leave" -) - -// SettleAsk is the settle ask a delivery is covered by. -type SettleAsk struct { - Key string `json:"key"` - Asking int `json:"asking,omitempty"` - Why string `json:"why"` - By string `json:"by,omitempty"` - At time.Time `json:"at"` - Answered *time.Time `json:"answered,omitempty"` - Answer string `json:"answer,omitempty"` - Words string `json:"words,omitempty"` -} - -// Settle is the `settle` verb: one ask of the operator for every delivery named, each held for a merge without a passing -// check. Nothing moves until the operator answers. -func (h *Holder) Settle(ids []string, why, by string) (map[string]any, error) { - why = strings.TrimSpace(why) - if why == "" { - return nil, fmt.Errorf("settle says why: it is shown to the operator") - } - if len(ids) == 0 { - return nil, fmt.Errorf("settle names at least one held delivery (ids)") - } - if h.Operator == nil { - return nil, fmt.Errorf("this owner has no way to ask the operator") - } - h.mu.Lock() - var covered []*Delivery - var refused []string - seen := map[string]bool{} - for _, id := range ids { - id = strings.TrimSpace(id) - if seen[id] { - continue - } - seen[id] = true - d := h.deliveries[id] - switch { - case d == nil: - refused = append(refused, id+": no such delivery") - case d.State != Held: - refused = append(refused, fmt.Sprintf("%s: it is %s, not held", id, d.State)) - case !heldUnchecked(d): - refused = append(refused, fmt.Sprintf("%s: it is held for another reason, which settle does not take (%s)", id, d.HeldWhy)) - default: - covered = append(covered, d) - } - } - if len(refused) > 0 { - h.mu.Unlock() - sort.Strings(refused) - return nil, fmt.Errorf("settle refused, nothing asked: %s", strings.Join(refused, "; ")) - } - sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) - var names []string - var heads []mdc.Head - for _, d := range covered { - names = append(names, d.ID) - heads = append(heads, mdc.Head{Repository: d.Repository, Number: d.Number, Head: d.Commit}) - } - group := mdc.SettleGroup(names) - key, err := mdc.MergeAskKey(group, heads) - if err != nil { - h.mu.Unlock() - return nil, fmt.Errorf("the settle ask has no key: %w", err) - } - now := h.Now() - // The same deliveries asked again (the ask lapsed, or was left unanswered) are a new asking of the same key, under its own - // router id: the router drops a second ask under an id it kept. - asking := 1 // the first asking (Ask.Asking: 0 or 1 is the first, n the n-th) - for _, d := range covered { - if d.Settle != nil && d.Settle.Key == key && max(d.Settle.Asking, 1) >= asking { - asking = max(d.Settle.Asking, 1) + 1 - } - } - ask, err := gateAsk(group, heads, asking, composeSettleAsk(mdc.AskingID(key, asking), covered, why, now)) - if err != nil { - h.mu.Unlock() - return nil, fmt.Errorf("the settle ask is not one the operator can be asked: %w", err) - } - earlier := map[string]bool{} - for _, d := range covered { - if d.Settle != nil && d.Settle.Answered == nil && d.Settle.Key != key { - earlier[d.Settle.Key] = true - } - d.Settle = &SettleAsk{Key: key, Asking: asking, Why: why, By: by, At: now} - h.keep(d) - } - // An earlier settle ask of another set that no delivery waits for any more is taken away (issue 774): its question is - // gone, and left standing it would hold the teller's room at the router until it lapsed. - for _, d := range h.deliveries { - if d.Settle != nil { - delete(earlier, d.Settle.Key) - } - } - gone := make([]string, 0, len(earlier)) - for k := range earlier { - gone = append(gone, k) - } - sort.Strings(gone) - h.mu.Unlock() - for _, k := range gone { - h.Logf("[mesh-delivery] the settle ask %s is taken away: settle asked again for another set, and no delivery waits for it", k) - h.cancelSettle(k)() - } - if err := h.Operator.Ask(key, ask); err != nil { - return nil, fmt.Errorf("the settle ask %s could not be written (call settle again): %w", key, err) - } - h.Logf("[mesh-delivery] asked the operator to settle %s: %s", strings.Join(names, ", "), why) - return map[string]any{"asked": key, "deliveries": names, - "said": fmt.Sprintf("The operator is asked once to settle %d deliveries; nothing moves until the answer.", len(covered))}, nil -} - -// settleAct is what a settle ask's answer binds: its verb (mesh-delivery.settle), this ask, the answer and the heads. -func settleAct(group, ask, option string, heads map[string]string) occ.Act { - return occ.Act{"verb": mdc.AskVerb(group), "ask": ask, "option": option, "heads": headsString(heads)} -} - -// composeSettleAsk is the operator's ask to settle these held deliveries. -func composeSettleAsk(id string, ds []*Delivery, why string, now time.Time) oc.Ask { - var a oc.Ask - a.ID, a.Who, a.Expires = id, occ.Operator, occ.FormatTime(now.Add(AskLasts)) - a.OnExpiry = "they stay held; settle asks again" - a.About = "delivery.settle" - var pulls, lines []string - heads := map[string]string{} - for _, d := range ds { - pulls = append(pulls, pullSaid(d)) - heads[d.ID] = d.Commit - lines = append(lines, phone(fmt.Sprintf("%s %q merged at %s without a passing check", pullSaid(d), d.Title, - shortOf(d.MergedAs, 12)), 400)) - } - a.Headline = phone(fmt.Sprintf("Settle %d merges made without a passing check", len(ds)), occ.HeadlineLength) - a.Explanation = strings.Join([]string{ - phone(fmt.Sprintf("%s reached main without a passing check and are held. Settle ends the hold: each reads merged, and on the machines once every machine runs it. Leave keeps them held.", - strings.Join(pulls, ", ")), occ.SummaryLength), - phone("Why: "+why, occ.PointLength), - }, "\n") - a.Details = strings.Join(lines, "\n") - group := mdc.SettleGroup(func() []string { - var ids []string - for _, d := range ds { - ids = append(ids, d.ID) - } - return ids - }()) - for _, o := range []struct{ id, label, does string }{ - {OptionSettle, "Settle", "each delivery named ends its hold and reads merged"}, - {OptionLeave, "Leave", "they stay held"}, - } { - binds, _ := occ.ActDigest(settleAct(group, id, o.id, heads)) - a.Options = append(a.Options, oc.Option{ID: o.id, Label: o.label, Does: o.does, Level: occ.Approve, Binds: binds}) - } - return a -} - -// isSettleKey is whether an ask key is a settle ask's; under the lock. -func (h *Holder) isSettleKey(key string) bool { - _, group, _, err := mdc.ParseAskKey(key) - return err == nil && strings.HasPrefix(group, mdc.SettleGroupPrefix) -} - -// settleKeys are the settle asks waiting for an answer; under the lock. -func (h *Holder) settleKeys() []string { - seen := map[string]bool{} - var out []string - for _, d := range h.deliveries { - if s := d.Settle; s != nil && s.Answered == nil && d.State == Held && !seen[s.Key] { - seen[s.Key] = true - out = append(out, s.Key) - } - } - sort.Strings(out) - return out -} - -// settled applies the teller's answer to a settle ask; under the lock. The teller held the warrant (the act it bound is -// mesh-delivery.settle); an answer to an earlier asking, or one choosing what the ask did not offer, moves nothing. -func (h *Holder) settled(key string, ans Answer) { - var covered []*Delivery - for _, d := range h.deliveries { - if d.Settle != nil && d.Settle.Key == key && d.Settle.Answered == nil { - covered = append(covered, d) - } - } - if len(covered) == 0 { - h.Logf("[mesh-delivery] the answer to the settle ask %s covers no delivery that waits for it: nothing settled", key) - return - } - sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) - if asked, at, ok := strings.Cut(ans.Warrant, "@"); ok && at != "" && asked != mdc.AskingID(key, covered[0].Settle.Asking) { - h.Logf("[mesh-delivery] the answer to the settle ask %s was given to another asking (%s): not taken", key, asked) - return - } - if ans.Option != OptionSettle && ans.Option != OptionLeave { - h.refuse(fmt.Errorf("the answer to the settle ask %s chose %s, which it did not offer: nothing settled", key, ans.Option)) - return - } - now := h.Now() - for _, d := range covered { - d.Settle.Answered, d.Settle.Answer, d.Settle.Words = &now, ans.Option, ans.Words - if ans.Option == OptionLeave { - h.Logf("[mesh-delivery] %s: the operator left it held (%s)", d.ID, ans.Words) - h.keep(d) - continue - } - t, err := Apply(d, EvSettled, true, Facts{Now: now, By: "the operator", Why: "settled: " + ans.Words}) - if err != nil { - h.refuse(err) - h.keep(d) - continue - } - d.HeldWhy = "" - h.moved(d, []Transition{t}) - } - // Answered: the ask is taken away, as a merge ask is once acted on. - h.cancels = append(h.cancels, h.cancelSettle(key)) -} - -// cancelSettle takes a settle ask away from the gate's asks. -func (h *Holder) cancelSettle(key string) func() { - return func() { - if h.Operator == nil { - return - } - if err := h.Operator.Cancel(key); err != nil { - h.Logf("[mesh-delivery] the settle ask %s could not be taken away (taken away again at the next look): %v", key, err) - } - } -} - -// endStaleSettles takes away every settle ask nobody answered within its bound; its deliveries stay held and settle may ask -// again. Under the lock; the cancels run with the look's asks. -func (h *Holder) endStaleSettles() { - done := map[string]bool{} - for _, d := range h.deliveries { - s := d.Settle - if s == nil || s.Answered != nil || done[s.Key] || h.Now().Before(s.At.Add(AskLasts)) { - continue - } - done[s.Key] = true - h.Logf("[mesh-delivery] the settle ask %s lapsed with no answer: taken away, %s stays held", s.Key, d.ID) - h.cancels = append(h.cancels, h.cancelSettle(s.Key)) - } - for _, d := range h.deliveries { - if s := d.Settle; s != nil && done[s.Key] { - now := h.Now() - s.Answered, s.Answer = &now, "lapsed" - h.keep(d) - } - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go deleted file mode 100644 index ee06dcc21..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/settle_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package main - -import ( - "slices" - "strings" - "testing" - "time" - - occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" - mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" -) - -// 1280 (novox/hq ADR 0556 clause 5): settle writes one ask for several deliveries held for a merge without a passing check and -// moves none of them; it refuses, asking nothing, a delivery held for another reason and one not held, naming why; the -// operator's Leave keeps them held, and Settle, asked again under a new asking, moves each to merged with the operator's words. -func TestSettleAsksTheOperatorOnceAndMovesOnlyOnTheAnswer(t *testing.T) { - w := newWorld(t) - w.op.auto = false - merged := w.now.Add(-time.Hour) - put := func(id string, state State, why string) *Delivery { - repo, commit, _ := strings.Cut(id, "@") - d := &Delivery{ID: id, Repository: repo, Commit: commit, Number: len(id), Title: "a fix", State: state, Since: merged, - MergedAs: "m" + commit, MergedAt: merged, HeldWhy: why} - w.h.mu.Lock() - w.h.keep(d) - w.h.mu.Unlock() - return d - } - a := put("novox/web@aaaaaaaaaaaa", Held, HeldUnchecked) - b := put("novox/sdk@bbbbbbbbbbbb", Held, HeldUnchecked) - put("novox/web@cccccccccccc", Held, "a conflict on the trunk") - put("novox/web@dddddddddddd", Merged, "") - for ids, words := range map[string]string{ - "novox/web@aaaaaaaaaaaa novox/web@cccccccccccc": "held for another reason", - "novox/web@aaaaaaaaaaaa novox/web@dddddddddddd": "not held", - "novox/web@aaaaaaaaaaaa novox/web@eeeeeeeeeeee": "no such delivery", - } { - if _, err := w.h.Settle(strings.Fields(ids), "on every machine since 10-25", aPerson); err == nil || !strings.Contains(err.Error(), words) { - t.Errorf("settle %s: %v, want refused naming %q", ids, err, words) - } - } - if _, err := w.h.Settle([]string{a.ID}, " ", aPerson); err == nil { - t.Error("a settle with no why was asked") - } - if w.op.count() != 0 { - t.Fatalf("a refused settle asked the operator: %d asks", w.op.count()) - } - out, err := w.h.Settle([]string{b.ID, a.ID}, "on every machine since 10-25", aPerson) - if err != nil { - t.Fatal(err) - } - key, _ := out["asked"].(string) - if open := w.op.open(); len(open) != 1 || open[0] != key || w.op.count() != 1 { - t.Fatalf("asks open %v (asked %d), want the one key %s", open, w.op.count(), key) - } - ask := w.op.ask(key) - heads := map[string]string{a.ID: a.Commit, b.ID: b.Commit} - group := mdc.SettleGroup([]string{a.ID, b.ID}) - for _, o := range ask.Options { - if want, _ := occ.ActDigest(settleAct(group, ask.ID, o.ID, heads)); o.Binds != want { - t.Errorf("the option %s binds %s, not mesh-delivery.settle of these heads", o.ID, o.Binds) - } - } - w.h.hearAsks() - if a.State != Held || b.State != Held { - t.Fatalf("moved before an answer: %s, %s", a.State, b.State) - } - w.op.answer(key, OptionLeave) - w.h.hearAsks() - if a.State != Held || a.Settle == nil || a.Settle.Answer != OptionLeave { - t.Fatalf("after Leave: %s %+v", a.State, a.Settle) - } - // Asked again: a new asking of the same key, under its own id. - if _, err := w.h.Settle([]string{a.ID, b.ID}, "asked again", aPerson); err != nil { - t.Fatal(err) - } - if again := w.op.ask(key); again.ID == ask.ID || a.Settle.Asking != 2 { - t.Fatalf("asked again under %s (asking %d), want a new id", again.ID, a.Settle.Asking) - } - // An answer under a settle key no delivery waits for moves nothing. - other, _ := mdc.MergeAskKey(mdc.SettleGroup([]string{"novox/web@ffffffffffff"}), []mdc.Head{{Repository: "novox/web", Number: 1, Head: "ffffffffffff"}}) - w.h.mu.Lock() - w.h.settled(other, Answer{Option: OptionSettle, Warrant: "x@now", Words: "chose Settle"}) - w.h.mu.Unlock() - if a.State != Held || b.State != Held || !strings.Contains(w.logbuf.String(), "covers no delivery") { - t.Fatalf("an answer for other ids moved %s, %s", a.State, b.State) - } - w.op.answer(key, OptionSettle) - w.h.hearAsks() - w.h.mu.Lock() - cancels := len(w.h.cancels) - w.h.mu.Unlock() - if cancels == 0 { - t.Error("the answered settle ask is not taken away") - } - for _, d := range []*Delivery{a, b} { - last := d.Transitions[len(d.Transitions)-1] - if d.State != Merged || last.Event != EvSettled || last.By != "the operator" || !strings.Contains(last.Why, "chose Settle") { - t.Errorf("%s after Settle: %s, %+v", d.ID, d.State, last) - } - } - // A settle ask nobody answers within its bound is taken away; its delivery stays held. - c := put("novox/web@999999999999", Held, HeldUnchecked) - if _, err := w.h.Settle([]string{c.ID}, "once more", aPerson); err != nil { - t.Fatal(err) - } - w.later(AskLasts + time.Minute) - w.h.mu.Lock() - w.h.endStaleSettles() - w.h.mu.Unlock() - if c.State != Held || c.Settle.Answer != "lapsed" { - t.Errorf("a lapsed settle ask: %s %+v", c.State, c.Settle) - } - // Issue 774: a settle asked again for another set of deliveries is another key; the earlier ask, which no delivery waits - // for any more, is taken away at once, not left to hold the teller's room at the router until it lapses. - e, f := put("novox/web@eeeeeeeeeeee", Held, HeldUnchecked), put("novox/web@ffffffffffff", Held, HeldUnchecked) - first, err := w.h.Settle([]string{e.ID}, "one", aPerson) - if err != nil { - t.Fatal(err) - } - if _, err := w.h.Settle([]string{e.ID, f.ID}, "both", aPerson); err != nil { - t.Fatal(err) - } - if k := first["asked"].(string); !slices.Contains(w.op.cancelled, k) || slices.Contains(w.op.open(), k) { - t.Errorf("the earlier settle ask %s stands once no delivery waits for it: open %v, taken away %v", k, w.op.open(), w.op.cancelled) - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go index 771116f70..e79102a52 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go @@ -77,7 +77,6 @@ const ( EvMerged Event = "merged" // it reached the trunk EvMergedUnchecked Event = "merged-unchecked" // it reached the trunk without a passing check EvTrunkMoved Event = "trunk moved" // another merge moved its trunk, and its verdict names the trunk commit before (novox/hq ADR 0323) - EvSettled Event = "settled" // the operator settled a merge held without a passing check (novox/hq ADR 0556 clause 5) ) // HeldUnchecked is the why of a delivery held because its merge reached the trunk without a passing check: the one hold a @@ -186,20 +185,6 @@ var Table = []Row{ {From: []State{Proposed, Rejected}, Event: EvMergedUnchecked, To: Held, Guard: "merged without a passing check — a verdict known with it is taken first: only a person decides what follows", Holds: func(d *Delivery, f Facts) error { return unless(d.MergedAs != "", "not merged") }}, - // **A person settles a merge held without a passing check** (novox/hq ADR 0556 clause 5): only on the operator's answer to - // the settle ask, with its words; a hold of another kind is not taken here. Whether it is on the machines is then derived. - {From: []State{Held}, Event: EvSettled, To: Merged, Act: true, - Guard: "held because its merge reached the trunk without a passing check, and the operator answered settle to the ask " + - "`settle` wrote, with the words of that answer", - Holds: func(d *Delivery, f Facts) error { - switch { - case !heldUnchecked(d): - return errors.New("it is not held for a merge without a passing check: " + d.HeldWhy) - case f.By == "" || f.Why == "": - return errors.New("a settle names the operator's answer and its words") - } - return nil - }}, {From: []State{Rejected, Ready}, Event: EvRecheck, To: Proposed, Act: true, Guard: "a person asked, with why, or its group changed", Holds: func(d *Delivery, f Facts) error { return unless(f.Why != "", "a recheck says why") }}, diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go index 1469a95a5..a9a034cec 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go @@ -34,9 +34,6 @@ func aDeliveryFor(r Row) (*Delivery, Facts) { case EvTrunkMoved: d.Check = &Verdict{Gate: "pass", Head: "c0ffee000000", Trunk: "d28ca95bbbbb"} d.TrunkNow = "690966dccccccccc" - case EvSettled: - d.MergedAs, d.HeldWhy = "aaaa1111", HeldUnchecked - f.Why, f.By = "settled: the operator chose Settle", "the operator" } return d, f } @@ -106,7 +103,7 @@ func TestEveryRowOfTheTableIsTakenWhenItsGuardHoldsAndRefusedWhenNot(t *testing. func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { events := []Event{EvAnnounced, EvAccepted, EvRefused, EvRecheck, EvReask, EvLost, EvNewHead, EvClosed, EvMerged, - EvMergedUnchecked, EvTrunkMoved, EvSettled} + EvMergedUnchecked, EvTrunkMoved} for _, from := range append([]State{None}, AllStates...) { for _, ev := range events { for _, act := range []bool{false, true} { @@ -131,17 +128,14 @@ func TestEveryPairTheTableDoesNotHoldIsRefusedByName(t *testing.T) { // The rules the table is for, read off the table. func TestTheTableKeepsItsRules(t *testing.T) { for _, r := range Table { - // The one row out of a final state is a person's settle of a merge held without a passing check, taken on the - // operator's answer (novox/hq ADR 0556 clause 5): it is already on the trunk, and moves from held to merged. - settle := r.Event == EvSettled && r.Act && len(r.From) == 1 && r.From[0] == Held && r.To == Merged for _, from := range r.From { // Nothing else leaves a final state. - if from.Final() && !settle { + if from.Final() { t.Errorf("%s is final and the table leaves it by %s", from, r.Event) } } // Only a ready delivery is merged, and nothing but its merge, or a merge without a passing check, puts one on the trunk. - if r.To == Merged && !settle && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) { + if r.To == Merged && !(r.Event == EvMerged && len(r.From) == 1 && r.From[0] == Ready) { t.Errorf("%v —%s→ merged: only a ready delivery's merge merges", r.From, r.Event) } if r.To == Held && r.Event != EvMergedUnchecked { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 18378d182..7c0ea4fe6 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -351,8 +351,5 @@ func (h *Holder) Status(listening string) map[string]any { // read; the gate moves nothing after the merge. func HeldLine(d *Delivery) string { line := fmt.Sprintf("%s held since %s: %s", d.ID, d.Since.UTC().Format(time.RFC3339), d.HeldWhy) - if heldUnchecked(d) { - line += " (mesh-delivery.settle asks the operator to settle it, ADR 0556)" - } return line } diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 141daa397..514b72be9 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -15,8 +15,7 @@ "table", "stalled", "recheck", - "close", - "settle" + "close" ] } ], @@ -327,7 +326,7 @@ ], "capabilities": [ "ledger-calls", - "ledger-fill" + "ledger-upkeep-calls" ], "reads": [ "mesh-delivery-checker.verdicts" -- 2.54.0 From dd4449de6168b8d818df2f6c660f98ecb9fdd05e Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:46:49 +0100 Subject: [PATCH 084/243] G findings: the view-only modules write their reader's findings through mesh-sdk's stdio writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit nfs-server, minio object-store, claude-code, mesh-watcher, mesh-provisioner and mesh-build-go hand their views reader views.WriterFindings over views.NewStdioWriter and start it; the removed meshconditionsclient.FindingWriter goes. Builds: design 47 G §686, §687, §689, A-3, A-16, A-17 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/claude-code/cmd/claude-code/main.go | 24 +++++++++++++++++-- modules/claude-code/go.mod | 3 +-- .../cmd/mesh-build-go/compose.go | 22 +++++++++++++++-- .../mesh-build-go/cmd/mesh-build-go/main.go | 10 ++++++++ .../cmd/mesh-provisioner/main.go | 24 +++++++++++++++++-- .../cmd/mesh-provisioner/provisioner_test.go | 3 +-- modules/mesh-provisioner/go.mod | 3 +-- modules/mesh-watcher/cmd/mesh-watcher/main.go | 24 +++++++++++++++++-- modules/mesh-watcher/go.mod | 3 +-- modules/minio/cmd/object-store/main.go | 24 +++++++++++++++++-- modules/minio/go.mod | 3 +-- modules/nfs-server/cmd/nfs-server/main.go | 18 ++++++++++++-- modules/nfs-server/go.mod | 3 +-- 13 files changed, 140 insertions(+), 24 deletions(-) diff --git a/modules/claude-code/cmd/claude-code/main.go b/modules/claude-code/cmd/claude-code/main.go index 53a18a1a0..9ab8baa7b 100644 --- a/modules/claude-code/cmd/claude-code/main.go +++ b/modules/claude-code/cmd/claude-code/main.go @@ -22,7 +22,7 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + condseat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" stdio "git.novox.be/novox/mesh-sdk/go" @@ -230,12 +230,22 @@ const ( // once of the owner's read tool, then followed on the owner's views family. It reads none of their state. var meshViews *views.Reader +// newFindingsWriter is this module's finding writer (design 47 G §689, A-16): it states the findings of its views reader and +// observes none of its own. +func newFindingsWriter() (*views.Writer[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent], error) { + return views.NewStdioWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{ + Module: "claude-code", State: stdio.State("findings"), + Observe: func(context.Context) (views.Observed[condseat.Finding], error) { + return views.Observed[condseat.Finding]{}, nil + }}) +} + // meshChanged is told, without waiting, that a followed state changed: the instructions' table is read again. var meshChanged = make(chan struct{}, 1) // newMeshViews builds the reader of those three states; Start asks them. func newMeshViews() *views.Reader { - v := &views.Reader{Module: Module, Logf: func(f string, a ...any) { say(f, a...) }, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, + v := &views.Reader{Module: Module, Logf: func(f string, a ...any) { say(f, a...) }, Acts: handActs{}, Changed: func(string, string, json.RawMessage, bool) { select { @@ -488,6 +498,16 @@ func main() { } } } + findings, err := newFindingsWriter() + if err != nil { + say("%v", err) + os.Exit(1) + } + meshViews.Findings = views.WriterFindings{Writer: findings} + if err := findings.Start(context.Background()); err != nil { + say("%v", err) + os.Exit(1) + } meshViews.Start(context.Background()) go refreshMeshTools(p) servers := stateOf{stdio.State("servers")} diff --git a/modules/claude-code/go.mod b/modules/claude-code/go.mod index 7b9a5cfc8..9c9f3a901 100644 --- a/modules/claude-code/go.mod +++ b/modules/claude-code/go.mod @@ -7,7 +7,6 @@ require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 @@ -26,6 +25,6 @@ require ( ) require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect ) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose.go b/modules/mesh-build-go/cmd/mesh-build-go/compose.go index a28671458..bae34801d 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose.go @@ -25,7 +25,7 @@ import ( "time" "git.novox.be/novox/mesh-catalog/modules/mesh-build-go/buildstep" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/composer" @@ -83,7 +83,25 @@ func (m *Module) compose(ctx context.Context, fetch buildstep.StepFunc, a sdkste // askManifest asks the indexer's read tool for one manifest by its key: nil when it keeps none. A call the indexer does not // answer is a finding of this module's own, which the condition keeper raises (ADR 0615 §6) and the next answer clears. func askManifest(key string) (json.RawMessage, error) { - return askManifestVia(stdio.Ask, meshconditionsclient.FindingWriter{State: stdio.State("findings")}, time.Now, key) + return askManifestVia(stdio.Ask, manifestFindings, time.Now, key) +} + +// manifestFindings is where askManifest writes its finding: the module's writer once main has started it (design 47 G §689, +// A-16); outside a launch by the runtime, which has no node and no state, nothing is written. +var manifestFindings views.FindingWriter = noFindings{} + +type noFindings struct{} + +func (noFindings) Put(views.Finding) error { return nil } +func (noFindings) Delete(views.Finding) error { return nil } + +// newFindingsWriter is this module's finding writer: it states the finding of askManifest and observes none of its own. +func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { + return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ + Module: "mesh-build-go", State: stdio.State("findings"), + Observe: func(context.Context) (views.Observed[seat.Finding], error) { + return views.Observed[seat.Finding]{}, nil + }}) } func askManifestVia(ask func(string, any) (json.RawMessage, error), findings views.FindingWriter, now func() time.Time, key string) (json.RawMessage, error) { diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 317911181..46872509f 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -17,6 +17,7 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" + "git.novox.be/novox/mesh-sdk/go/views" ) // The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module @@ -71,6 +72,15 @@ func main() { // Outside a launch by the runtime — a build, a check — there is no node and no state to write. if m.Node != "" { go m.Start(ctx) + findings, err := newFindingsWriter() + if err == nil { + err = findings.Start(ctx) + } + if err != nil { + logf("the finding writer: %v", err) + os.Exit(1) + } + manifestFindings = views.WriterFindings{Writer: findings} } go startKeeper(ctx) // Every state it keeps, said on the bus before it serves (ADR 0495 §2; issue 1790). diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 68708baa2..5220c92d1 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -16,7 +16,7 @@ import ( "sync/atomic" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" stdio "git.novox.be/novox/mesh-sdk/go" @@ -185,7 +185,7 @@ const ( // newViews is the provisioner's pictures of the assigner's assignments and wants and the indexer's modules and manifests: // each asked once of its owner's read tool, then followed by the owner's events (ADR 0615). func newViews() *views.Reader { - v := &views.Reader{Module: Module, Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} + v := &views.Reader{Module: Module, Logf: logf, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} v.Follow(assignerModule, "assignments", AssignmentsTool) v.Follow(assignerModule, "wants", WantsTool) v.Follow(indexerModule, "modules", ModulesTool) @@ -215,9 +215,25 @@ func watchedOver(v *views.Reader) func(string, func(stdio.StateChange) error) er } } +// newFindingsWriter is this module's finding writer (design 47 G §689, A-16): it states the findings of its views reader and +// observes none of its own. +func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { + return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ + Module: "mesh-provisioner", State: stdio.State("findings"), + Observe: func(context.Context) (views.Observed[seat.Finding], error) { + return views.Observed[seat.Finding]{}, nil + }}) +} + func main() { s := runtimeStore{} v := newViews() + findings, err := newFindingsWriter() + if err != nil { + logf("%v", err) + os.Exit(1) + } + v.Findings = views.WriterFindings{Writer: findings} // One copy of what it reads, each state by its watch (ADR 0580 clause 2); the indexer's and the assigner's by their // pictures, a manifest by its digest from the picture too. ws := stdio.WatchedStatesOver(watchedOver(v), func(state, key string) ([]byte, bool, error) { @@ -230,6 +246,10 @@ func main() { } return e.Value, true, nil }) + if err := findings.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } v.Start(context.Background()) h := &Holder{Store: Watching(s, ws), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} h.Compose = FromCompositions(h.Store) // the node-engines' published compositions, from the copy (ADR 0593 §8) diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index 0b1837325..886324ebb 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -11,7 +11,6 @@ import ( "sync" "testing" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -322,7 +321,7 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { t.Errorf("the claim: %+v", m.Claims) } // needs stays declared this release so its first start can empty it (ADR 0570 clause 1). - if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState, RefusedState, meshconditionsclient.FindingsState}; !slices.Equal(m.State, want) { + if want := []string{Registry, Needs, Applied, Collect, ProvisionsState, PlansState, RefusedState, "findings"}; !slices.Equal(m.State, want) { t.Errorf("the state is %v, want %v", m.State, want) } // Every state and act has its tool (ADR 0570 Consequences): the verb reads the clients' states, the holder's tools diff --git a/modules/mesh-provisioner/go.mod b/modules/mesh-provisioner/go.mod index 00b8dd54e..f02f70bb8 100644 --- a/modules/mesh-provisioner/go.mod +++ b/modules/mesh-provisioner/go.mod @@ -6,7 +6,6 @@ require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel v0.0.0-00010101000000-000000000000 @@ -36,7 +35,7 @@ require ( require ( git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 // indirect - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry v0.0.0-00010101000000-000000000000 // indirect diff --git a/modules/mesh-watcher/cmd/mesh-watcher/main.go b/modules/mesh-watcher/cmd/mesh-watcher/main.go index ecded8338..b4152ba21 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/main.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/main.go @@ -17,8 +17,8 @@ import ( "sync" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" meshselfcheck "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck" stdio "git.novox.be/novox/mesh-sdk/go" @@ -76,8 +76,18 @@ func main() { node := os.Getenv("MESH_NODE") w := NewWatcher(tg, time.Now, logf, node) l := &listening{now: "not yet: starting"} - assignments := &views.Reader{Module: "mesh-watcher", Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} + assignments := &views.Reader{Module: "mesh-watcher", Logf: logf, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} + findings, err := newFindingsWriter() + if err != nil { + logf("%v", err) + os.Exit(1) + } + assignments.Findings = views.WriterFindings{Writer: findings} assignments.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) + if err := findings.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } go run(w, l, node, assignments) if err := stdio.Serve("", append(tools(w, l), assignments.Tools("watcher")...)); err != nil { logf("%v", err) @@ -85,6 +95,16 @@ func main() { } } +// newFindingsWriter is this module's finding writer (design 47 G §689, A-16): it states the findings of its views reader and +// observes none of its own. +func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { + return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ + Module: "mesh-watcher", State: stdio.State("findings"), + Observe: func(context.Context) (views.Observed[seat.Finding], error) { + return views.Observed[seat.Finding]{}, nil + }}) +} + // run keeps time, makes the round trips, reads where the controller is, and listens for heartbeats. // The clock runs whatever the bus does: it is what notices the bus is gone. func run(w *Watcher, l *listening, node string, assignments *views.Reader) { diff --git a/modules/mesh-watcher/go.mod b/modules/mesh-watcher/go.mod index 4453cc660..0970045fa 100644 --- a/modules/mesh-watcher/go.mod +++ b/modules/mesh-watcher/go.mod @@ -7,7 +7,6 @@ require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nats.go v1.54.0 // indirect @@ -20,6 +19,6 @@ require ( require git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck v0.0.0-00010101000000-000000000000 require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect ) diff --git a/modules/minio/cmd/object-store/main.go b/modules/minio/cmd/object-store/main.go index 575a383d9..6b9f7712b 100644 --- a/modules/minio/cmd/object-store/main.go +++ b/modules/minio/cmd/object-store/main.go @@ -15,7 +15,7 @@ import ( mesh "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" "os" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -30,7 +30,13 @@ const ( ) func main() { - seats := &views.Reader{Module: "minio", Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[object-store] "+f+"\n", a...) }} + seats := &views.Reader{Module: "minio", Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "[object-store] "+f+"\n", a...) }} + findings, err := newFindingsWriter() + if err != nil { + fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) + os.Exit(1) + } + seats.Findings = views.WriterFindings{Writer: findings} seats.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) s, err := newServer(os.Getenv(ConfigWord), os.Getenv(LinksWord), os.Getenv(ReceivesWord), func() (map[string][]byte, error) { return seats.Values(assignerModule + "." + mesh.StateSeatHolders) }) @@ -38,6 +44,10 @@ func main() { fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) os.Exit(1) } + if err := findings.Start(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) + os.Exit(1) + } seats.Start(context.Background()) if err := stdio.Serve("", append(tools(s), seats.Tools("object_store")...)); err != nil { fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) @@ -52,6 +62,16 @@ const assignerModule = "mesh-delivery-assigner" const SeatHoldersTool = assignerModule + ".assigner_read_seat_holders" +// newFindingsWriter is this module's finding writer (design 47 G §689, A-16): it states the findings of its views reader and +// observes none of its own. +func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { + return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ + Module: "minio", State: stdio.State("findings"), + Observe: func(context.Context) (views.Observed[seat.Finding], error) { + return views.Observed[seat.Finding]{}, nil + }}) +} + func tools(s *server) []stdio.Tool { return []stdio.Tool{{ Name: "object-store.link", diff --git a/modules/minio/go.mod b/modules/minio/go.mod index d398b5ea2..390a722a6 100644 --- a/modules/minio/go.mod +++ b/modules/minio/go.mod @@ -7,7 +7,6 @@ require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nats.go v1.54.0 // indirect @@ -18,6 +17,6 @@ require ( ) require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect ) diff --git a/modules/nfs-server/cmd/nfs-server/main.go b/modules/nfs-server/cmd/nfs-server/main.go index 153d5c186..17feebc89 100644 --- a/modules/nfs-server/cmd/nfs-server/main.go +++ b/modules/nfs-server/cmd/nfs-server/main.go @@ -22,8 +22,8 @@ import ( "time" "git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/views" @@ -51,8 +51,22 @@ func main() { return } m := Here() + w, err := views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ + Module: "nfs-server", State: stdio.State("findings"), + Observe: func(context.Context) (views.Observed[seat.Finding], error) { + return views.Observed[seat.Finding]{}, nil + }}) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + seats.Findings = views.WriterFindings{Writer: w} seats.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) seats.Start(context.Background()) + if err := w.Start(context.Background()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } go ShareExports(m) if err := stdio.Serve("", append(Tools(m), seats.Tools("nfs_server")...)); err != nil { fmt.Fprintln(os.Stderr, err) @@ -132,7 +146,7 @@ const SeatHoldersTool = assignerModule + ".assigner_read_seat_holders" // seats is the picture of the assignments seat's state of who holds which seat: asked of the assigner once and followed by // its events, never read from its state (ADR 0615). -var seats = &views.Reader{Module: "nfs-server", Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "nfs-server: "+f+"\n", a...) }} +var seats = &views.Reader{Module: "nfs-server", Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}, Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, "nfs-server: "+f+"\n", a...) }} // seatHolderKeys is the assignments seat's state of who holds which seat, every key. func seatHolderKeys() (map[string][]byte, error) { diff --git a/modules/nfs-server/go.mod b/modules/nfs-server/go.mod index 510814cc0..5c20efeb4 100644 --- a/modules/nfs-server/go.mod +++ b/modules/nfs-server/go.mod @@ -7,7 +7,6 @@ require git.novox.be/novox/mesh-sdk/go v0.0.0-00010101000000-000000000000 require ( git.novox.be/novox/mesh-contracts/go/clients/meshmoduleassignerclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 github.com/klauspost/compress v1.20.1 // indirect github.com/nats-io/nats.go v1.54.0 // indirect @@ -22,6 +21,6 @@ require git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings v0.0.0-00 replace git.novox.be/novox/mesh-catalog/modules/disk-load/diskreadings => ../disk-load/diskreadings require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 // indirect ) -- 2.54.0 From 9677f200f31bf03743b20181ddf6598573ff8c73 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:47:00 +0100 Subject: [PATCH 085/243] mesh-self-check writes its findings through mesh-sdk's finding writer and the generated finding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §686, §687, §689, A-3, A-13, A-14, A-16 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-self-check/findings.go | 52 +++++++ .../cmd/mesh-self-check/main.go | 20 ++- .../cmd/mesh-self-check/probes.go | 46 +++---- .../cmd/mesh-self-check/probes_data.go | 14 +- .../cmd/mesh-self-check/probes_fill.go | 22 ++- .../cmd/mesh-self-check/probes_mesh.go | 12 +- .../cmd/mesh-self-check/probes_root.go | 12 +- .../cmd/mesh-self-check/probes_test.go | 50 +++---- .../cmd/mesh-self-check/selfcheck.go | 130 ++++++++++-------- .../cmd/mesh-self-check/selfcheck_test.go | 109 +++++++++------ .../mesh-self-check/whole_integration_test.go | 31 ++++- .../cmd/mesh-self-check/words.go | 30 ++-- 12 files changed, 312 insertions(+), 216 deletions(-) create mode 100644 modules/mesh-self-check/cmd/mesh-self-check/findings.go diff --git a/modules/mesh-self-check/cmd/mesh-self-check/findings.go b/modules/mesh-self-check/cmd/mesh-self-check/findings.go new file mode 100644 index 000000000..5cba453e7 --- /dev/null +++ b/modules/mesh-self-check/cmd/mesh-self-check/findings.go @@ -0,0 +1,52 @@ +package main + +// The self-check's findings are the generated finding of the seat mesh-conditions, written through mesh-sdk's finding +// writer (design 47 G §461, §689); the probes build them with these two helpers. + +import ( + "strings" + + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// Finding, and the writer's other three bodies, are the seat mesh-conditions' own types. +type ( + Finding = meshconditions.Finding + FindingCleared = meshconditions.FindingCleared + FindingsStated = meshconditions.FindingsStated + FindingsAsked = meshconditions.FindingsAskedEvent + FindingsWriter = views.Writer[Finding, FindingCleared, FindingsStated, FindingsAsked] +) + +// idOf is a finding's id: the id of what it is about, and a ref that tells findings of one kind on one id apart after it +// (`.`, each escaped as a name of a key is; design 47 G A-14). +func idOf(id, ref string) string { + if ref == "" { + return keys.EscapeName(id) + } + return keys.EscapeName(id) + "." + keys.EscapeName(ref) +} + +// finding is f about scope and id (with ref when findings of one kind on one id need telling apart), on machine: it sets +// the key's scope and id and the subject's scope and id equal (design 47 G A-14). +func finding(scope, id, ref, machine string, f Finding) Finding { + f.Scope, f.ID = scope, idOf(id, ref) + f.Subject = meshconditions.Subject{Scope: scope, ID: f.ID, Machine: machine} + return f +} + +// said is the one observation a finding keeps as evidence; its moment is the finding's own (`at`), so that a finding +// that says the same is not written again. +func said(s string) []meshconditions.Evidence { + return []meshconditions.Evidence{{Said: strings.TrimSpace(s)}} +} + +// keyOf is where f is kept: `..`. +func keyOf(f Finding) string { return views.FindingKey(f.Scope, f.ID, f.Kind) } + +// findingKey is the key of the finding of kind about scope and id with ref, as finding makes it. +func findingKey(scope, id, ref, kind string) string { + return views.FindingKey(scope, idOf(id, ref), kind) +} diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go index 7c8ae0e17..985011397 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/main.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -14,7 +14,6 @@ import ( placing "git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" @@ -65,18 +64,26 @@ func (r runtimeStore) Watch(state string, handle func(key string, value []byte, // NewChecker is the self-check over the runtime. func NewChecker() *Checker { - store := runtimeStore{views: newViews()} - return &Checker{Store: store, Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), + c := &Checker{Registry: Registry(), Now: time.Now, Logf: logf, Host: os.Getenv("MESH_NODE"), Enabled: enabledProbes, Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, Publish: stdio.SeatPublish} + w, err := views.NewStdioWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ + Module: Module, State: stdio.State(FindingsState), Observe: c.Observe, Logf: logf}) + if err != nil { + logf("%v", err) + os.Exit(1) + } + c.Writer = w + c.Store = runtimeStore{views: newViews(w)} + return c } // newViews is the self-check's pictures of the indexer's and the assigner's states it probes (ADR 0615): each asked once of // its owner's read tool, then followed by the owner's events. A first view unanswered past its bound is a finding of the // self-check's own (first-view-unanswered), written into its state findings. -func newViews() *views.Reader { - r := &views.Reader{Module: Module, Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} +func newViews(w *FindingsWriter) *views.Reader { + r := &views.Reader{Module: Module, Logf: logf, Findings: views.WriterFindings{Writer: w}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} r.Follow(assignerModule, placing.StateAssignments, AssignmentsTool) r.Follow(assignerModule, placing.StateSeatHolders, SeatHoldersTool) r.Follow(indexerModule, "modules", ModulesTool) @@ -90,6 +97,9 @@ func main() { <-stdio.Served() c.Store.(runtimeStore).views.Start(context.Background()) c.WatchKept(context.Background()) + if err := c.Writer.Start(context.Background()); err != nil { + logf("the finding writer did not start: %v", err) + } c.Keep(context.Background()) }() if err := stdio.Serve("", append(Tools(c), c.Store.(runtimeStore).views.Tools("mesh_self_check")...)); err != nil { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes.go b/modules/mesh-self-check/cmd/mesh-self-check/probes.go index 2ed296128..39a5614bc 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes.go @@ -140,7 +140,7 @@ func (c *Checker) streams(stream string) ([]streamRow, error) { } // probeStreams is D7: every stream the mesh defines exists as defined. -func probeStreams(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeStreams(ctx context.Context, c *Checker) ([]Finding, error) { rows, err := c.streams("") if err != nil { return nil, err @@ -149,19 +149,19 @@ func probeStreams(ctx context.Context, c *Checker) ([]meshconditionsclient.Obser for _, r := range rows { held[r.Name] = r } - var out []meshconditionsclient.Observation + var out []Finding for _, s := range buscompose.MeshStreams() { have, ok := held[s.Name] if !ok { - out = append(out, meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: s.Name, - Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), - Said: "no stream " + s.Name}) + out = append(out, finding(meshconditionsclient.ScopeBus, s.Name, "stream-missing", "", Finding{Kind: "stream-wrong", Severity: meshconditionsclient.Urgent, + Summary: fmt.Sprintf("the stream %s is not on the bus: %s", s.Name, firstLine(s.Why)), + Evidence: said("no stream " + s.Name)})) continue } if differs := streamDiffers(s, have); differs != "" { - out = append(out, meshconditionsclient.Observation{Ref: "stream-redefined", Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: s.Name, - Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), - Said: differs}) + out = append(out, finding(meshconditionsclient.ScopeBus, s.Name, "stream-redefined", "", Finding{Kind: "stream-wrong", Severity: meshconditionsclient.Warning, + Summary: fmt.Sprintf("the stream %s is not as the mesh defines it: %s", s.Name, differs), + Evidence: said(differs)})) } } return sorted(out), nil @@ -213,7 +213,7 @@ type closedAnswer struct { } // probeReconnects is D15: each user of the bus whose connections were dropped more than twelve times an hour. -func probeReconnects(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeReconnects(_ context.Context, c *Checker) ([]Finding, error) { raw, err := c.Ask(ClosedTool, map[string]any{"hours": 1}) if err != nil { return nil, fmt.Errorf("the bus module did not answer %s: %w", ClosedTool, err) @@ -226,7 +226,7 @@ func probeReconnects(_ context.Context, c *Checker) ([]meshconditionsclient.Obse if hours <= 0 { hours = 1 } - var out []meshconditionsclient.Observation + var out []Finding for _, u := range read.Users { if u.User == "" || strings.HasPrefix(u.User, "(") || float64(u.Dropped)/hours <= reconnectMax { continue // refused before it logged in is nobody's reconnect loop @@ -254,16 +254,14 @@ func probeReconnects(_ context.Context, c *Checker) ([]meshconditionsclient.Obse partial = " (at least: the server's record of closed connections does not reach back the whole hour)" } who, machine := busUserWords(u.User) - out = append(out, meshconditionsclient.Observation{Kind: "bus-reconnects", Severity: meshconditionsclient.Warning, - Scope: meshconditionsclient.ScopeBus, ID: u.User, Machine: machine, + out = append(out, finding(meshconditionsclient.ScopeBus, u.User, "", machine, Finding{Kind: "bus-reconnects", Severity: meshconditionsclient.Warning, Summary: fmt.Sprintf("the bus dropped %s's connection %d times in the last hour%s, more than %d: a client "+ "reconnecting in a loop", u.User, u.Dropped, partial, reconnectMax), - Said: fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, - hours, strings.Join(names, ", "), strings.Join(why, ", ")), + Evidence: said(fmt.Sprintf("%d of %d closed connections dropped in %.0f h; by name %s; why %s", u.Dropped, u.Closed, + hours, strings.Join(names, ", "), strings.Join(why, ", "))), Headline: capital(who) + " keeps losing the bus", Explanation: capital(fmt.Sprintf("%s lost its connection to the bus %d times in the last hour and connected "+ - "again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped)), - Resolved: "Resolved: " + who + " stays connected"}) + "again each time. While it reconnects, what it says and what it is asked waits.", who, u.Dropped))})) } return sorted(out), nil } @@ -290,12 +288,12 @@ func busUserWords(user string) (string, string) { // probeWatchdogs is DW: the watchdogs answer. The self-check and the watchdogs watch each other: S10, the // watchman's pulse on `checked`, is the other half. -func probeWatchdogs(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeWatchdogs(_ context.Context, c *Checker) ([]Finding, error) { if _, err := c.Ask(SignalsVerb, map[string]any{}); err != nil { - return []meshconditionsclient.Observation{{Ref: "silent", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeCore, ID: "watchdogs", + return []Finding{finding(meshconditionsclient.ScopeCore, "watchdogs", "", "", Finding{Kind: "watchdogs-silent", Severity: meshconditionsclient.Urgent, Summary: "the watchdogs do not answer: the watchman holding mesh-watchdogs did not answer signals, so no " + "late signal is being said", - Said: firstLine(err.Error())}}, nil + Evidence: said(firstLine(err.Error()))})}, nil } return nil, nil } @@ -304,7 +302,7 @@ func probeWatchdogs(_ context.Context, c *Checker) ([]meshconditionsclient.Obser // machine and back. The mesh defines no durable consumer of its own since the controller ended: the account consumers // are the grants part's to make and judge (issue 1126). The question to the bus module is that request: an answer is its round trip, // and no answer is no pass. -func probeBusHealth(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeBusHealth(ctx context.Context, c *Checker) ([]Finding, error) { rows, err := c.streams("") if err != nil { return nil, fmt.Errorf("no request crossed the bus to its module and came back: %w", err) @@ -322,13 +320,13 @@ func probeBusHealth(ctx context.Context, c *Checker) ([]meshconditionsclient.Obs if len(problems) == 0 { return nil, nil } - return []meshconditionsclient.Observation{{Ref: "unhealthy", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "mesh", - Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Said: strings.Join(problems, "; ")}}, nil + return []Finding{finding(meshconditionsclient.ScopeBus, "mesh", "", "", Finding{Kind: "core-unhealthy", Severity: meshconditionsclient.Urgent, + Summary: "the bus is not healthy: " + strings.Join(problems, "; "), Evidence: said(strings.Join(problems, "; "))})}, nil } // sorted is findings in a stated order, so two runs over one mesh say the same. -func sorted(out []meshconditionsclient.Observation) []meshconditionsclient.Observation { - sort.SliceStable(out, func(i, j int) bool { return out[i].Key() < out[j].Key() }) +func sorted(out []Finding) []Finding { + sort.SliceStable(out, func(i, j int) bool { return keyOf(out[i]) < keyOf(out[j]) }) return out } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go index 22f8658a0..42ce1fd4a 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_data.go @@ -192,7 +192,7 @@ func (c *Checker) dataFacts(ctx context.Context) (dataFacts, error) { } // probeData is D13. -func probeData(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeData(ctx context.Context, c *Checker) ([]Finding, error) { f, err := c.dataFacts(ctx) if err != nil { return nil, err @@ -263,14 +263,13 @@ func firstSeen(r diskreadings.Reading, now time.Time) time.Time { return now } -func obs(kind string, id, machine string, sev string, summary, said string) meshconditionsclient.Observation { - return meshconditionsclient.Observation{Kind: kind, Ref: kind, Severity: sev, Summary: summary, Said: said, - Scope: ScopeMachine, ID: id, Machine: machine} +func obs(kind string, id, machine string, sev string, summary, saying string) Finding { + return finding(ScopeMachine, id, "", machine, Finding{Kind: kind, Severity: sev, Summary: summary, Evidence: said(saying)}) } // dataFindings is every condition the data on record raises now: a function of what is known, as the controller's was. -func dataFindings(f dataFacts) []meshconditionsclient.Observation { - var out []meshconditionsclient.Observation +func dataFindings(f dataFacts) []Finding { + var out []Finding arrays := map[string][]diskreadings.Reading{} type shrunk struct { machine, dataset, class string @@ -301,7 +300,6 @@ func dataFindings(f dataFacts) []meshconditionsclient.Observation { "`cleanup delete %s %s %s --why …` once a person has decided, or assign %s there again", r.ID, r.Module, r.Node, r.Class, sizeWords(r.SizeBytes), int(f.now.Sub(r.RetiredAt).Hours()/24), r.RetiredWhy, r.Node, r.Module, r.ID, r.Module), "kept at "+orUnknownPath(r.Path))) - out[len(out)-1].Ref = "cleanup" } continue } @@ -428,7 +426,7 @@ func dataFindings(f dataFacts) []meshconditionsclient.Observation { fmt.Sprintf("%s of %s on %s holds %s, and the copy %s kept on %s holds %s: %s is running on "+ "an empty replacement of its data. Move the data, or assign it back where its data is", a.ID, a.Module, a.Node, sizeWords(a.SizeBytes), o.Module, o.Node, sizeWords(o.SizeBytes), a.Module), "") - ob.Also = []string{o.Node} + ob.Subject.Also = []string{o.Node} out = append(out, ob) break } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go index b4d96dcca..b3574445e 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_fill.go @@ -55,39 +55,37 @@ func mibWords(b uint64) string { return fmt.Sprintf("%.1f MiB", float64(b)/(1024 // fillKey is where a bucket's or log's fill finding is kept. func fillKey(stream string) string { - return meshconditionsclient.Observation{Scope: meshconditionsclient.ScopeBus, ID: stream, Ref: "filling"}.Key() + return findingKey(meshconditionsclient.ScopeBus, stream, "", "bucket-or-log-filling") } // fillObservation is what one bucket's or log's fill says: a finding at fillRaiseAt or above, and, while its finding // is written, at fillClearBelow or above too; nothing otherwise, which is what clears it. -func fillObservation(stream string, r fillRow, open bool) (meshconditionsclient.Observation, bool) { +func fillObservation(stream string, r fillRow, open bool) (Finding, bool) { module, kind, name, ok := bucketOrLog(stream) if !ok || r.MaxBytes == 0 { - return meshconditionsclient.Observation{}, false // no module's, or with no cap: it cannot fill + return Finding{}, false // no module's, or with no cap: it cannot fill } // Compared in bytes, not rounded percent: 74.9% is not 75%. atRaise := r.Bytes*100 >= r.MaxBytes*fillRaiseAt aboveClear := r.Bytes*100 >= r.MaxBytes*fillClearBelow if !atRaise && !(open && aboveClear) { - return meshconditionsclient.Observation{}, false + return Finding{}, false } pct := r.Bytes * 100 / r.MaxBytes - return meshconditionsclient.Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: meshconditionsclient.Warning, - Scope: meshconditionsclient.ScopeBus, ID: stream, + return finding(meshconditionsclient.ScopeBus, stream, "", "", Finding{Kind: "bucket-or-log-filling", Severity: meshconditionsclient.Warning, Summary: fmt.Sprintf("%s's %s %s holds %s of %s, %d%%: at its cap the bus refuses the module's next write", module, kind, name, mibWords(r.Bytes), mibWords(r.MaxBytes), pct), - Said: fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes, - r.MaxBytes, pct), + Evidence: said(fmt.Sprintf("module %s, %s %s (stream %s): %d of %d bytes, %d%%", module, kind, name, stream, r.Bytes, + r.MaxBytes, pct)), Headline: "A module's bucket or log on the bus is filling up", Needs: "decide whether to raise its cap or have the module keep less.", Explanation: "A bucket or log a module keeps on the bus holds three quarters of its cap or more. When it is " + - "full, the bus refuses what the module writes there.", - Resolved: "It has room again"}, true + "full, the bus refuses what the module writes there."}), true } // probeFill is D-fill: every module's bucket and log on the bus judged against its cap. The bus module not answering // is the probe failing to run, which clears nothing; a finding already written counts as open. -func probeFill(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeFill(_ context.Context, c *Checker) ([]Finding, error) { raw, err := c.Ask(StreamsTool, map[string]any{}) if err != nil { return nil, fmt.Errorf("the bus module did not answer %s: %w", StreamsTool, err) @@ -103,7 +101,7 @@ func probeFill(_ context.Context, c *Checker) ([]meshconditionsclient.Observatio return nil, fmt.Errorf("its own findings cannot be read, so a fill between the thresholds cannot be judged: %w", err) } sort.Slice(a.Detail, func(i, j int) bool { return a.Detail[i].Name < a.Detail[j].Name }) - var out []meshconditionsclient.Observation + var out []Finding for _, r := range a.Detail { if o, said := fillObservation(r.Name, r, written.has(fillKey(r.Name))); said { out = append(out, o) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go index 18546ccfa..6cbd40ed5 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_mesh.go @@ -128,7 +128,7 @@ var lookupHost = func(ctx context.Context, host string) ([]string, error) { // probeBans is D8: no address the mesh owns is in any machine's ban list. The mesh's own addresses are every machine's // private address and the address its endpoint names, from assignments; the ban lists are asked of each machine's holder // of node-intrusion-prevention that is heard from. -func probeBans(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeBans(ctx context.Context, c *Checker) ([]Finding, error) { places, err := c.assignments() if err != nil { return nil, err @@ -183,7 +183,7 @@ func probeBans(ctx context.Context, c *Checker) ([]meshconditionsclient.Observat }(i, node) } wg.Wait() - var out []meshconditionsclient.Observation + var out []Finding var unasked []string for i, node := range holders { if errs[i] != nil { @@ -206,16 +206,14 @@ func probeBans(ctx context.Context, c *Checker) ([]meshconditionsclient.Observat } } if len(banned) > 0 { - out = append(out, meshconditionsclient.Observation{Kind: "own-address-banned", Ref: "bans-the-mesh", Severity: meshconditionsclient.Urgent, - Scope: ScopeMachine, ID: node, Machine: node, + out = append(out, finding(ScopeMachine, node, "", node, Finding{Kind: "own-address-banned", Severity: meshconditionsclient.Urgent, Summary: fmt.Sprintf("%s's ban list holds %d of the mesh's own address(es) — %s: the mesh is locked out "+ "of itself there (ADR 0186)", node, len(banned), strings.Join(whose, ", ")), - Said: strings.Join(banned, ", "), + Evidence: said(strings.Join(banned, ", ")), Headline: node + " has banned the mesh", Needs: fmt.Sprintf("lift the ban on %s, or wait for it to expire.", node), Explanation: fmt.Sprintf("The intrusion protection on %s banned one of the mesh's own addresses, so the "+ - "mesh is locked out of it.", node), - Resolved: node + " no longer bans the mesh"}) + "mesh is locked out of it.", node)})) } } if len(unasked) > 0 { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go index e81de034b..e174917db 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_root.go @@ -205,7 +205,7 @@ func (c *Checker) confined(node string, place placing.Assignments, heard bool) ( // probeAgentAccounts is DA: every machine that names an agent account has it judged, on its node-engine's newest // statement, unable to become root without a person (ADR 0266). -func probeAgentAccounts(_ context.Context, c *Checker) ([]meshconditionsclient.Observation, error) { +func probeAgentAccounts(_ context.Context, c *Checker) ([]Finding, error) { places, err := c.assignments() if err != nil { return nil, err @@ -219,7 +219,7 @@ func probeAgentAccounts(_ context.Context, c *Checker) ([]meshconditionsclient.O nodes = append(nodes, n) } sort.Strings(nodes) - var out []meshconditionsclient.Observation + var out []Finding for _, n := range nodes { if places[n].Decided.AgentAccount == "" { continue @@ -233,17 +233,15 @@ func probeAgentAccounts(_ context.Context, c *Checker) ([]meshconditionsclient.O if v.Confined || v.Quiet { continue } - out = append(out, meshconditionsclient.Observation{Kind: "agent-can-become-root", Ref: "agent-root", Severity: meshconditionsclient.Urgent, - Scope: ScopeMachine, ID: n, Machine: n, + out = append(out, finding(ScopeMachine, n, "", n, Finding{Kind: "agent-can-become-root", Severity: meshconditionsclient.Urgent, Summary: fmt.Sprintf("on %s, %s (ADR 0266): an agent there may become root without a person, and no "+ "answer from a channel authorises an act there (ADR 0259 §8)", n, v.Why), - Said: v.Why, + Evidence: said(v.Why), Headline: "Sessions on " + n + " could become root", Needs: "take the sessions' own account out of every group and rule that grants root; the details say which.", Explanation: fmt.Sprintf("Assistant sessions on %s run under an account of their own, so that none "+ "can take over the machine without you. The machine cannot show that this holds now, so an answer "+ - "from your phone authorises nothing there until it does.", n), - Resolved: "Sessions on " + n + " cannot become root again"}) + "from your phone authorises nothing there until it does.", n)})) } return sorted(out), nil } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go index 4dc7d65c2..b013e3478 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/probes_test.go @@ -107,10 +107,10 @@ func (b bus) ask(key string, body any) (json.RawMessage, error) { return nil, errors.New("nats: no responders available for request") } -func keysOf(found []meshconditionsclient.Observation) []string { +func keysOf(found []Finding) []string { var out []string for _, f := range found { - out = append(out, f.Key()) + out = append(out, keyOf(f)) } return out } @@ -119,7 +119,7 @@ func keysOf(found []meshconditionsclient.Observation) []string { func TestTheBusProbes(t *testing.T) { cases := []struct { name string - probe func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) + probe func(context.Context, *Checker) ([]Finding, error) bus bus want []string err string @@ -128,24 +128,24 @@ func TestTheBusProbes(t *testing.T) { {"D7 all as defined", probeStreams, bus{}, nil, "", nil}, {"D7 missing and redefined", probeStreams, bus{drop: map[string]bool{"NODES": true}, change: func(rows map[string]*streamRow) { rows["EVENTS"].Retention = "workqueue" - }}, []string{"bus.EVENTS.stream-redefined", "bus.NODES.stream-missing"}, "", nil}, + }}, []string{"bus.EVENTS.stream-redefined.stream-wrong", "bus.NODES.stream-missing.stream-wrong"}, "", nil}, {"D15 quiet", probeReconnects, bus{}, nil, "", nil}, {"D15 a loop", probeReconnects, bus{closed: `{"hours":1,"reaches":true,"users":[ {"user":"g14.node-tools","closed":30,"dropped":20,"names":[{"name":"node-tools","dropped":20,"reasons":{"Stale Connection":20}}]}, {"user":"ace.records","closed":5,"dropped":3},{"user":"(unauthenticated)","closed":90,"dropped":90}]}`}, - []string{"bus.g14.node-tools.bus-reconnects"}, "", nil}, + []string{"bus.g14=2enode-tools.bus-reconnects"}, "", nil}, {"DW answers", probeWatchdogs, bus{}, nil, "", nil}, - {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.silent"}, "", nil}, + {"DW silent", probeWatchdogs, bus{signals: errors.New("no responders")}, []string{"core.watchdogs.watchdogs-silent"}, "", nil}, {"H-bus healthy", probeBusHealth, bus{}, nil, "", nil}, {"H-bus a stream missing", probeBusHealth, - bus{drop: map[string]bool{"NODES": true}}, []string{"bus.mesh.unhealthy"}, "", nil}, + bus{drop: map[string]bool{"NODES": true}}, []string{"bus.mesh.core-unhealthy"}, "", nil}, {"H-bus no round trip", probeBusHealth, bus{asksFail: errors.New("timeout")}, nil, "no request crossed", nil}, // D8 (issue 238): the assignments' addresses and endpoints against each heard holder's ban list. {"D8 nothing of the mesh banned", probeBans, bus{banned: map[string]string{"ace": `{"banned":["198.51.100.7"]}`, "novox": `{"banned":[]}`}}, nil, "", machines()}, {"D8 a private address and an endpoint banned", probeBans, bus{banned: map[string]string{ "ace": `{"jails":{"sshd":["10.10.0.1","198.51.100.7"]}}`, "novox": `{"banned":["203.0.113.5"]}`}}, - []string{"machine.ace.bans-the-mesh", "machine.novox.bans-the-mesh"}, "", machines()}, + []string{"machine.ace.own-address-banned", "machine.novox.own-address-banned"}, "", machines()}, {"D8 a machine not heard from is not asked", probeBans, bus{banned: map[string]string{"novox": `{"banned":[]}`, "ace": `{"banned":[]}`}}, nil, "", machines()}, {"D8 a heard holder that does not answer is no pass", probeBans, bus{banned: map[string]string{}}, nil, @@ -154,9 +154,9 @@ func TestTheBusProbes(t *testing.T) { // the manifests declare; the kinds, keys and words are the controller's. {"D13 all held", probeData, bus{}, nil, "", dataMesh(func(*dataRig) {})}, {"D13 shrank, gone, quiet, unprotected and stale", probeData, bus{}, []string{ - "machine.ace.pg.data.data-quiet", "machine.ace.pg.data.backup-stale", "machine.ace.pg.dumps.data-missing", - "machine.ace.plex.movies.protection-missing", "machine.ace.plex.thumbs.data-shrank", - "machine.ace.array.md-md127.array-degraded"}, + "machine.ace=2epg=2edata.data-quiet", "machine.ace=2epg=2edata.backup-stale", "machine.ace=2epg=2edumps.data-missing", + "machine.ace=2eplex=2emovies.protection-missing", "machine.ace=2eplex=2ethumbs.data-shrank", + "machine.ace=2earray=2emd-md127.array-degraded"}, "", dataMesh(func(r *dataRig) { r.size("ace", "pg", "data", 100<<20, 100<<20).write("ace", "pg", "data", -80*time.Hour) r.gone("ace", "pg", "dumps") @@ -165,23 +165,23 @@ func TestTheBusProbes(t *testing.T) { r.size("ace", "plex", "library", 1<<30, 1<<30).array("ace", "plex", "library", "md", "md127", false) r.lastGood("ace", "pg", "data", -100*time.Hour) })}, - {"D13 a dataset shrank once for its items", probeData, bus{}, []string{"machine.ace.dataset.storage-media.data-shrank"}, + {"D13 a dataset shrank once for its items", probeData, bus{}, []string{"machine.ace=2edataset=2estorage-media.data-shrank"}, "", dataMesh(func(r *dataRig) { r.size("ace", "plex", "thumbs", 10<<20, 600<<20).dataset("ace", "plex", "thumbs", "storage/media") r.size("ace", "plex", "meta", 10<<20, 600<<20).dataset("ace", "plex", "meta", "storage/media") })}, {"D13 a retired item is kept, not judged, and cleanup waits after thirty days", probeData, bus{}, []string{ - "machine.ace.pg.dumps.cleanup"}, "", dataMesh(func(r *dataRig) { + "machine.ace=2epg=2edumps.cleanup-waiting"}, "", dataMesh(func(r *dataRig) { r.size("ace", "pg", "dumps", 1<<20, 1<<30).retired("ace", "pg", "dumps", -31*24*time.Hour) r.size("ace", "pg", "old", 1<<20, 1<<30).retired("ace", "pg", "old", -2*24*time.Hour) })}, {"D13 a copy far smaller than a retired one elsewhere is an empty replacement", probeData, bus{}, - []string{"machine.ace.plex.thumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) { + []string{"machine.ace=2eplex=2ethumbs.empty-replacement"}, "", dataMesh(func(r *dataRig) { r.size("ace", "plex", "thumbs", 10<<20, 10<<20) r.size("g14", "plex", "thumbs", 600<<20, 600<<20).retired("g14", "plex", "thumbs", -24*time.Hour) })}, {"D13 no good backup is counted from when the item was first measured", probeData, bus{}, - []string{"machine.ace.pg.data.backup-stale"}, "", dataMesh(func(r *dataRig) { + []string{"machine.ace=2epg=2edata.backup-stale"}, "", dataMesh(func(r *dataRig) { delete(r.good, diskreadings.ReadingKey("ace", "pg", "data")) delete(r.good, diskreadings.ReadingKey("ace", "plex", "meta")) r.firstSeen("ace", "pg", "data", -100*time.Hour).firstSeen("ace", "plex", "meta", -time.Hour) @@ -193,24 +193,24 @@ func TestTheBusProbes(t *testing.T) { {Name: "LOG_mesh-delivery-builder_build-log", Bytes: 10, MaxBytes: 100}, {Name: "KV_nocap_x", Bytes: 999}}}, nil, "", nil}, {"D-fill a bucket and a log filling", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 75, MaxBytes: 100}, {Name: "LOG_mesh-delivery-builder_build-log", Bytes: 99, MaxBytes: 100}}}, - []string{"bus.KV_mesh-issues_issues.filling", "bus.LOG_mesh-delivery-builder_build-log.filling"}, "", nil}, + []string{"bus.KV_mesh-issues_issues.bucket-or-log-filling", "bus.LOG_mesh-delivery-builder_build-log.bucket-or-log-filling"}, "", nil}, {"D-fill between the thresholds raises nothing new", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, nil, "", nil}, {"D-fill between the thresholds keeps what is said", probeFill, bus{fills: []fillRow{{Name: "KV_mesh-issues_issues", Bytes: 72, MaxBytes: 100}}}, - []string{"bus.KV_mesh-issues_issues.filling"}, "", filling("KV_mesh-issues_issues")}, + []string{"bus.KV_mesh-issues_issues.bucket-or-log-filling"}, "", filling("KV_mesh-issues_issues")}, {"D-fill the bus module silent", probeFill, bus{asksFail: errors.New("no responders")}, nil, "did not answer", nil}, // DA (ADR 0266): the agent account each machine names, judged on its engine's report while it is heard from. {"DA confined", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": stateHealthy})}, - {"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + {"DA a way to root found", probeAgentAccounts, bus{}, []string{"machine.novox.agent-can-become-root"}, "", agents(map[string]string{"novox": stateUnhealthy})}, - {"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + {"DA no verdict on it", probeAgentAccounts, bus{}, []string{"machine.novox.agent-can-become-root"}, "", agents(map[string]string{"novox": ""})}, {"DA the search runs, within its quiet", probeAgentAccounts, bus{}, nil, "", agents(map[string]string{"novox": "pending"})}, - {"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-root"}, + {"DA a machine not heard from has no verdict on now", probeAgentAccounts, bus{}, []string{"machine.g14.agent-can-become-root"}, "", agents(map[string]string{"novox": stateHealthy, "g14": stateHealthy})}, - {"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-root"}, "", + {"DA an engine older than the judging", probeAgentAccounts, bus{}, []string{"machine.novox.agent-can-become-root"}, "", agents(map[string]string{"novox": "old"})}, } for _, tc := range cases { @@ -301,10 +301,10 @@ func placed(s *memStore) map[string]placing.Assignments { // filling is a store holding the self-check's written fill finding of a stream: the condition open. func filling(stream string) *memStore { s := newStore() - o := meshconditionsclient.Observation{Kind: "bucket-or-log-filling", Ref: "filling", Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: stream, - Summary: "filling", Source: "D-fill"} - raw, _ := json.Marshal(o.Finding(Module, "run", time.Now())) - _ = s.Put(FindingsState, o.Key(), json.RawMessage(raw)) + o := finding(meshconditionsclient.ScopeBus, stream, "", "", Finding{Kind: "bucket-or-log-filling", Severity: meshconditionsclient.Warning, + Summary: "filling", Source: "D-fill", Observer: Module, Run: "run", At: time.Now(), Since: time.Now()}) + raw, _ := json.Marshal(o) + _ = s.Put(FindingsState, keyOf(o), json.RawMessage(raw)) return s } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go index ba9ac720a..3ec8cca8d 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck.go @@ -2,7 +2,7 @@ package main // The self-check (novox/hq ADR 0564 §6): the registry of probes, each a live invariant of a design, run every five // minutes, each probe within thirty seconds. What a probe finds is written as a finding in this module's bucket -// `findings`, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the +// `findings`, through mesh-sdk's finding writer, which the condition keeper raises and clears conditions from (ADR 0564 §2); every run is kept in the // bucket `verdicts` for a week and the last as the key `last` of `verdict`, and is said as `checked`, which the // watchman and mesh-watcher hear (S10, ADR 0227 §6). A probe that errors or does not finish is never a pass: it is // said in the verdict at once and written as a finding of its own when the next run cannot run it either. @@ -20,6 +20,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" meshselfcheck "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshselfcheck" + "git.novox.be/novox/mesh-sdk/go/views" ) // Module and Seat are this module and the seat it holds once for the mesh. @@ -95,7 +96,7 @@ type Probe struct { // Deferred names the facts it lacks on the bus, for a probe whose facts have no home there yet (ADR 0564 §6): // answered deferred, never a pass. Deferred string - Run func(ctx context.Context, c *Checker) ([]meshconditionsclient.Observation, error) + Run func(ctx context.Context, c *Checker) ([]Finding, error) } // ProbeVerdict, Counts and Run are a run as the seat mesh-self-check's contract holds it: the body of `checked`, the controller's @@ -119,6 +120,9 @@ type Checker struct { Logf func(format string, args ...any) Host string Registry []Probe + // Writer is the finding writer over this module's state `findings` (views.NewStdioWriter; design 47 G A-16), whose + // observe is Observe. + Writer *FindingsWriter running sync.Mutex mu sync.Mutex @@ -144,14 +148,36 @@ func (c *Checker) LastRun() *Run { } type result struct { - found []meshconditionsclient.Observation + found []Finding err error took time.Duration } -// RunOnce runs every probe of the registry, writes what changed in `findings`, keeps the verdict and says `checked`. -// One run at a time: a person's run during a scheduled one waits for it. +// The context a run is asked with: why it runs, and where its verdict is answered. +type ( + whyKey struct{} + runOut struct{} +) + +// RunOnce runs every probe of the registry through the findings writer, which writes what changed in `findings` and says +// `findings-stated` (design 47 G §461, §485), keeps the verdict and says `checked`. One run at a time: a person's run +// during a scheduled one waits for it. func (c *Checker) RunOnce(ctx context.Context, why string) Run { + var out Run + ctx = context.WithValue(context.WithValue(ctx, whyKey{}, why), runOut{}, &out) + if err := c.Writer.State(ctx, ""); err != nil { + c.Logf("[%s] what this run found is not all written: %v", Module, err) + } + return out +} + +// Observe is the writer's observe: it runs every probe, keeps the verdict and says `checked`, and answers the findings +// this run observes and the keys it cannot judge (those of a probe that could not run). +func (c *Checker) Observe(ctx context.Context) (views.Observed[Finding], error) { + why, _ := ctx.Value(whyKey{}).(string) + if why == "" { + why = "a statement of the findings" + } c.running.Lock() defer c.running.Unlock() c.mu.Lock() @@ -179,9 +205,9 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { written, readErr := c.written() seen := map[string]bool{} - want := map[string]meshconditionsclient.Observation{} // every finding this run writes, by key - ran := map[string]bool{} // the sources whose findings this run judged whole - var blind []meshconditionsclient.Observation + want := map[string]Finding{} // every finding this run observes, by key + ran := map[string]bool{} // the sources whose findings this run judged whole + var blind []Finding for i, p := range c.Registry { if !on(p.ID) { // Off by the setting: it ran nothing and sees nothing, so what it raised before is cleared, and it has no @@ -199,10 +225,10 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { case r.err != nil: v.Verdict, v.Error = VerdictFailedToRun, r.err.Error() run.Counts.FailedToRun++ - blind = append(blind, meshconditionsclient.Observation{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning, - Scope: meshconditionsclient.ScopeProbe, ID: p.ID, Source: sourceSelf, - Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), - Said: firstLine(r.err.Error())}) + blind = append(blind, finding(meshconditionsclient.ScopeProbe, p.ID, "", "", Finding{Kind: "probe-failed", + Severity: meshconditionsclient.Warning, Source: sourceSelf, + Summary: fmt.Sprintf("the probe %s (%s) could not run: what it checks is not known — never a pass", p.ID, p.Asserts), + Evidence: said(firstLine(r.err.Error()))})) default: ran[p.ID] = true for _, f := range r.found { @@ -211,7 +237,7 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { } f.Source = p.ID f = worded(f) - key := f.Key() + key := keyOf(f) seen[key] = true if confirm(f) && !c.seenBefore(key) && !written.has(key) { v.Unconfirmed = append(v.Unconfirmed, f.Summary) @@ -236,18 +262,12 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { ran[sourceSelf] = true for _, f := range blind { f = worded(f) - key := f.Key() + key := keyOf(f) seen[key] = true if c.seenBefore(key) || written.has(key) { want[key] = f } } - if readErr != nil { - // Blind to its own bucket: nothing is written or deleted (ADR 0227 §4), and the log says so. - c.Logf("[%s] its findings cannot be read, so none is written or cleared this run: %v", Module, readErr) - } else if err := c.write(written, want, ran, run.Run); err != nil { - c.Logf("[%s] what this run found is not all written: %v", Module, err) - } c.mu.Lock() c.seen = seen c.mu.Unlock() @@ -258,7 +278,32 @@ func (c *Checker) RunOnce(ctx context.Context, why string) Run { run.Unsaid = int64(c.unsaid) c.mu.Unlock() c.keep(run) - return run + if out, ok := ctx.Value(runOut{}).(*Run); ok { + *out = run + } + if readErr != nil { + // Blind to its own findings: no set is said, so nothing is written or cleared (ADR 0227 §4). + return views.Observed[Finding]{}, fmt.Errorf("its findings cannot be read, so none is written or cleared this run: %w", readErr) + } + keys := make([]string, 0, len(want)) + for k := range want { + keys = append(keys, k) + } + sort.Strings(keys) + obs := views.Observed[Finding]{} + for _, k := range keys { + f := want[k] + f.Run = run.Run + obs.Findings = append(obs.Findings, f) + } + // A finding of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). + for k, f := range written { + if _, still := want[k]; !still && !ran[f.Source] { + obs.Pending = append(obs.Pending, views.PendingKey{Key: k, Since: f.Since, Bound: Every.String()}) + } + } + sort.Slice(obs.Pending, func(i, j int) bool { return obs.Pending[i].Key < obs.Pending[j].Key }) + return obs, nil } // probe runs one probe within its bound; a panic or an overrun is an error, never a pass. @@ -291,8 +336,8 @@ func (c *Checker) seenBefore(key string) bool { return c.seen[key] } -// writtenFindings are the findings in the bucket, by key. -type writtenFindings map[string]meshconditions.FindingsByKeyValue +// writtenFindings are the findings in the state, by key. +type writtenFindings map[string]Finding func (w writtenFindings) has(key string) bool { _, ok := w[key]; return ok } @@ -311,7 +356,7 @@ func (c *Checker) written() (writtenFindings, error) { if !found { continue } - var f meshconditions.FindingsByKeyValue + var f Finding if err := json.Unmarshal(raw, &f); err != nil { return out, fmt.Errorf("the finding %s is not readable: %w", k, err) } @@ -320,43 +365,6 @@ func (c *Checker) written() (writtenFindings, error) { return out, nil } -// write puts each finding whose words changed and deletes each one a source that ran whole no longer sees. A finding -// of a source that could not run is left as it is: a blind observer clears nothing (ADR 0227 §4). -func (c *Checker) write(written writtenFindings, want map[string]meshconditionsclient.Observation, ran map[string]bool, run string) error { - var errs []string - keys := make([]string, 0, len(want)) - for k := range want { - keys = append(keys, k) - } - sort.Strings(keys) - now := c.Now() - for _, k := range keys { - f := want[k].Finding(Module, run, now) - if was, ok := written[k]; ok && meshconditionsclient.Says(was) == meshconditionsclient.Says(f) { - continue - } - raw, err := json.Marshal(f) - if err == nil { - err = c.Store.Put(FindingsState, k, json.RawMessage(raw)) - } - if err != nil { - errs = append(errs, k+": "+err.Error()) - } - } - for k, f := range written { - if _, still := want[k]; still || !ran[f.Source] { - continue - } - if err := c.Store.Delete(FindingsState, k); err != nil { - errs = append(errs, k+": "+err.Error()) - } - } - if len(errs) > 0 { - return fmt.Errorf("%s", strings.Join(errs, "; ")) - } - return nil -} - // keep keeps the run as the last verdict and in the week's verdicts, and says it as `checked`. Not said is said in // the log: the watchman's S10 says the self-check silent when `checked` stops. func (c *Checker) keep(run Run) { diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index f615b3cc2..04922b800 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -13,7 +13,8 @@ import ( "time" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" ) // memStore is the bus's buckets in memory, counting the writes and the reads one key at a time, and handing every @@ -25,6 +26,7 @@ type memStore struct { gets map[string]int watches map[string][]func(key string, value []byte, gone bool) keysErr error + events []string // the subjects the finding writer said its events on } func newStore() *memStore { @@ -97,6 +99,22 @@ func (s *memStore) findingKeys() []string { return keys } +// findingState is the writer's view of the store's state `findings`. +type findingState struct{ s *memStore } + +func (f findingState) Get(key string) (*stdio.StateEntry, error) { + raw, ok, err := f.s.Get(FindingsState, key) + if err != nil || !ok { + return nil, err + } + return &stdio.StateEntry{Key: key, Value: raw}, nil +} +func (f findingState) Put(key string, value any) (uint64, error) { + return 0, f.s.Put(FindingsState, key, value) +} +func (f findingState) Delete(key string) error { return f.s.Delete(FindingsState, key) } +func (f findingState) Keys() ([]string, error) { return f.s.Keys(FindingsState) } + // checker is a self-check over memory, with a registry the test drives. func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { var said []string @@ -108,6 +126,17 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { said = append(said, subject+" "+id+" "+string(raw)) return nil }} + w, err := views.NewWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ + Module: Module, Node: "novox", State: findingState{store}, Observe: c.Observe, Logf: c.Logf, Now: c.Now, + Publish: func(_ context.Context, subject string, _ []byte, _ string) error { + store.events = append(store.events, subject) + return nil + }, + Listen: func(context.Context, string, func([]byte)) (func(), error) { return func() {}, nil }}) + if err != nil { + panic(err) + } + c.Writer = w return c, &said } @@ -115,37 +144,36 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { // could not run clears nothing and is written as its own finding at the second run in a row; a finding one look can // be wrong about waits for the second run; a deferred probe is never a pass (ADR 0564 §2, §6; issue 1952 SC1). func TestARunWritesFindingsAsTheyChange(t *testing.T) { - var found []meshconditionsclient.Observation + var found []Finding var fail error - steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { return found, fail }} + steady := Probe{ID: "P1", Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Finding, error) { return found, fail }} deferred := Probe{ID: "P2", Kind: "resolver-wrong", Deferred: "the overlay's addresses"} - missing := meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "EVENTS", - Summary: "the stream EVENTS is not on the bus"} - behind := meshconditionsclient.Observation{Kind: kindDataUnmeasured, Ref: "behind", Severity: meshconditionsclient.Warning, - Scope: meshconditionsclient.ScopeBus, ID: "EVENTS.controller", Summary: "far behind"} + missing := finding(meshconditionsclient.ScopeBus, "EVENTS", "stream-missing", "", Finding{Severity: meshconditionsclient.Urgent, + Summary: "the stream EVENTS is not on the bus", Evidence: said("no stream EVENTS")}) + behind := finding(meshconditionsclient.ScopeBus, "EVENTS.controller", "", "", Finding{Kind: kindDataUnmeasured, Severity: meshconditionsclient.Warning, + Summary: "far behind"}) store := newStore() c, said := checker(store, steady, deferred) steps := []struct { - name string - found []meshconditionsclient.Observation - fail error - want []string // finding keys written after the run - verdict string - missingPs int // puts of the missing finding so far + name string + found []Finding + fail error + want []string // finding keys written after the run + verdict string }{ - {"a finding is written", []meshconditionsclient.Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"the same again is no write", []meshconditionsclient.Observation{missing}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"one look can be wrong: held", []meshconditionsclient.Observation{missing, behind}, nil, []string{"bus.EVENTS.stream-missing"}, VerdictFail, 1}, - {"seen twice: written", []meshconditionsclient.Observation{missing, behind}, nil, - []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFail, 1}, + {"a finding is written", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing.stream-wrong"}, VerdictFail}, + {"the same again is stated once more", []Finding{missing}, nil, []string{"bus.EVENTS.stream-missing.stream-wrong"}, VerdictFail}, + {"one look can be wrong: held", []Finding{missing, behind}, nil, []string{"bus.EVENTS.stream-missing.stream-wrong"}, VerdictFail}, + {"seen twice: written", []Finding{missing, behind}, nil, + []string{"bus.EVENTS.stream-missing.stream-wrong", "bus.EVENTS=2econtroller." + kindDataUnmeasured}, VerdictFail}, {"blind once: clears nothing, held", nil, errors.New("timed out"), - []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing"}, VerdictFailedToRun, 1}, + []string{"bus.EVENTS.stream-missing.stream-wrong", "bus.EVENTS=2econtroller." + kindDataUnmeasured}, VerdictFailedToRun}, {"blind twice: its own finding", nil, errors.New("timed out"), - []string{"bus.EVENTS.controller.behind", "bus.EVENTS.stream-missing", "probe.P1.failed"}, VerdictFailedToRun, 1}, - {"no longer seen: cleared", nil, nil, nil, VerdictPass, 1}, + []string{"bus.EVENTS.stream-missing.stream-wrong", "bus.EVENTS=2econtroller." + kindDataUnmeasured, "probe.P1.probe-failed"}, VerdictFailedToRun}, + {"no longer seen: cleared", nil, nil, nil, VerdictPass}, } - for _, s := range steps { + for i, s := range steps { found, fail = s.found, s.fail run := c.RunOnce(context.Background(), "the test") if got := store.findingKeys(); !slices.Equal(got, s.want) { @@ -157,11 +185,12 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { if run.Probes[1].Verdict != VerdictDeferred || !strings.Contains(run.Probes[1].Error, "overlay") { t.Errorf("%s: a deferred probe answered %+v", s.name, run.Probes[1]) } - if n := store.puts[FindingsState+"/bus.EVENTS.stream-missing"]; n != s.missingPs { - t.Errorf("%s: the unchanged finding was written %d times", s.name, n) + // The writer says one `findings-stated` a run, on the subject it forms itself (design 47 G A-13). + if n := len(store.events); n != i+1 || store.events[i] != "mesh.stream.mesh-self-check.event.findings-stated.novox" { + t.Errorf("%s: the writer said %v", s.name, store.events) } } - raw, _, _ := store.Get(FindingsState, "probe.P1.failed") + raw, _, _ := store.Get(FindingsState, "probe.P1.probe-failed") if raw != nil { t.Errorf("the blind finding outlived the probe running again: %s", raw) } @@ -189,12 +218,12 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { t.Errorf("the last verdict counts %+v", r.Counts) } // A finding carries the operator's words of its kind, the probe that found it and who observed it. - found, fail = []meshconditionsclient.Observation{missing}, nil + found, fail = []Finding{missing}, nil c.RunOnce(context.Background(), "the test") - raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing") + raw, _, _ = store.Get(FindingsState, "bus.EVENTS.stream-missing.stream-wrong") f, err := decodeFinding(raw) if err != nil || f.Kind != "stream-wrong" || f.Source != "P1" || f.Observer != Module || f.Headline == "" || - f.Explanation == "" || f.Ref != "stream-missing" || len(f.Evidence) != 1 || f.Run == "" { + f.Explanation == "" || f.ID != "EVENTS.stream-missing" || f.Subject.ID != f.ID || f.Subject.Scope != f.Scope || len(f.Evidence) != 1 || f.Run == "" || f.Since.IsZero() { t.Errorf("the written finding is %+v", f) } // Blind to its own bucket, a run writes and clears nothing. @@ -202,7 +231,7 @@ func TestARunWritesFindingsAsTheyChange(t *testing.T) { found = nil c.RunOnce(context.Background(), "the test") store.keysErr = nil - if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing"}) { + if got := store.findingKeys(); !slices.Equal(got, []string{"bus.EVENTS.stream-missing.stream-wrong"}) { t.Errorf("a run blind to its findings changed them: %v", got) } } @@ -212,9 +241,9 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { within := Within Within = 50 * time.Millisecond t.Cleanup(func() { Within = within }) - for name, run := range map[string]func(context.Context, *Checker) ([]meshconditionsclient.Observation, error){ - "panic": func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { panic("boom") }, - "overrun": func(ctx context.Context, _ *Checker) ([]meshconditionsclient.Observation, error) { + for name, run := range map[string]func(context.Context, *Checker) ([]Finding, error){ + "panic": func(context.Context, *Checker) ([]Finding, error) { panic("boom") }, + "overrun": func(ctx context.Context, _ *Checker) ([]Finding, error) { time.Sleep(time.Second) return nil, nil }, @@ -232,12 +261,12 @@ func TestAProbeThatPanicsOrOverrunsCouldNotRun(t *testing.T) { // every state the module keeps (ADR 0564 §12). func TestTheVerbsAndReads(t *testing.T) { store := newStore() - many := make([]meshconditionsclient.Observation, 25) + many := make([]Finding, 25) for i := range many { - many[i] = meshconditionsclient.Observation{Severity: meshconditionsclient.Warning, Scope: meshconditionsclient.ScopeBus, ID: fmt.Sprintf("S%02d", i), Summary: "x"} + many[i] = finding(meshconditionsclient.ScopeBus, fmt.Sprintf("S%02d", i), "", "", Finding{Severity: meshconditionsclient.Warning, Summary: "x"}) } c, _ := checker(store, Probe{ID: "D7", Kind: "stream-wrong", - Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { return many, nil }}) + Run: func(context.Context, *Checker) ([]Finding, error) { return many, nil }}) if _, err := c.Doctor(false, "", "x"); err == nil || !strings.Contains(err.Error(), "first run") { t.Fatalf("before a run: %v", err) } @@ -283,11 +312,11 @@ func TestTheVerbsAndReads(t *testing.T) { // and is reported by the verb as off by setting; the default list is D13 and D-fill (the operator, 2026-10-30). func TestOnlyTheProbesTheSettingListsRun(t *testing.T) { ran := map[string]int{} - finds := meshconditionsclient.Observation{Ref: "stream-missing", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeBus, ID: "EVENTS", Summary: "gone"} + finds := finding(meshconditionsclient.ScopeBus, "EVENTS", "stream-missing", "", Finding{Severity: meshconditionsclient.Urgent, Summary: "gone"}) probe := func(id string) Probe { - return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]meshconditionsclient.Observation, error) { + return Probe{ID: id, Kind: "stream-wrong", Run: func(context.Context, *Checker) ([]Finding, error) { ran[id]++ - return []meshconditionsclient.Observation{finds}, nil + return []Finding{finds}, nil }} } store := newStore() @@ -325,8 +354,8 @@ func TestOnlyTheProbesTheSettingListsRun(t *testing.T) { } // decodeFinding reads a finding as the seat mesh-conditions lays it out. -func decodeFinding(raw []byte) (meshconditions.FindingsByKeyValue, error) { - var f meshconditions.FindingsByKeyValue +func decodeFinding(raw []byte) (Finding, error) { + var f Finding err := json.Unmarshal(raw, &f) return f, err } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go index 73c1b96e5..fcb372384 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/whole_integration_test.go @@ -157,6 +157,9 @@ func (p *player) handle(m wire) { err = kv.Delete(p.ctx, a.Key) } p.answer(m.ID, map[string]any{}, err) + case "mesh/subscribe": + // The finding writer's consumer of the asks for findings (design 47 G A-16); no ask comes in this test. + p.answer(m.ID, map[string]any{}, nil) case "mesh/seat.publish": var e struct { Subject string `json:"subject"` @@ -252,6 +255,10 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { if err != nil { t.Fatal(err) } + statements, err := nc.SubscribeSync("mesh.mod." + Module + ".event.findings-stated") + if err != nil { + t.Fatal(err) + } cmd := exec.CommandContext(ctx, bin) cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) stdin, _ := cmd.StdinPipe() @@ -282,16 +289,26 @@ func TestTheSelfCheckWholeRunsOnTheBus(t *testing.T) { verdicts["D2"] != VerdictDeferred || answer.Run.Controller != "ace" || !strings.Contains(answer.Run.Why, "person.jochen") { t.Errorf("the run answered %+v", answer.Run) } - msg, err := checked.NextMsg(10 * time.Second) - if err != nil { - t.Fatalf("checked was not said: %v", err) - } + // The writer's statement at the start says checked too: the run of the person's ask is the one this waits for. var said Run - if err := json.Unmarshal(msg.Data, &said); err != nil || said.Run != answer.Run.Run { - t.Errorf("checked said %s", msg.Data) + for said.Run != answer.Run.Run { + msg, err := checked.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("checked of %s was not said: %v", answer.Run.Run, err) + } + if err := json.Unmarshal(msg.Data, &said); err != nil { + t.Fatalf("checked said %s", msg.Data) + } + } + for stated := ""; !strings.Contains(stated, `"id":"EVENTS.stream-missing"`); { + msg, err := statements.NextMsg(10 * time.Second) + if err != nil { + t.Fatalf("the writer did not say findings-stated with the missing stream (last %s): %v", stated, err) + } + stated = string(msg.Data) } findings, _ := js.KeyValue(ctx, Module+"_"+FindingsState) - e, err := findings.Get(ctx, "bus.EVENTS.stream-missing") + e, err := findings.Get(ctx, "bus.EVENTS.stream-missing.stream-wrong") if err != nil { t.Fatalf("the missing stream is not a finding on the bus: %v", err) } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/words.go b/modules/mesh-self-check/cmd/mesh-self-check/words.go index dcf0d904d..21ab3f845 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/words.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/words.go @@ -1,12 +1,10 @@ package main -// The operator's words for the kinds the self-check raises; the observation and its finding are the conditions -// client's (novox/hq ADR 0564 §2, ADR 0634 §4). - -import "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" +// The operator's words for the kinds the self-check raises, written into a finding's headline and explanation +// (novox/hq ADR 0564 §2; design 47 G §689). // worded gives a finding the operator's words registered for its kind, where it says none of its own. -func worded(f meshconditionsclient.Observation) meshconditionsclient.Observation { +func worded(f Finding) Finding { w, ok := words[f.Kind] if !ok { return f @@ -17,46 +15,38 @@ func worded(f meshconditionsclient.Observation) meshconditionsclient.Observation if f.Explanation == "" { f.Explanation = w.Explanation } - if f.Resolved == "" { - f.Resolved = w.Resolved - } if f.Needs == "" { f.Needs = w.Needs } return f } -type plain struct{ Headline, Explanation, Resolved, Needs string } +type plain struct{ Headline, Explanation, Needs string } // words are the controller's words for each kind the self-check raises (mesh-controller plain_words.go), the // controller's name for itself replaced by the self-check's where the words spoke of it. var words = map[string]plain{ "stream-wrong": {Headline: "Part of the bus's storage is wrong", Needs: "check the machine the bus runs on; the details say what is missing.", - Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost.", - Resolved: "The bus's storage is right again"}, + Explanation: "A store the mesh keeps on the bus is missing or not as defined, so what it keeps there may be lost."}, "bus-reconnects": {Headline: "A client keeps losing the bus", Explanation: "One of the mesh's clients lost its connection to the bus again and again in the last hour. While " + - "it reconnects, what it says and what it is asked waits.", - Resolved: "Resolved: the client stays connected"}, + "it reconnects, what it says and what it is asked waits."}, "core-unhealthy": {Headline: "The bus is not healthy", Needs: "check the machine the bus runs on; the details say what fails.", Explanation: "The bus, which carries every message in the mesh, is not healthy. Instructions, builds and tool " + - "calls may fail until it recovers.", - Resolved: "The bus is healthy again"}, + "calls may fail until it recovers."}, "watchdogs-silent": {Headline: "The mesh's watchdogs stopped", Needs: "restart the watchman if this stays, from the mesh MCP server.", Explanation: "The watchdogs, which notice when something expected does not happen, do not answer, so missed " + - "signals are not noticed.", - Resolved: "The watchdogs run again"}, + "signals are not noticed."}, "probe-failed": {Headline: "A check of the mesh could not run", Explanation: "One of the mesh's own checks could not run, so what it watches is unknown for now. It runs again " + - "every few minutes.", - Resolved: "The check runs again"}, + "every few minutes."}, } // confirm says one look can be wrong about the finding: it is written only when two runs in a row see it (issue 277). // The kinds are the self-check's own: a probe that could not run, and a holder that measured nothing of the data. -func confirm(o meshconditionsclient.Observation) bool { +func confirm(o Finding) bool { return o.Kind == "probe-failed" || o.Kind == kindDataUnmeasured } -- 2.54.0 From 11d1be3ef81efce61e68d10fd30089545fabce86 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:48:19 +0100 Subject: [PATCH 086/243] GATE A-18: the gate's record type drops the remove operation, and table, recheck and close go with their handlers and tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §587, §305, A-18 (G-20) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/configheld.go | 4 +- .../cmd/mesh-delivery-gate/holder_test.go | 13 +----- .../cmd/mesh-delivery-gate/main.go | 29 ------------ .../cmd/mesh-delivery-gate/manifest_test.go | 4 +- .../cmd/mesh-delivery-gate/onestatus_test.go | 15 ------- .../cmd/mesh-delivery-gate/record.go | 3 +- .../cmd/mesh-delivery-gate/record_test.go | 6 ++- .../cmd/mesh-delivery-gate/table.go | 30 ------------- .../cmd/mesh-delivery-gate/verbs.go | 45 ------------------- modules/mesh-delivery-gate/module.json | 13 ++---- 10 files changed, 13 insertions(+), 149 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/configheld.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/configheld.go index 8b457a63e..698811892 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/configheld.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/configheld.go @@ -87,9 +87,7 @@ func (h *Holder) askAgainForConfig(id, why string) { h.mu.Unlock() switch { case refused == nil: - if _, err := h.Recheck(id, why, byOwner); err != nil { - h.Logf("[mesh-delivery] %s: its check could not be asked again for a config value: %v", id, err) - } + h.Logf("[mesh-delivery] %s: its check could not be asked again for a config value: %v", id, checkerRefusal(id)) case proposed: h.askProposedAgain(id, why) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go index c369104d7..9ad6342c5 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go @@ -329,15 +329,12 @@ func (f *fakeLinks) Linked(repository string, number int) ([]int, error) { return out, nil } -// Stalled: a delivery past its state's bound is listed with what H2 may do; close takes only that. Only a proposed +// Stalled: a delivery past its state's bound is listed with what H2 may do. Only a proposed // delivery has a bound now: its check was asked and not answered. -func TestStalledAndCloseWorkFromTheTable(t *testing.T) { +func TestStalledWorksFromTheTable(t *testing.T) { w := newWorld(t) id := IDOf("novox/app", head) w.h.PullUpdated(pr("novox/app", 7, head, "feat/x", "src/a.go")) - if _, err := w.h.Close(id, "too early"); err == nil { - t.Fatal("closed within its bound") - } w.later(3 * time.Hour) stalled := w.h.Stalled() if len(stalled) != 1 || stalled[0].ID != id || stalled[0].State != Proposed { @@ -352,16 +349,10 @@ func TestStalledAndCloseWorkFromTheTable(t *testing.T) { t.Fatalf("a merged delivery is stalled: %+v", s) } } - if _, err := w.h.Close(id, "x"); err == nil { - t.Fatal("H2 closed a merged delivery") - } // A merge without a passing check is held: a person's, with no transition for H2. w.h.PullUpdated(pr("novox/app", 8, "ffffffffffff0000", "feat/q")) w.h.PullMerged(merged("novox/app", 8, "ffffffffffff0000", "eeeeeeeeeeee0000")) w.later(48 * time.Hour) - if _, err := w.h.Close(IDOf("novox/app", "ffffffffffff0000"), "x"); err == nil { - t.Fatal("H2 closed a held delivery") - } } // novox/hq issue 867, ADR 0382 §1 (contract A): each member of a group in `groups` says its pull request number, full diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 5b73dbf71..694da81e7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -298,9 +298,6 @@ func boolArg(a map[string]any, k string) bool { return false } -// aPerson is who an act through the seat is by: the seat does not name its caller. -const aPerson = "a person, through the mesh-delivery seat" - func need(a map[string]any, keys ...string) error { for _, k := range keys { if strArg(a, k) == "" { @@ -367,36 +364,10 @@ func tools(h *Holder, l *listening) []stdio.Tool { } return h.Checks(strArg(a, "repository"), number, strArg(a, "commit")) }}, - {Name: seat + "table", - Description: "The state table every delivery runs by: each transition with its guard, each state's bound " + - "and what healer H2 may do once it has passed; ", - Run: func(map[string]any) (any, error) { return TableText(), nil }}, {Name: seat + "stalled", Description: "Every delivery held past its state's bound, with the transition the table lets healer H2 take. " + "Also a merge refused or not made past its bound, a group half merged, and a head with no merge check.", Run: func(a map[string]any) (any, error) { return h.Stalled(), nil }}, - {Name: seat + "recheck", - Description: "Refused for every delivery: the checker checks a head again on every move of the head, its trunk or its " + - "group, and nothing is asked of anyone here (novox/hq ADR 0447, issue 1318). Kept so a caller is told where to read.", - Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), - "why": str("why, kept with the transition")}, "required": []string{"id", "why"}}, - Run: func(a map[string]any) (any, error) { - if err := need(a, "id", "why"); err != nil { - return nil, err - } - return h.Recheck(strArg(a, "id"), strArg(a, "why"), aPerson) - }}, - {Name: seat + "close", - Description: "Healer H2's verb: for a proposed delivery past its bound, take the transition the table " + - "names for that state. Refused for any other.", - Input: map[string]any{"type": "object", "properties": map[string]any{"id": str("the delivery's id"), - "why": str("the condition it answers")}, "required": []string{"id", "why"}}, - Run: func(a map[string]any) (any, error) { - if err := need(a, "id", "why"); err != nil { - return nil, err - } - return h.Close(strArg(a, "id"), strArg(a, "why")) - }}, {Name: "delivery_merge_again", Description: "On the operator's word, ask again at once the merge of an approved delivery the forge refused, or of an approved " + "group held or half merged, at the heads the Approve named (novox/hq issue 1275): every member is read first and " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 2a340989e..6a6f502a7 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -77,7 +77,7 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { } // The verbs the seat keeps: times, what-if, release, stop and retire-history went with the plans (ADR 0539 §17), and // settle with the merge states (design 47 G §431). - if want := []string{"deliveries", "show", "groups", "source", "checks", "table", "stalled", "recheck", "close"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { + if want := []string{"deliveries", "show", "groups", "source", "checks", "stalled"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want) } // The forge messages' layout is the bridge's published package forgemsg, taken by its series (ADR 0634 §3, design 32). @@ -293,7 +293,7 @@ func TestTheGateHoldsTheSeatAsTheSDKsTableOfRelease6HasIt(t *testing.T) { t.Errorf("the manifest serves %s, which G removed with the merge states", removed) } } - for _, verb := range []string{"deliveries", "show", "groups", "checks", "table", "stalled", "recheck", "close"} { + for _, verb := range []string{"deliveries", "show", "groups", "checks", "stalled"} { if !served[verb] { t.Errorf("the gate does not serve %s, which the seat keeps", verb) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_test.go index f1d40f348..9d4378eea 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_test.go @@ -177,21 +177,6 @@ func TestAFailingMeshCheckRejects(t *testing.T) { } } -// 1302 (lead's (2)): the recheck verb refuses a delivery on a mesh/check base, saying the checker checks again on every -// move and where to read it. -func TestRecheckRefusesAMeshCheckDelivery(t *testing.T) { - w := newWorld(t) - p := pr(oneRepo, 1, oneHead, "feat/one") - requiresCheck(w, p) - w.h.PullUpdated(p) - w.h.CommitStatus(checkStatus(oneRepo, oneHead, "failure", oneTrunk)) - w.settleAll() - _, err := w.h.Recheck(IDOf(oneRepo, oneHead), "try again", "tester") - if err == nil || !strings.Contains(err.Error(), "checker_pull") || !strings.Contains(err.Error(), "checker") { - t.Fatalf("recheck answered %v, want a refusal naming the checker and checker_pull", err) - } -} - // 1318 (the gate's part): a pull request closed unmerged asks the controller to forget nothing: forget-checks is not called. func TestAClosedPullRequestAsksNoForgetChecks(t *testing.T) { w := newWorld(t) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 1d5123b51..27e998d4e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -55,13 +55,12 @@ const ( OpFree = "free" // remove from pinned, by revision OpEnd = "end" // set-fields: ended OpAct = "act" // set-entry into acts, by id: an act owed the forge with its outcome - OpRemove = "remove" // remove the instance, at its retention OpHold = "hold" // a month's list: set-entry into records, by instance OpDrop = "drop" // a month's list: remove from records, by instance ) // RecordOperations are a record's operations, in the order the manifest's type allows them. -var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpAct, OpEnd, OpRemove} +var RecordOperations = []string{OpOpen, OpGroup, OpTrace, OpEntry, OpOutcome, OpAway, OpBack, OpPin, OpFree, OpAct, OpEnd} // HeldOperations are the month's list's operations. var HeldOperations = []string{OpHold, OpDrop} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index d1a7f09e4..066cb1d55 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -146,8 +146,10 @@ func (f *ledgerFake) Apply(instance, operation string, body any) (uint64, error) } func (f *ledgerFake) Remove(instance string) error { - _, err := f.Apply(instance, OpRemove, map[string]any{}) - return err + f.seq[instance]++ + f.states[instance] = json.RawMessage("null") + f.history = append(f.history, instance+" remove") + return nil } func (f *ledgerFake) Get(instance string) (json.RawMessage, uint64, error) { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go index e79102a52..dadf6b98c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go @@ -381,33 +381,3 @@ func pastBound(d *Delivery, now time.Time) bool { b, bounded := boundOf(d) return bounded && !d.State.Final() && (b.For == 0 || now.Sub(boundSince(d)) > b.For) } - -// TableText is the table as the `table` verb answers it. -func TableText() map[string]any { - var rows []map[string]any - for _, r := range Table { - from := make([]string, 0, len(r.From)) - for _, s := range r.From { - if s == None { - from = append(from, "(none)") - continue - } - from = append(from, string(s)) - } - kind := "observed" - if r.Act { - kind = "asked" - } - rows = append(rows, map[string]any{"from": strings.Join(from, ", "), "event": r.Event, "to": r.To, - "guard": r.Guard, "taken": kind}) - } - bounds := map[string]any{} - for s, b := range Bounds { - var h2 []string - for _, e := range b.H2 { - h2 = append(h2, string(e)) - } - bounds[string(s)] = map[string]any{"bound": b.For.String(), "h2": h2, "says": b.Says} - } - return map[string]any{"transitions": rows, "bounds": bounds} -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go index 7c0ea4fe6..ccc8d919e 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/verbs.go @@ -252,51 +252,6 @@ func (h *Holder) Stalled() []StalledLine { return out } -// Recheck is the `recheck` verb. The checker judges every head and checks it again on every move of its trunk (ADR 0447 §4), and -// the controller judges none (ADR 0524 §5), so there is no ask to make: a delivery that exists is refused, naming the checker -// (the lead's decision for issue 1302), and an unknown id is refused as before. -func (h *Holder) Recheck(id, why, by string) (string, error) { - h.mu.Lock() - defer h.mu.Unlock() - if h.deliveries[id] == nil { - return "", fmt.Errorf("no delivery %q", id) - } - return "", checkerRefusal(id) -} - -// Close is healer H2's verb: for a proposed delivery past its bound, its check is asked once more; only a transition -// the table names for that state counts, any other answer is refused. -func (h *Holder) Close(id, why string) (string, error) { - h.mu.Lock() - d := h.deliveries[id] - if d == nil { - h.mu.Unlock() - return "", fmt.Errorf("no delivery %q", id) - } - b, bounded := boundOf(d) - if !bounded || len(b.H2) == 0 { - h.mu.Unlock() - return "", fmt.Errorf("%s is %s: H2 has no transition there — the state is the operator's", id, d.State) - } - if !pastBound(d, h.Now()) { - h.mu.Unlock() - return "", fmt.Errorf("%s has been %s for %s, within its bound of %s: nothing to close", id, d.State, - h.Now().Sub(boundSince(d)).Round(time.Second), b.For) - } - if d.State == Proposed { - return h.reask(d, why) // unlocks - } - h.mu.Unlock() - return "", fmt.Errorf("%s is %s: H2 has no transition there", id, d.State) -} - -// reask is H2's close of a proposed delivery past its bound. A head's check is the checker's, which asks nothing of anyone -// and checks again on its own (ADR 0447 §4), so there is no check to ask once more. Called under the lock; it unlocks. -func (h *Holder) reask(d *Delivery, why string) (string, error) { - h.mu.Unlock() - return "", checkerRefusal(d.ID) -} - // Status is the module's own `delivery_status`. func (h *Holder) Status(listening string) map[string]any { h.mu.Lock() diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 514b72be9..b015d3760 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -12,10 +12,7 @@ "groups", "source", "checks", - "table", - "stalled", - "recheck", - "close" + "stalled" ] } ], @@ -248,10 +245,6 @@ "ended" ], "sequence": true - }, - { - "name": "remove", - "effect": "remove" } ] }, @@ -343,7 +336,7 @@ "config": { "refusal-stall-floor-minutes": { "type": "number", - "description": "minutes, at least, before a merge refused because its trunk moved, whose check the gate asked again, is said as stalled; twice the repository's last check time when that is longer (ADR 0505 §4).", + "description": "minutes, at least, before a merge refused because its trunk moved, whose check the gate asked again, is said as stalled; twice the repository's last check time when that is longer (ADR 0505 \u00a74).", "minimum": 1, "maximum": 1440, "trusted": false, @@ -351,7 +344,7 @@ }, "author-new-head-minutes": { "type": "number", - "description": "minutes a delivery that needs its author's new head (a conflict, a failed check, a rejected group) waits for one before the operator hears of it (ADR 0505 §5).", + "description": "minutes a delivery that needs its author's new head (a conflict, a failed check, a rejected group) waits for one before the operator hears of it (ADR 0505 \u00a75).", "minimum": 1, "maximum": 10080, "trusted": false, -- 2.54.0 From b78e90c0c702c687934a60ffbe5fad389c1eabca Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:49:17 +0100 Subject: [PATCH 087/243] mesh-healers writes its findings through mesh-sdk's finding writer and the generated finding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §686, §689, A-3, A-13, A-14, A-16 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-healers/cmd/mesh-healers/findings.go | 96 +++++++++++++++++++ .../mesh-healers/cmd/mesh-healers/healers.go | 90 +++++++++-------- .../cmd/mesh-healers/healers_test.go | 47 ++++++++- modules/mesh-healers/cmd/mesh-healers/main.go | 12 +++ .../mesh-healers/cmd/mesh-healers/wanted.go | 50 ++++++---- .../mesh-healers/whole_integration_test.go | 10 ++ 6 files changed, 233 insertions(+), 72 deletions(-) create mode 100644 modules/mesh-healers/cmd/mesh-healers/findings.go diff --git a/modules/mesh-healers/cmd/mesh-healers/findings.go b/modules/mesh-healers/cmd/mesh-healers/findings.go new file mode 100644 index 000000000..4dbba1a07 --- /dev/null +++ b/modules/mesh-healers/cmd/mesh-healers/findings.go @@ -0,0 +1,96 @@ +package main + +// The healers' findings are the generated finding of the seat mesh-conditions, written through mesh-sdk's finding writer +// (design 47 G §461, §689). + +import ( + "context" + "encoding/json" + "fmt" + "sort" + "time" + + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// Finding, and the writer's other three bodies, are the seat mesh-conditions' own types. +type ( + Finding = meshconditions.Finding + FindingCleared = meshconditions.FindingCleared + FindingsStated = meshconditions.FindingsStated + FindingsAsked = meshconditions.FindingsAskedEvent + FindingsWriter = views.Writer[Finding, FindingCleared, FindingsStated, FindingsAsked] +) + +// finding is f about scope and id: it sets the key's scope and id and the subject's scope and id equal (design 47 G A-14). +func finding(scope, id string, f Finding) Finding { + f.Scope, f.ID = scope, keys.EscapeName(id) + f.Subject = meshconditions.Subject{Scope: scope, ID: f.ID} + return f +} + +// evidence is the one observation a finding keeps as evidence; its moment is the finding's own, so that a finding that says +// the same is not written again. +func evidence(s string) []meshconditions.Evidence { return []meshconditions.Evidence{{Said: s}} } + +// keyOf is where f is kept: `..`. +func keyOf(f Finding) string { return views.FindingKey(f.Scope, f.ID, f.Kind) } + +// PendingBound is how long a finding the healers cannot judge yet stays open (design 47 G §384). +const PendingBound = 5 * time.Minute + +// stored are the findings in the state `findings`, by key. +func (h *Healers) stored() (map[string]Finding, error) { + keys, err := h.Store.Keys(FindingsState) + if err != nil { + return nil, err + } + out := map[string]Finding{} + for _, k := range keys { + raw, found, err := h.Store.Get(FindingsState, k) + if err != nil { + return nil, err + } + if !found { + continue + } + var f Finding + if json.Unmarshal(raw, &f) == nil { + out[k] = f + } + } + return out, nil +} + +// Observe is the writer's observe (design 47 G §384): every finding the healers observe now, judged from what they +// keep, and the stored findings they cannot judge yet. +func (h *Healers) Observe(ctx context.Context) (views.Observed[Finding], error) { + stored, err := h.stored() + if err != nil { + return views.Observed[Finding]{}, fmt.Errorf("its own findings cannot be read: %w", err) + } + now := h.Now() + var out views.Observed[Finding] + pend := func(source string) { + for k, f := range stored { + if f.Source == source { + out.Pending = append(out.Pending, views.PendingKey{Key: k, Since: f.Since, Bound: PendingBound.String()}) + } + } + } + if all, err := h.attempts(); err != nil { + out.Findings = append(out.Findings, blindFinding(err)) + pend("brake") + } else if braked, said := h.brakeHolds(actsWithin(all, now.Add(-BrakeWindow)), now); braked { + out.Findings = append(out.Findings, brakeFinding(actsWithin(all, now.Add(-BrakeWindow)), said)) + } + if wanted, _, seeded := h.wantedNow(now); seeded { + out.Findings = append(out.Findings, wanted...) + } else { + pend("S15") + } + sort.Slice(out.Pending, func(i, j int) bool { return out.Pending[i].Key < out.Pending[j].Key }) + return out, nil +} diff --git a/modules/mesh-healers/cmd/mesh-healers/healers.go b/modules/mesh-healers/cmd/mesh-healers/healers.go index 92138a3dd..f1ad1caf6 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers.go @@ -14,6 +14,7 @@ package main // bucket's age); nothing writes there any more. import ( + "context" "encoding/json" "errors" "fmt" @@ -29,6 +30,7 @@ import ( handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts" meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" ) // Module and Seat are this module and the seat it holds once for the mesh. @@ -181,12 +183,19 @@ type Healers struct { Now func() time.Time Logf func(format string, args ...any) Registry []Healer + // Writer is the finding writer over this module's state `findings` (views.NewStdioWriter; design 47 G A-16), whose + // observe is Observe. + Writer *FindingsWriter mu sync.Mutex // declined is why each condition was last passed over, said once. declined map[string]string - // acts are the hand-act log as watched, for S15. - acts map[string]HandAct + // actsMu guards acts and seeded, which the writer's observe reads while it holds the writer's lock; it is never held + // while a finding is written. + actsMu sync.Mutex + // acts are the hand-act log as watched, for S15; seeded is whether the seat's answer at the start was taken. + acts map[string]HandAct + seeded bool } // actingWhy is why the healers act: what the tool `healers` says. @@ -240,7 +249,7 @@ func (h *Healers) Judge(c Condition, by string) (string, error) { h.see("blind", nil) acts := actsWithin(all, now.Add(-BrakeWindow)) if braked, said := h.brakeHolds(acts, now); braked { - h.see("brake", []meshconditionsclient.Observation{brakeFinding(acts, said)}) + h.see("brake", []Finding{brakeFinding(acts, said)}) return "every healer has stopped: " + said, nil } h.see("brake", nil) @@ -483,10 +492,11 @@ func (h *Healers) brakeHolds(acts []Attempt, now time.Time) (bool, string) { return false, said } -var brakeKey = meshconditionsclient.Key(meshconditionsclient.ScopeMesh, "healers", "braked") +// brakeKey is where the brake's finding is kept. +var brakeKey = views.FindingKey(meshconditionsclient.ScopeMesh, "healers", "healers-braked") // brakeFinding is the brake, as the urgent finding that says it. -func brakeFinding(acts []Attempt, said string) meshconditionsclient.Observation { +func brakeFinding(acts []Attempt, said string) Finding { counts := map[string]int{} for _, a := range acts { counts[a.Healer+" on "+a.Condition]++ @@ -496,67 +506,53 @@ func brakeFinding(acts []Attempt, said string) meshconditionsclient.Observation most = append(most, fmt.Sprintf("%s ×%d", what, n)) } sort.Strings(most) - return meshconditionsclient.Observation{Kind: "healers-braked", Ref: "braked", Severity: meshconditionsclient.Urgent, Scope: meshconditionsclient.ScopeMesh, ID: "healers", + return finding(meshconditionsclient.ScopeMesh, "healers", Finding{Kind: "healers-braked", Severity: meshconditionsclient.Urgent, + Source: "brake", Summary: "every healer has stopped: the healers acted more often in an hour than the mesh allows, which is a " + "healer looping, not repairing — " + said, - Said: strings.Join(most, ", "), + Evidence: evidence(strings.Join(most, ", ")), Headline: "All automatic repairs stopped", Needs: "read the details and decide whether the repairs may run again.", Explanation: "The healers (the mesh's automatic repairs) acted more often in an hour than allowed, which looks " + - "like a loop, so all of them stopped.", - Resolved: "Automatic repairs run again"} + "like a loop, so all of them stopped."}) +} + +// blindFinding is the healers' own blindness: they cannot read what they did. +func blindFinding(err error) Finding { + return finding(meshconditionsclient.ScopeProbe, "healers", Finding{Kind: "probe-failed", Severity: meshconditionsclient.Warning, + Source: "blind", + Summary: "the healers cannot read what they did, so they count no budget and act on nothing", + Evidence: evidence(firstLine(err.Error())), Headline: "A check of the mesh could not run", + Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can."}) } // blind writes the healers' own blindness: they cannot read what they did. -func (h *Healers) blind(err error) { - h.see("blind", []meshconditionsclient.Observation{{Kind: "probe-failed", Ref: "failed", Severity: meshconditionsclient.Warning, - Scope: meshconditionsclient.ScopeProbe, ID: "healers", - Summary: "the healers cannot read what they did, so they count no budget and act on nothing", - Said: firstLine(err.Error()), Headline: "A check of the mesh could not run", - Explanation: "The automatic repairs cannot read what they did, so they act on nothing until they can.", - Resolved: "The automatic repairs read what they did again"}}) -} +func (h *Healers) blind(err error) { h.see("blind", []Finding{blindFinding(err)}) } -// see writes the healers' own findings of one source, each only when it changes, and deletes those of the source it -// no longer sees (ADR 0564 §2). -func (h *Healers) see(source string, found []meshconditionsclient.Observation) { - now := h.Now() - want := map[string]meshconditions.FindingsByKeyValue{} - for _, o := range found { - o.Source = source - want[o.Key()] = o.Finding(Module, "", now) +// see writes the healers' own findings of one source through the finding writer, each only when it changes, and clears +// those of the source it no longer sees (ADR 0564 §2; design 47 G §461). +func (h *Healers) see(source string, found []Finding) { + ctx := context.Background() + want := map[string]Finding{} + for _, f := range found { + f.Source = source + want[keyOf(f)] = f } - keys, err := h.Store.Keys(FindingsState) + stored, err := h.stored() if err != nil { h.Logf("[%s] its own findings cannot be read, so none is written or cleared: %v", Module, err) return } - for _, k := range keys { - raw, found, err := h.Store.Get(FindingsState, k) - if err != nil || !found { + for k, was := range stored { + if _, still := want[k]; still || was.Source != source { continue } - var was meshconditions.FindingsByKeyValue - err = json.Unmarshal(raw, &was) - if err != nil || was.Source != source { - continue - } - if f, still := want[k]; still { - if meshconditionsclient.Says(was) == meshconditionsclient.Says(f) { - delete(want, k) - } - continue - } - if err := h.Store.Delete(FindingsState, k); err != nil { - h.Logf("[%s] the finding %s no longer seen is not deleted: %v", Module, k, err) + if err := h.Writer.Clear(ctx, was.Scope, was.ID, was.Kind, "the healers no longer see it"); err != nil { + h.Logf("[%s] the finding %s no longer seen is not cleared: %v", Module, k, err) } } for k, f := range want { - raw, err := json.Marshal(f) - if err == nil { - err = h.Store.Put(FindingsState, k, json.RawMessage(raw)) - } - if err != nil { + if err := h.Writer.Write(ctx, f); err != nil { h.Logf("[%s] the finding %s is not written: %v", Module, k, err) } } diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index 1959fb3f0..6bb3d37c1 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -15,6 +16,8 @@ import ( meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" handacts "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts" + stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" ) type memStore struct { @@ -131,6 +134,8 @@ type world struct { now time.Time asked []string said []Acted + // events are the subjects the finding writer said its events on. + events []string } func newWorld() *world { @@ -155,8 +160,35 @@ func (w *world) fresh() { return nil }, } + wr, err := views.NewWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ + Module: Module, Node: "ace", State: findingState{w.store}, Observe: w.h.Observe, Now: func() time.Time { return w.now }, + Publish: func(_ context.Context, subject string, _ []byte, _ string) error { + w.events = append(w.events, subject) + return nil + }, + Listen: func(context.Context, string, func([]byte)) (func(), error) { return func() {}, nil }}) + if err != nil { + panic(err) + } + w.h.Writer = wr } +// findingState is the writer's view of the store's state `findings`. +type findingState struct{ s *memStore } + +func (f findingState) Get(key string) (*stdio.StateEntry, error) { + raw, ok, err := f.s.Get(FindingsState, key) + if err != nil || !ok { + return nil, err + } + return &stdio.StateEntry{Key: key, Value: raw}, nil +} +func (f findingState) Put(key string, value any) (uint64, error) { + return 0, f.s.Put(FindingsState, key, value) +} +func (f findingState) Delete(key string) error { return f.s.Delete(FindingsState, key) } +func (f findingState) Keys() ([]string, error) { return f.s.Keys(FindingsState) } + // drill is a healer of the tests alone: no healer acts here since H3 ended with the controller (issue 1126), and the // budget, the settle, the escalation and the brake are every healer's, judged on it. var drill = Healer{ID: "HD", Kinds: []string{"drill-broken"}, Condition: "a drill", Repair: "ask the drill again", @@ -288,11 +320,18 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { t.Errorf("past the brake: %q, asked %v", said, b.asked) } raw, braked, _ := b.store.Get(FindingsState, brakeKey) - var f meshconditions.FindingsByKeyValue + var f Finding _ = json.Unmarshal(raw, &f) - if !braked || f.Severity != meshconditionsclient.Urgent || f.Kind != "healers-braked" || f.Observer != Module { + if !braked || f.Severity != meshconditionsclient.Urgent || f.Kind != "healers-braked" || f.Observer != Module || f.Since.IsZero() { t.Errorf("the brake is written as %s", raw) } + if !slices.Contains(b.events, "mesh.stream.mesh-healers.event.finding-written.ace") { + t.Errorf("the brake was not said as finding-written on the writer's subject: %v", b.events) + } + // Judged anew at a start, from what the healers keep: the brake still holds, and states the same finding. + if obs, err := b.h.Observe(context.Background()); err != nil || len(obs.Findings) != 1 || obs.Findings[0].Kind != "healers-braked" { + t.Errorf("a statement of the brake: %+v, %v", obs, err) + } b.now = b.now.Add(55 * time.Minute) // fewer than twelve within the hour, but the brake is said: held if said, _ := b.h.Judge(lost("mail"), ""); !strings.Contains(said, "held until") { t.Errorf("the brake let go before an hour after the last act: %q", said) @@ -310,7 +349,7 @@ func TestBudgetSettleEscalationAndBrake(t *testing.T) { if _, err := z.h.Judge(lost("mail"), ""); err == nil || len(z.asked) != 0 { t.Errorf("blind, the healers acted: %v", err) } - if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.failed"}) { + if keys := z.store.keys(FindingsState); !slices.Equal(keys, []string{"probe.healers.probe-failed"}) { t.Errorf("blind, the findings are %v", keys) } } @@ -381,7 +420,7 @@ func TestACauseRepairedByHandTwiceIsAHealerWanted(t *testing.T) { "mesh-vault.secret_vault_export"} { act(fmt.Sprintf("n%d", i), verb, "operator", time.Hour) } - want := []string{"mesh.hand-acts.consumer-behind.healer-wanted"} + want := []string{"mesh.hand-acts=2econsumer-behind.healer-wanted"} if keys := w.store.keys(FindingsState); !slices.Equal(keys, want) { t.Fatalf("findings %v, want %v", keys, want) } diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 07c2980dc..969f0a6fe 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -7,6 +7,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -14,6 +15,7 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" ) func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } @@ -55,6 +57,13 @@ func main() { h := &Healers{Store: runtimeStore{}, Registry: Registry(), Now: time.Now, Logf: logf, Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, Publish: stdio.SeatPublish} + w, err := views.NewStdioWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ + Module: Module, State: stdio.State(FindingsState), Observe: h.Observe, Logf: logf}) + if err != nil { + logf("%v", err) + os.Exit(1) + } + h.Writer = w // At start, a settle that ended while no instance watched is judged (Rearm); the waits still running are keys on // the bus, heard on their expiry. go h.Rearm() @@ -66,6 +75,9 @@ func main() { // durable consumer (the controller's bucket is no longer read, issue 774). go func() { <-stdio.Served() + if err := h.Writer.Start(context.Background()); err != nil { + logf("[%s] the finding writer did not start: %v", Module, err) + } for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { answer, err := stdio.Ask("seat:mesh-hand-acts.hand-acts", map[string]any{"days": int(Within.Hours() / 24)}) if err == nil { diff --git a/modules/mesh-healers/cmd/mesh-healers/wanted.go b/modules/mesh-healers/cmd/mesh-healers/wanted.go index 0e3274a43..44a9b6b49 100644 --- a/modules/mesh-healers/cmd/mesh-healers/wanted.go +++ b/modules/mesh-healers/cmd/mesh-healers/wanted.go @@ -82,12 +82,12 @@ func (h *Healers) ActRecorded(body []byte) { if json.Unmarshal(body, &a) != nil || a.ID == "" { return } - h.mu.Lock() + h.actsMu.Lock() if h.acts == nil { h.acts = map[string]HandAct{} } h.acts[handacts.Seat+":"+a.ID] = a - h.mu.Unlock() + h.actsMu.Unlock() h.Wanted() } @@ -106,6 +106,9 @@ func (h *Healers) Seeded(answer []byte) (int, error) { for _, raw := range l.Acts { h.ActRecorded(raw) } + h.actsMu.Lock() + h.seeded = true + h.actsMu.Unlock() // A partial answer (the seat bounds it, issue 1966) seeds all the same: the newest acts are what a fortnight's // count needs most, and the older ones are heard no more, so asking again would only repeat it. if l.Complete != nil && !*l.Complete { @@ -115,11 +118,25 @@ func (h *Healers) Seeded(answer []byte) (int, error) { } // Wanted counts every cause repaired by hand within the fortnight and writes a finding for each counted twice or more, -// deleting the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. -func (h *Healers) Wanted() []meshconditionsclient.Observation { - h.mu.Lock() - defer h.mu.Unlock() +// clearing the rest; its wait `fortnight` expires when the oldest counted act of a wanted cause leaves the fortnight. +func (h *Healers) Wanted() []Finding { now := h.Now() + found, next, _ := h.wantedNow(now) + h.see("S15", found) + if !next.IsZero() { + ttl := next.Sub(now).Round(time.Second) + time.Second + if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil { + h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err) + } + } + return found +} + +// wantedNow is the finding of every cause repaired by hand twice or more within the fortnight, and when the oldest act +// of a wanted cause leaves it; seeded says the seat's acts of the start were taken, so that none is judged before. +func (h *Healers) wantedNow(now time.Time) (found []Finding, next time.Time, seeded bool) { + h.actsMu.Lock() + defer h.actsMu.Unlock() byCause := map[string][]HandAct{} for _, a := range h.acts { if personsDecision(a) || now.Sub(a.At) > Within { @@ -134,8 +151,6 @@ func (h *Healers) Wanted() []meshconditionsclient.Observation { } } sort.Strings(causes) - var found []meshconditionsclient.Observation - var next time.Time for _, cause := range causes { acts := byCause[cause] sort.Slice(acts, func(i, j int) bool { return acts[i].At.Before(acts[j].At) }) @@ -153,27 +168,20 @@ func (h *Healers) Wanted() []meshconditionsclient.Observation { } } words := strings.NewReplacer(".", " ", "_", " ").Replace(cause) - found = append(found, meshconditionsclient.Observation{Kind: "healer-wanted", Ref: "healer-wanted", Severity: meshconditionsclient.Warning, - Scope: meshconditionsclient.ScopeMesh, ID: "hand-acts." + cause, + f := finding(meshconditionsclient.ScopeMesh, "hand-acts."+cause, Finding{Kind: "healer-wanted", Severity: meshconditionsclient.Warning, + Source: "S15", Summary: fmt.Sprintf("%q was repaired by hand %d times in %d days, the last by %s: %s", cause, len(acts), int(Within.Hours()/24), newest.By, wanted), - Said: strings.Join(lines, "; "), + Evidence: evidence(strings.Join(lines, "; ")), Headline: capital(words) + " keeps being fixed by hand", Explanation: fmt.Sprintf("A person repaired %s by hand %d times in %d days, so an automatic repair is wanted "+ - "for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24)), - Resolved: "Resolved: no more hand repairs of " + words}) + "for it. Nothing is broken now.", words, len(acts), int(Within.Hours()/24))}) + found = append(found, f) if leaves := acts[0].At.Add(Within); next.IsZero() || leaves.Before(next) { next = leaves } } - h.see("S15", found) - if !next.IsZero() { - ttl := next.Sub(now).Round(time.Second) + time.Second - if err := h.Store.PutTTL(WakesState, fortnightKey, map[string]any{"at": next.UTC()}, ttl); err != nil { - h.Logf("[%s] the fortnight's end is not written as a wait, so it is counted at the next act: %v", Module, err) - } - } - return found + return found, next, h.seeded } func capital(s string) string { diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go index 926a30d76..58cf1869e 100644 --- a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -30,6 +30,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/testbus" + "git.novox.be/novox/mesh-sdk/go/views" ) type wire struct { @@ -360,6 +361,10 @@ func TestTheHealersWholeHearTheKeeperAndPassOverWhatEnded(t *testing.T) { if err != nil { t.Fatal(err) } + stated, err := nc.SubscribeSync("mesh.mod." + Module + ".event.findings-stated") + if err != nil { + t.Fatal(err) + } cmd := exec.CommandContext(ctx, bin) cmd.Env = append(os.Environ(), "MESH_NODE=ace", "MESH_MODULE="+Module) stdin, _ := cmd.StdinPipe() @@ -384,6 +389,11 @@ func TestTheHealersWholeHearTheKeeperAndPassOverWhatEnded(t *testing.T) { // The runtime's consumer holds an event until the bundle subscribes: delivered once it hears them. eventually(t, "the healers hearing the keeper's conditions", func() bool { return p.hears(RaisedEvent) && p.hears(ChangedEvent) }) + // The writer hears the keeper's asks for findings, and states at its start the findings it observes: none. + eventually(t, "the writer hearing findings-asked", func() bool { return p.hears(views.FindingsAskedEvent) }) + if msg, err := stated.NextMsg(10 * time.Second); err != nil || !strings.Contains(string(msg.Data), `"findings":[]`) { + t.Errorf("findings-stated at the start: %v", err) + } // What ended with the controller is passed over: no act begun, nothing asked, nothing said. p.deliver(RaisedEvent, raised("bus.EVENTS.records.missing", "records")) if msg, err := acted.NextMsg(3 * time.Second); err == nil { -- 2.54.0 From cbd4da3a116cfc2d5c383f74771082d6c8c42507 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:49:54 +0100 Subject: [PATCH 088/243] =?UTF-8?q?GATE=20=C2=A7613:=20deliveries,=20group?= =?UTF-8?q?s,=20owed=20and=20conditions=20are=20template=20state=20entries?= =?UTF-8?q?=20under=20the=20same=20names,=20taking=20the=20buckets=20they?= =?UTF-8?q?=20had?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit merges and source stay in the old state section; pulls and asks wait for the removal of the merge states. Builds: design 47 G §613, A-8, A-20 (G-17, G-20) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/keeps_test.go | 48 ++++++++++++++++++- modules/mesh-delivery-gate/module.json | 32 +++++++++---- 2 files changed, 71 insertions(+), 9 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps_test.go index e3ef8b05e..4694f8f7c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/keeps_test.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" ) @@ -30,7 +31,12 @@ func TestTheStatesTheModuleAnnouncesAreTheStatesItsManifestDeclares(t *testing.T t.Fatal(err) } var m struct { - State []json.RawMessage `json:"state"` + State []json.RawMessage `json:"state"` + Requires []struct { + Name string `json:"name"` + Storage string `json:"storage"` + Template string `json:"template"` + } `json:"requires"` } if err := json.Unmarshal(raw, &m); err != nil { t.Fatal(err) @@ -47,6 +53,12 @@ func TestTheStatesTheModuleAnnouncesAreTheStatesItsManifestDeclares(t *testing.T } declared = append(declared, name) } + // The kept data of the template `state` are asked for in `requires`, under their local names (design 47 G §613). + for _, r := range m.Requires { + if r.Storage == "key-value" && r.Template == "state" { + declared = append(declared, r.Name) + } + } announced := []string{} for _, s := range Kept { announced = append(announced, s.Name) @@ -62,3 +74,37 @@ func TestTheStatesTheModuleAnnouncesAreTheStatesItsManifestDeclares(t *testing.T } } } + +// Design 47 G §613 (A-20): deliveries, groups, owed and conditions are `requires` entries of the template `state` under +// their local names, taking the buckets they already had; merges and source stay in the old `state` section. +func TestTheFourStatesMoveToTheTemplateAndTwoStayOld(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + var m struct { + State []json.RawMessage `json:"state"` + Requires []struct{ Name, Storage, Template string } + } + if err := json.Unmarshal(raw, &m); err != nil { + t.Fatal(err) + } + moved := map[string]bool{} + for _, r := range m.Requires { + if r.Storage == "key-value" && r.Template == "state" { + moved[r.Name] = true + } + } + old := string(raw[strings.Index(string(raw), `"state": [`):]) + old = old[:strings.Index(old, "]")] + for _, n := range []string{"deliveries", "groups", "owed", "conditions"} { + if !moved[n] || strings.Contains(old, `"`+n+`"`) { + t.Errorf("%s: template state %v, old section lists it %v", n, moved[n], strings.Contains(old, `"`+n+`"`)) + } + } + for _, n := range []string{"merges", "source"} { + if moved[n] || !strings.Contains(old, `"`+n+`"`) { + t.Errorf("%s stays in the old section", n) + } + } +} diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index b015d3760..62651668b 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -278,6 +278,29 @@ ] } ] + }, + { + "name": "deliveries", + "storage": "key-value", + "template": "state" + }, + { + "name": "groups", + "storage": "key-value", + "template": "state" + }, + { + "name": "owed", + "storage": "key-value", + "template": "state", + "options": { + "history": 5 + } + }, + { + "name": "conditions", + "storage": "key-value", + "template": "state" } ], "packages": { @@ -296,8 +319,6 @@ "mesh-delivery-view.view_reach" ], "state": [ - "deliveries", - "groups", "pulls", { "name": "asks", @@ -307,15 +328,10 @@ "name": "merges", "history": 10 }, - { - "name": "owed", - "history": 5 - }, { "name": "source", "history": 64 - }, - "conditions" + } ], "capabilities": [ "ledger-calls", -- 2.54.0 From ab5423f87fdf2fb619395f39ff2649a3bd701152 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:52:11 +0100 Subject: [PATCH 089/243] GATE: the gate's manifest consumes the seat git's events with the hyphens the seat gives them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §396, §310 (G-9: the seat git's events take hyphens) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/bus_test.go | 10 +++++----- .../cmd/mesh-delivery-gate/checkerbus_test.go | 2 +- .../cmd/mesh-delivery-gate/manifest_test.go | 4 ++-- .../cmd/mesh-delivery-gate/onestatus_bus_test.go | 4 ++-- modules/mesh-delivery-gate/module.json | 12 ++++++------ 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go index 29f225922..7093f6451 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bus_test.go @@ -578,7 +578,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB announced := pr("novox/app", 7, head, "fix/x", "src/a.go") announced.BaseSHA = "a1a1a1a1a1a1" announced.Title = "Fix the apostrophe (issue 975)" - b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(announced)) + b.publish(t, "mesh.seat.git.event.pull-updated", seatUpdated(announced)) rt.until("the pull request's key to hold its head", eventWithin, func() bool { e, err := b.kvs["pulls"].Get(ctx, "novox.app.7") var k PullKey @@ -637,7 +637,7 @@ func TestAPassingVerdictAsksTheApproveAndItsApproveMergesWithinSecondsOverARealB b.forgeSays(t, forgemsg.Forge{Kind: forgemsg.ForgeMerged, Repository: "novox/app", Number: 7, Head: head, Merge: strings.Repeat("b", 40), Root: "r", Trees: map[string]string{"app": "t"}}) // The forge's own word that the pull request merged follows the bridge's (its webhook): the delivery ends merged. - b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged("novox/app", 7, head, strings.Repeat("b", 40)))) + b.publish(t, "mesh.seat.git.event.pull-merged", seatMerged(merged("novox/app", 7, head, strings.Repeat("b", 40)))) rt.until("source.main to hold the merge", eventWithin, func() bool { e, err := b.kvs["source"].Get(ctx, mdc.SourceKey(MergeBranch)) if err != nil { @@ -820,7 +820,7 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe for _, p := range []PullEvent{pr("novox/mesh-controller", 1, ctlHead, "feat/x", "internal/a.go"), pr("novox/mesh-catalog", 2, catHead, "feat/x", "modules/app/module.json")} { p.BaseSHA = "a1a1a1a1a1a1" - b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(p)) + b.publish(t, "mesh.seat.git.event.pull-updated", seatUpdated(p)) } // Both members pass at the checker, whose one verdict covers both heads: the group is asked once, no composed check is asked // of the controller (issue 1318). @@ -877,8 +877,8 @@ func TestAGroupOfTwoWithGreenChecksGetsOneAskAndOnItsApproveBothMergeOneAfterThe // The forge's own word that each pull request merged follows the bridge's (its webhook), after the second was handed (the // checker judges the members still open by the trunk the first merge moved, which this test does not play): both members // end merged, the group with them, and the controller was never called (R6-C3). - b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged(one.Repository, one.Number, one.Head.Head, strings.Repeat("a", 40)))) - b.publish(t, "mesh.seat.git.event.pull.merged", seatMerged(merged(two.Repository, two.Number, two.Head.Head, strings.Repeat("b", 40)))) + b.publish(t, "mesh.seat.git.event.pull-merged", seatMerged(merged(one.Repository, one.Number, one.Head.Head, strings.Repeat("a", 40)))) + b.publish(t, "mesh.seat.git.event.pull-merged", seatMerged(merged(two.Repository, two.Number, two.Head.Head, strings.Repeat("b", 40)))) for _, id := range []string{IDOf("novox/mesh-controller", ctlHead), IDOf("novox/mesh-catalog", catHead)} { id := id rt.until(id+" to be merged", eventWithin, func() bool { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go index c4ab1125b..3df8d159c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go @@ -54,7 +54,7 @@ func (b *testBus) checkPasses(t *testing.T, rt *runtime, group, trunk string, pu Required: []string{ctxCheck}, Statuses: []string{ctxCheck}, Checks: map[string]HeadCheck{ctxCheck: {State: "success", Description: line}}}) } rt.mu.Unlock() - b.publish(t, "mesh.seat.git.event.forge.status", gitseat.ForgeStatusEvent{Repository: owner + "/" + repo, Commit: p.head, + b.publish(t, "mesh.seat.git.event.forge-status", gitseat.ForgeStatusEvent{Repository: owner + "/" + repo, Commit: p.head, Context: ctxCheck, State: "success", Description: line}) } } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index 6a6f502a7..d21a57d0c 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -122,10 +122,10 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { if !reflect.DeepEqual(m.Emits, []string{"transition", "group", "merged"}) { t.Fatalf("emits %v", m.Emits) } - // Eight buckets, and no `asked` (the teller asks, ADR 0437 §6): ADR 0450 §3 and the conditions it raised (issue 1275). No + // Four buckets still in the old `state`, the four of the template `state` moved to `requires` (design 47 G §613), and no `asked` (the teller asks, ADR 0437 §6): ADR 0450 §3 and the conditions it raised (issue 1275). No // `turns`, no turn tool and no `check-gates-merge`: no check gates a merge (novox/hq ADR 0661 §1, §2, §6). buckets := m.buckets() - if !reflect.DeepEqual(buckets, []string{"deliveries", "groups", "pulls", "asks", "merges", "owed", "source", "conditions"}) { + if !reflect.DeepEqual(buckets, []string{"pulls", "asks", "merges", "source"}) { t.Fatalf("state %v", buckets) } for _, tool := range m.Tools { diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go index ae815a941..bca7d30bb 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go @@ -39,7 +39,7 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) { Statuses: []string{}, Checks: map[string]HeadCheck{}}} rt.mu.Unlock() announced := pr("novox/app", 8, head, "fix/y", "src/b.go") - b.publish(t, "mesh.seat.git.event.pull.updated", seatUpdated(announced)) + b.publish(t, "mesh.seat.git.event.pull-updated", seatUpdated(announced)) // The checker's verdict at the head, then the forge's word that its bridge set mesh/check. hash, err := mdc.HeadsHash([]mdc.Head{{Repository: "novox/app", Number: 8, Head: head}}) @@ -56,7 +56,7 @@ func TestADeliveryOnMeshCheckGoesReadyAskedAndMergedOverARealBus(t *testing.T) { rt.openPulls[0].Statuses = []string{ctxCheck} rt.openPulls[0].Checks = map[string]HeadCheck{ctxCheck: {State: "success", Description: line}} rt.mu.Unlock() - set := b.publish(t, "mesh.seat.git.event.forge.status", gitseat.ForgeStatusEvent{Repository: "novox/app", Commit: head, + set := b.publish(t, "mesh.seat.git.event.forge-status", gitseat.ForgeStatusEvent{Repository: "novox/app", Commit: head, Context: ctxCheck, State: "success", Description: line}) var key string diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index 62651668b..ce584cab1 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -17,12 +17,12 @@ } ], "consumes": [ - "git.pull.updated", - "git.pull.merged", - "git.pull.closed", - "git.forge.pull", - "git.forge.status", - "git.repo.writable", + "git.pull-updated", + "git.pull-merged", + "git.pull-closed", + "git.forge-pull", + "git.forge-status", + "git.repo-writable", "mesh-delivery-forge-bridge.forge.**", "mesh-delivery-forge-bridge.owed.ended", "mesh-delivery-teller.answered", -- 2.54.0 From a587125047e8f00dd6f5f46c8a667757a67b8ee3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:55:57 +0100 Subject: [PATCH 090/243] =?UTF-8?q?GATE=20A-23,=20=C2=A7431:=20the=20gate?= =?UTF-8?q?=20serves=20recovery,=20marking=20the=20delivery=20record's=20f?= =?UTF-8?q?ield=20recovery,=20and=20no=20longer=20serves=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §81, §132, §431, A-23 (G-20), A-18 as corrected Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/main.go | 23 ++++------ .../cmd/mesh-delivery-gate/manifest_test.go | 8 ++-- .../cmd/mesh-delivery-gate/record.go | 10 ++++ .../cmd/mesh-delivery-gate/record_test.go | 31 +++++++++++++ .../cmd/mesh-delivery-gate/recovery.go | 46 +++++++++++++++++++ modules/mesh-delivery-gate/module.json | 8 +++- 6 files changed, 107 insertions(+), 19 deletions(-) create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go index 694da81e7..618c43b08 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/main.go @@ -10,6 +10,7 @@ package main import ( "encoding/json" + "errors" "fmt" "os" "strconv" @@ -345,24 +346,20 @@ func tools(h *Holder, l *listening) []stdio.Tool { Input: map[string]any{"type": "object", "properties": map[string]any{ "branch": str("the watched branch, escaped as a key name is; absent: the list of the branches the gate keeps a source revision of")}}, Run: func(a map[string]any) (any, error) { return h.SourceRevision(strArg(a, "branch")), nil }}, - {Name: seat + "checks", - Description: "What the mesh's checks said of a pull request's head or of one commit: each of the commit's " + - "statuses (mesh/check, mesh/delivery, …) with its state, description and when it " + - "was set; the merge check's verdict as the checker's mesh/check line says it, and when it was heard; and whether the branch's protection would let it " + - "merge, every required status being success.", + {Name: seat + "recovery", + Description: "Name a merge a recovery (design 47 G §81): the delivery record that holds the commit is marked with who named it " + + "(the caller the runtime proved), why and the commit, and recovery-merged is said for every channel.", Input: map[string]any{"type": "object", "properties": map[string]any{"repository": str("owner/repository"), - "number": str("a pull request's number: its head is read"), - "commit": str("a commit's sha, or the start of one, instead of a pull request")}, - "required": []string{"repository"}}, + "commit": str("the merge commit"), "why": str("why the merge is a recovery")}, + "required": []string{"repository", "commit", "why"}}, Run: func(a map[string]any) (any, error) { - if err := need(a, "repository"); err != nil { + if err := need(a, "repository", "commit", "why"); err != nil { return nil, err } - number, err := intArg(a, "number") - if err != nil { - return nil, err + if h.Records == nil { + return nil, errors.New("this gate keeps no delivery records") } - return h.Checks(strArg(a, "repository"), number, strArg(a, "commit")) + return h.Records.Recovery(strArg(a, "repository"), strArg(a, "commit"), strArg(a, "why"), stdio.CallerOf(a)) }}, {Name: seat + "stalled", Description: "Every delivery held past its state's bound, with the transition the table lets healer H2 take. " + diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go index d21a57d0c..d16a2bf9b 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/manifest_test.go @@ -76,8 +76,8 @@ func TestItClaimsTheDeliverySeatAndFollowsWhatItHandles(t *testing.T) { t.Fatal("the manifest declares a seat; mesh-delivery is the mesh's own") } // The verbs the seat keeps: times, what-if, release, stop and retire-history went with the plans (ADR 0539 §17), and - // settle with the merge states (design 47 G §431). - if want := []string{"deliveries", "show", "groups", "source", "checks", "stalled"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { + // settle and checks with the merge states (design 47 G §431); recovery is G's (§81, A-23). + if want := []string{"deliveries", "show", "groups", "source", "recovery", "stalled"}; !reflect.DeepEqual(m.Claims[0].Serves, want) { t.Fatalf("serves %v, wanted %v", m.Claims[0].Serves, want) } // The forge messages' layout is the bridge's published package forgemsg, taken by its series (ADR 0634 §3, design 32). @@ -285,7 +285,7 @@ func TestTheGateHoldsTheSeatAsTheSDKsTableOfRelease6HasIt(t *testing.T) { served[verb] = true } } - for _, removed := range []string{"settle"} { // design 47 G §431 + for _, removed := range []string{"settle", "checks"} { // design 47 G §431 if served[removed] { t.Errorf("the gate serves %s, which G removed with the merge states", removed) } @@ -293,7 +293,7 @@ func TestTheGateHoldsTheSeatAsTheSDKsTableOfRelease6HasIt(t *testing.T) { t.Errorf("the manifest serves %s, which G removed with the merge states", removed) } } - for _, verb := range []string{"deliveries", "show", "groups", "checks", "stalled"} { + for _, verb := range []string{"deliveries", "show", "groups", "stalled", "recovery"} { if !served[verb] { t.Errorf("the gate does not serve %s, which the seat keeps", verb) } diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go index 27e998d4e..9e5d214be 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record.go @@ -98,6 +98,16 @@ type Record struct { // Acts are the acts owed the forge for the record's pull requests, by id, with their outcomes (owedended.go). Acts map[string]RecordAct `json:"acts,omitempty"` Ended *RecordEnded `json:"ended,omitempty"` + // Recovery marks the merge its author named a recovery by the seat's verb recovery (design 47 G §81, A-23). + Recovery *RecordRecovery `json:"recovery,omitempty"` +} + +// RecordRecovery is the field recovery of a record: who named the merge a recovery (the caller the runtime proved), why, +// and the merge commit. +type RecordRecovery struct { + By string `json:"by"` + Why string `json:"why"` + Commit string `json:"commit"` } // RecordGroup is the group's state as the record keeps it: what `merges` held for the group, and its members. diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go index 066cb1d55..21f70d038 100644 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/record_test.go @@ -789,6 +789,37 @@ func TestTheRecordsToolsReadThemAndEndOneByHand(t *testing.T) { t.Fatalf("a second repair did something: %+v", lines) } }) + t.Run("recovery marks the record that holds the commit through its ordinary update and says recovery-merged once (design 47 G A-23)", func(t *testing.T) { + rs, r, said := setup(t) + commit := strings.Repeat("1", 40) + got, err := rs.Recovery("novox/app", commit, "the live delivery was broken", "jochen") + if err != nil || !got.Recovery || got.By != "jochen" || got.Commit != commit || got.Repository != "novox/app" { + t.Fatalf("%+v %v", got, err) + } + held, err := rs.Read(r.Instance) + if err != nil || held.Recovery == nil || *held.Recovery != (RecordRecovery{By: "jochen", Why: "the live delivery was broken", Commit: commit}) { + t.Fatalf("the record is not marked: %+v %v", held, err) + } + subject := meshdelivery.SubjectRecoveryMerged + if _, err := rs.Recovery("novox/app", commit[:12], "the live delivery was broken", "jochen"); err != nil { + t.Fatal(err) + } + n := 0 + for _, s := range *said { + if strings.HasPrefix(s, subject+" ") { + n++ + } + } + if n != 1 { + t.Fatalf("recovery-merged said %d times: %v", n, *said) + } + if _, err := rs.Recovery("novox/other", commit, "why", "jochen"); err == nil { + t.Fatal("a commit no record holds was marked") + } + if _, err := rs.Recovery("novox/app", commit, " ", "jochen"); err == nil { + t.Fatal("a recovery with no why was taken") + } + }) t.Run("a removal asks the runtime's ledger removal of the instance, and one said twice changes nothing", func(t *testing.T) { was := ledgerCall t.Cleanup(func() { ledgerCall = was }) diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go new file mode 100644 index 000000000..2c7764c25 --- /dev/null +++ b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/recovery.go @@ -0,0 +1,46 @@ +package main + +import ( + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshdelivery" +) + +// Recovery names a merge a recovery (design 47 G §81, A-23): the delivery record that holds the commit is marked through its +// ordinary update, with one field recovery holding who named it (the caller the runtime proved), why and the commit, and +// recovery-merged is said. Said twice for the same mark, it changes nothing and says nothing more. +func (rs *Records) Recovery(repository, commit, why, by string) (meshdelivery.RecoveryAnswer, error) { + repository, commit, why = strings.ToLower(strings.TrimSpace(repository)), strings.TrimSpace(commit), strings.TrimSpace(why) + if repository == "" || commit == "" || why == "" { + return meshdelivery.RecoveryAnswer{}, errors.New("a recovery names its repository, its merge commit and why") + } + kept, err := rs.Kept() + if err != nil { + return meshdelivery.RecoveryAnswer{}, fmt.Errorf("the delivery records cannot be read: %w", err) + } + for _, r := range kept { + for id, head := range r.Heads { + if !strings.HasPrefix(strings.ToLower(id), repository+"@") || !strings.HasPrefix(head, commit) { + continue + } + want := RecordRecovery{By: by, Why: why, Commit: head} + if r.Recovery != nil && *r.Recovery == want { + return meshdelivery.RecoveryAnswer{By: by, Commit: head, Recovery: true, Repository: repository}, nil + } + seq, err := rs.Ledger.Apply(r.Instance, OpTrace, map[string]any{"recovery": want}) + if err != nil { + return meshdelivery.RecoveryAnswer{}, err + } + r.Recovery, r.Sequence = &want, seq + now := rs.now() + if err := rs.say(meshdelivery.SubjectRecoveryMerged, meshdelivery.RecoveryMergedEvent{At: &now, By: by, Commit: head, + Repository: repository, V: 1, Why: why}, "recovery-merged."+r.Instance+"."+head); err != nil { + return meshdelivery.RecoveryAnswer{}, fmt.Errorf("the record %s is marked but recovery-merged is not said: %w", r.Instance, err) + } + return meshdelivery.RecoveryAnswer{By: by, Commit: head, Recovery: true, Repository: repository}, nil + } + } + return meshdelivery.RecoveryAnswer{}, fmt.Errorf("no delivery record holds %s at %s: the merge is taken in first", repository, commit) +} diff --git a/modules/mesh-delivery-gate/module.json b/modules/mesh-delivery-gate/module.json index ce584cab1..09398c700 100644 --- a/modules/mesh-delivery-gate/module.json +++ b/modules/mesh-delivery-gate/module.json @@ -11,7 +11,7 @@ "show", "groups", "source", - "checks", + "recovery", "stalled" ] } @@ -127,6 +127,9 @@ }, "words": { "type": "string" + }, + "recovery": { + "type": "object" } }, "operations": [ @@ -154,7 +157,8 @@ "name": "trace", "effect": "set-fields", "fields": [ - "trace" + "trace", + "recovery" ], "sequence": true }, -- 2.54.0 From 95f29d6c54f4731a215ced24733923591f43eafe Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:58:08 +0100 Subject: [PATCH 091/243] G findings: the modules with findings of their own write them through mesh-sdk's writer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit artifact-store-tools (image-missing, sweep-refused), distribution (program and tools bundle, one writer each over its own kinds), gitea's package-registry (and the token-refused key the TypeScript bundle watches) and mesh-config keep their findings open and write and clear them through the generic writer; mesh-vault states its assignments view's. A part of an id told apart by a ref is in the id (.), each escaped; the writer starts once Serve has the runtime's channel; the removed findings helpers and resolved words go. Builds: design 47 G §686, §687, §689, A-3, A-14, A-16, A-22 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/artifact-store-tools/holder.go | 143 ++++++++++-------- .../holder_integration_test.go | 2 +- .../cmd/artifact-store-tools/holder_test.go | 33 +++- .../cmd/artifact-store-tools/main.go | 21 ++- .../cmd/artifact-store-tools/sweepseats.go | 31 ++-- modules/artifact-store-tools/go.mod | 1 - modules/claude-code/cmd/claude-code/main.go | 12 +- .../cmd/distribution-tools/main.go | 41 ++--- modules/distribution/cmd/distribution/main.go | 19 ++- .../distribution/cmd/distribution/windows.go | 26 ---- modules/distribution/go.mod | 1 - .../distribution/internal/holder/findings.go | 77 ++++++++++ .../gitea/cmd/package-registry/findings.go | 76 ++++++++++ modules/gitea/cmd/package-registry/forge.go | 4 +- modules/gitea/cmd/package-registry/main.go | 21 +++ modules/gitea/cmd/package-registry/seats.go | 21 +-- .../gitea/cmd/package-registry/seats_test.go | 6 +- .../whole_integration_test.go | 7 +- modules/gitea/go.mod | 1 - modules/gitea/packages-account.ts | 2 +- .../mesh-build-go/cmd/mesh-build-go/main.go | 11 +- modules/mesh-build-go/go.mod | 2 +- modules/mesh-config/cmd/mesh-config/config.go | 4 + .../mesh-config/cmd/mesh-config/findings.go | 133 +++++++++------- modules/mesh-config/cmd/mesh-config/main.go | 22 ++- .../mesh-config/cmd/mesh-config/renames.go | 4 +- .../cmd/mesh-config/renames_test.go | 21 ++- .../mesh-config/cmd/mesh-config/retiring.go | 4 +- .../cmd/mesh-config/whole_integration_test.go | 3 +- modules/mesh-config/go.mod | 1 - .../cmd/mesh-provisioner/main.go | 12 +- modules/mesh-vault/cmd/mesh-vault/main.go | 18 ++- .../mesh-vault/cmd/mesh-vault/vault_test.go | 2 +- modules/mesh-vault/go.mod | 3 +- modules/mesh-watcher/cmd/mesh-watcher/main.go | 12 +- modules/minio/cmd/object-store/main.go | 12 +- modules/nfs-server/cmd/nfs-server/main.go | 12 +- 37 files changed, 569 insertions(+), 252 deletions(-) create mode 100644 modules/distribution/internal/holder/findings.go create mode 100644 modules/gitea/cmd/package-registry/findings.go diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go index fc598c633..e6e626cb9 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder.go @@ -29,12 +29,12 @@ import ( "sync" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -45,8 +45,6 @@ const ( HavesRead = "node-engine.have" // ManifestsRead is the indexer's manifests: the images a manifest pins by itself. ManifestsRead = indexerModule + "." + catalog.StateManifests - // FindingsState is the holder's own findings (meshconditions.StateFindings): image-missing. - FindingsState = meshconditions.StateFindings // Module is this module, the observer its findings name. Module = "artifact-store-tools" ) @@ -83,8 +81,10 @@ type Door interface { // Holder is the seat's holder. type Holder struct { - Bus Bus - Door func() (Door, error) + Bus Bus + // Findings is the module's finding writer: S25's and the sweep's findings go through it. + Findings Findings + Door func() (Door, error) // Listed is the store as its seats list it now (Store.View); nil reads none, and then nothing is said gone or collected. Listed func(ctx context.Context) (*View, error) Acts meshhandactsclient.Recorder @@ -93,9 +93,12 @@ type Holder struct { Now func() time.Time Logf func(string, ...any) - sweep sync.Mutex // one sweep at a time - mu sync.Mutex - ended map[string]time.Time // a done key's end, read once from its history + sweep sync.Mutex // one sweep at a time + mu sync.Mutex + fmu sync.Mutex // the findings said and the sweep's refusal + said map[string]meshconditions.Finding // S25's findings as last written, by key + refused *meshconditions.Finding // the sweep's finding while a seat's refusal stands + ended map[string]time.Time // a done key's end, read once from its history } // Build is one done build of a module, as its key holds it, with when it ended. @@ -852,16 +855,75 @@ func inStoreAt(image, address string) string { return r } -// Observe writes S25's findings (image-missing): each want whose build names an image the store does not hold, on the -// want's machine, and deletes each finding that no longer holds. Nothing is said without a listing. +// Observe says S25's findings (image-missing): each want whose build names an image the store does not hold, on the +// want's machine, written once and cleared once it no longer holds. Nothing is said without a listing. func (h *Holder) Observe(ctx context.Context) error { - j, err := h.judge(ctx) + now, err := h.observed(ctx) if err != nil { return err } - if j.view == nil { + if now == nil { return nil } + h.fmu.Lock() + defer h.fmu.Unlock() + if h.said == nil { + h.said = map[string]meshconditions.Finding{} + } + for k, was := range h.said { + if _, still := now[k]; !still { + if err := h.Findings.Clear(ctx, was.Scope, was.ID, was.Kind, "the store holds the image again"); err != nil { + return err + } + delete(h.said, k) + } + } + for k, f := range now { + if was, ok := h.said[k]; ok && was.Summary == f.Summary { + continue // written only when what it says changes + } + if err := h.Findings.Write(ctx, f); err != nil { + return err + } + h.said[k] = f + } + return nil +} + +// Findings is the module's finding writer (views.Writer): the holder writes and clears through it (design 47 G §689). +type Findings interface { + Write(ctx context.Context, finding meshconditions.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error +} + +// FindingsObserved answers the findings this module observes now, for its writer's statement (views.WriterConfig.Observe): +// S25's findings and the sweep's, when a seat refused it and no clean sweep cleared it. +func (h *Holder) FindingsObserved(ctx context.Context) (views.Observed[meshconditions.Finding], error) { + now, err := h.observed(ctx) + if err != nil { + return views.Observed[meshconditions.Finding]{}, err + } + var out views.Observed[meshconditions.Finding] + for _, f := range now { + out.Findings = append(out.Findings, f) + } + h.fmu.Lock() + defer h.fmu.Unlock() + if h.refused != nil { + out.Findings = append(out.Findings, *h.refused) + } + return out, nil +} + +// observed is S25's findings now by key, nil without a listing. +func (h *Holder) observed(ctx context.Context) (map[string]meshconditions.Finding, error) { + j, err := h.judge(ctx) + if err != nil { + return nil, err + } + if j.view == nil { + return nil, nil + } address := "" if d, err := h.Door(); err == nil { address = d.Address() @@ -870,7 +932,7 @@ func (h *Holder) Observe(ctx context.Context) error { for _, b := range j.built { builds[b.Module+" "+b.FP] = b } - now := map[string]meshconditions.FindingsByKeyValue{} + now := map[string]meshconditions.Finding{} for _, w := range j.wants { b, ok := builds[w.Module+" "+w.FP] if !ok { @@ -895,63 +957,26 @@ func (h *Holder) Observe(ctx context.Context) error { continue } f := imageMissing(w.Node, w.Module, reference, h.now()) - now[f.Key] = f + now[views.FindingKey(f.Scope, f.ID, f.Kind)] = f break } } - keys, err := h.Bus.Keys(FindingsState) - if err != nil { - return fmt.Errorf("the findings: %w", err) - } - for _, k := range keys { - if k == sweepFindingKey { - continue // the sweep's own finding, which the sweep clears - } - if _, still := now[k]; !still { - // A finding of a first view unanswered is the reader's (views.Reader), cleared by its answer, not by a sweep. - if raw, found, err := h.Bus.Get(FindingsState, k); err == nil && found { - if f, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && f.Kind == views.UnansweredKind { - continue - } - } - if err := h.Bus.Delete(FindingsState, k); err != nil { - return err - } - } - } - for k, f := range now { - raw, found, err := h.Bus.Get(FindingsState, k) - if err != nil { - return err - } - if found { - if was, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && was.Summary == f.Summary { - continue // written only when what it says changes - } - } - if err := meshconditionsclient.CheckFinding(f); err != nil { - return err - } - if err := h.Bus.Put(FindingsState, k, f); err != nil { - return err - } - } - return nil + return now, nil } -// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages). -func imageMissing(node, module, image string, at time.Time) meshconditions.FindingsByKeyValue { - id := module + "." + node - return meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: meshconditionsclient.Key(meshconditionsclient.ScopeModule, id, ImageMissing), Kind: ImageMissing, - Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: id, Machine: node}, - Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator, +// imageMissing is S25's finding, in the controller's words (cmd/mesh-controller/image_missing.go watchMissingImages); its id +// is the module's and the machine's, each escaped (design 47 G A-14). +func imageMissing(node, module, image string, at time.Time) meshconditions.Finding { + id := keys.EscapeName(module) + "." + keys.EscapeName(node) + return meshconditions.Finding{Scope: "module", ID: id, Kind: ImageMissing, + Subject: meshconditions.Subject{Scope: "module", ID: id, Machine: node}, + Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Summary: fmt.Sprintf("%s is left out of every send to %s: its declaration names an image that the artifact store does "+ "not hold (the evidence names it); the running container is untouched", module, node), Headline: capital(module) + " is not updated on " + node, Explanation: fmt.Sprintf("The new version of %s names a container image that the mesh's store does not hold, so it "+ "was not sent to %s. What runs there is untouched, and the rest of %s is sent as usual.", module, node, node), Needs: fmt.Sprintf("build %s again; the details name the image.", module), - Resolved: "Resolved: the store holds the image of " + module + " again", Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: "image " + image + " is not in the store"}}, Source: "S25", Observer: Module, At: at.UTC()} } diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go index 575dfcc00..3963d3749 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_integration_test.go @@ -198,7 +198,7 @@ func TestTheSeatSweepsAtABuildsEndOnARealBusWithNoLease(t *testing.T) { } return m.Data, nil } - h := &Holder{Bus: findingsAt{realBus: bus}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf} + h := &Holder{Bus: findingsAt{realBus: bus}, Findings: findingsThrough{findingsAt{realBus: bus}}, Door: mem.Door, Listed: mem.Listed, Seats: ask, Now: time.Now, Logf: t.Logf} h.Acts = meshhandactsclient.Recorder{Ask: ask} // A build's end heard from the builds bucket, as follow hands it over. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go index 6aa10a50f..a5df807cd 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/holder_test.go @@ -17,10 +17,10 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference" reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" @@ -229,7 +229,7 @@ func aMesh(t *testing.T) (*Holder, *memBus, *memDoor, *[]string) { want, _ := json.Marshal(meshwants.WantsByWantNodeModuleValue{V: 1, Fp: fp(1), Self: true}) _ = b.Put(WantsRead, "want.ace.letta", json.RawMessage(want)) var acts []string - h := &Holder{Bus: b, Door: func() (Door, error) { return door, nil }, Seats: door.seat, + h := &Holder{Bus: b, Findings: findingsThrough{b}, Door: func() (Door, error) { return door, nil }, Seats: door.seat, Listed: func(context.Context) (*View, error) { return &View{L: listing}, nil }, Now: func() time.Time { return day.Add(48 * time.Hour) }, Acts: meshhandactsclient.Recorder{Ask: func(key string, body any) (json.RawMessage, error) { @@ -370,7 +370,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) if c.letGo[0] == archiveRef("letta", 2) && !slices.Contains(door.calls, "seat:oci-artifact-registry.remove letta/files@"+digestOf(archiveRef("letta", 2))+" "+Retention) { t.Errorf("the seats were called %v", door.calls) } - _, finding, _ := b.Get(FindingsState, sweepFindingKey) + _, finding, _ := b.Get(FindingsState, views.FindingKey("module", Module, SweepRefused)) if finding != (c.refuse != "") { t.Errorf("the sweep's finding is written: %v, want %v", finding, c.refuse != "") } @@ -379,7 +379,7 @@ func TestTheSeatKeepsTheFiveNewestAndTheWantedAndSweepsWithNoLease(t *testing.T) if _, err := h.Sweep(context.Background(), Sweeping{Most: MostPerSweep, By: "again"}); err != nil { t.Fatal(err) } - if _, still, _ := b.Get(FindingsState, sweepFindingKey); still { + if _, still, _ := b.Get(FindingsState, views.FindingKey("module", Module, SweepRefused)); still { t.Error("a clean sweep left the sweep's finding") } return @@ -410,9 +410,10 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi if err := h.Observe(context.Background()); err != nil { t.Fatal(err) } - key := meshconditionsclient.Key(meshconditionsclient.ScopeModule, "letta.ace", ImageMissing) + key := views.FindingKey("module", "letta.ace", ImageMissing) raw, found, _ := b.Get(FindingsState, key) - f, err := meshconditions.DecodeFindingsByKeyValue(raw) + var f meshconditions.Finding + err := json.Unmarshal(raw, &f) if !found || err != nil || f.Subject.Machine != "ace" || f.Source != "S25" || !strings.Contains(f.Evidence[0].Said, imageRef("letta", 1)) { t.Fatalf("the finding is %+v (%v, %v)", f, found, err) } @@ -441,3 +442,23 @@ func TestAWantedImageTheStoreLacksIsAFindingAndImagesNamesWantsAndHaves(t *testi t.Fatalf("the images of ace are %+v: %v", images, err) } } + +// FindingsState is the module's state `findings`, where the tests' writer keeps what the holder writes through it. +const FindingsState = "findings" + +// findingsThrough is the holder's finding writer in a test: it keeps a finding in its key and deletes it as the module's +// writer does (views.Writer), without the bus. +type findingsThrough struct { + bus interface { + Put(state, key string, value any) error + Delete(state, key string) error + } +} + +func (f findingsThrough) Write(_ context.Context, finding meshconditions.Finding) error { + return f.bus.Put(FindingsState, views.FindingKey(finding.Scope, finding.ID, finding.Kind), finding) +} + +func (f findingsThrough) Clear(_ context.Context, scope, id, kind, _ string) error { + return f.bus.Delete(FindingsState, views.FindingKey(scope, id, kind)) +} diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index 3af7eacc5..611ef1130 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -13,8 +13,8 @@ import ( "os" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/views" @@ -24,12 +24,29 @@ func logf(format string, a ...any) { fmt.Fprintf(os.Stderr, format+"\n", a...) } func main() { s := &Store{Ask: stdio.Ask} - v := &views.Reader{Module: "artifact-store-tools", Logf: logf, Findings: meshconditionsclient.FindingWriter{State: stdio.State("findings")}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} + v := &views.Reader{Module: "artifact-store-tools", Logf: logf, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} v.Follow(assignerModule, "wants", WantsTool) v.Follow(indexerModule, "manifests", ManifestsTool) h := &Holder{Bus: runtimeBus{views: v}, Door: DoorAt(os.Getenv("MESH_ARTIFACT_STORE_ADDRESS_FILE"), s), Listed: s.View, Seats: stdio.Ask, Now: time.Now, Logf: logf} h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask} + // The module's one writer (design 47 G §689, A-16): the holder's findings and the reader's go through it. + w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused}, Observe: h.FindingsObserved}) + if err != nil { + logf("%v", err) + os.Exit(1) + } + h.Findings = w + v.Findings = views.WriterFindings{Writer: w} + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := w.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } + }() v.Start(context.Background()) go follow(h, v) // An empty name serves as the module the runtime names (MESH_SERVED_MODULE): artifact-store-tools. diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go index 568b62781..34c85d483 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/sweepseats.go @@ -10,15 +10,16 @@ package main // sweep's finding, sweep-refused; the next sweep with no refusal clears it. import ( + "context" "encoding/json" "fmt" "strings" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" artifactref "git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient/reference" ociseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/ociartifactregistry" "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/keys" reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" ) @@ -26,9 +27,6 @@ import ( // SweepRefused is the sweep's finding: a seat refused a hold or a remove the sweep asked. const SweepRefused = "sweep-refused" -// sweepFindingKey is the one key of the sweep's finding in the module's findings. -var sweepFindingKey = meshconditionsclient.Key(meshconditionsclient.ScopeModule, Module, SweepRefused) - // canonical is a kept reference in the older form, `artifact-store://`, whichever form it came in; another is as // it was. func canonical(reference string) string { @@ -115,28 +113,35 @@ func (h *Holder) removeOnSeat(reference, why string) error { // sweepFinding writes the sweep's finding: what the seat refused, and why. func (h *Holder) sweepFinding(what string, err error) { at := h.now().UTC() - f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: sweepFindingKey, Kind: SweepRefused, - Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: Module}, - Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator, + f := meshconditions.Finding{Scope: "module", ID: keys.EscapeName(Module), Kind: SweepRefused, + Subject: meshconditions.Subject{Scope: "module", ID: keys.EscapeName(Module)}, + Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Summary: fmt.Sprintf("the artifact sweep stopped: a registry seat refused %s (the evidence names why)", what), Headline: "The artifact sweep stopped", Explanation: "The sweep keeps what the mesh still needs and removes the rest through the registry seats. A seat " + "refused one of its asks, so it stopped before anything more was removed; what it kept stays kept.", Needs: "read the seat holder's journal for the refusal; the next sweep asks again.", - Resolved: "Resolved: the artifact sweep ran through without a refusal", Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: fmt.Sprintf("%s: %v", what, err)}}, Source: "CAT-AS1", Observer: Module, At: at} - if werr := h.Bus.Put(FindingsState, sweepFindingKey, f); werr != nil { + h.fmu.Lock() + h.refused = &f + h.fmu.Unlock() + if werr := h.Findings.Write(context.Background(), f); werr != nil { h.logf("the sweep's finding (%s: %v) could not be written: %v", what, err, werr) } } // sweepClean clears the sweep's finding after a sweep no seat refused. func (h *Holder) sweepClean() { - if _, found, err := h.Bus.Get(FindingsState, sweepFindingKey); err == nil && found { - if err := h.Bus.Delete(FindingsState, sweepFindingKey); err != nil { - h.logf("the sweep's finding could not be cleared: %v", err) - } + h.fmu.Lock() + was := h.refused + h.refused = nil + h.fmu.Unlock() + if was == nil { + return + } + if err := h.Findings.Clear(context.Background(), was.Scope, was.ID, was.Kind, "a sweep ran through without a refusal"); err != nil { + h.logf("the sweep's finding could not be cleared: %v", err) } } diff --git a/modules/artifact-store-tools/go.mod b/modules/artifact-store-tools/go.mod index 2e3263e2d..8b71604f0 100644 --- a/modules/artifact-store-tools/go.mod +++ b/modules/artifact-store-tools/go.mod @@ -5,7 +5,6 @@ go 1.26.0 require ( git.novox.be/novox/mesh-contracts/go/clients/nodeengineclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhandacts v0.0.0-00010101000000-000000000000 diff --git a/modules/claude-code/cmd/claude-code/main.go b/modules/claude-code/cmd/claude-code/main.go index 9ab8baa7b..2ed3b5b03 100644 --- a/modules/claude-code/cmd/claude-code/main.go +++ b/modules/claude-code/cmd/claude-code/main.go @@ -504,10 +504,14 @@ func main() { os.Exit(1) } meshViews.Findings = views.WriterFindings{Writer: findings} - if err := findings.Start(context.Background()); err != nil { - say("%v", err) - os.Exit(1) - } + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := findings.Start(context.Background()); err != nil { + say("%v", err) + os.Exit(1) + } + }() meshViews.Start(context.Background()) go refreshMeshTools(p) servers := stateOf{stdio.State("servers")} diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index 41c949a4a..e43ca6b53 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -18,12 +18,12 @@ import ( "sync" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" reg "git.novox.be/novox/mesh-delivery/contracts/go/seats/containerimageregistry" "git.novox.be/novox/mesh-delivery/sdk/parts" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "distribution/internal/holder" ) @@ -67,21 +67,6 @@ func say(seat, event string, body holder.Event) error { return stdio.SeatPublish("mesh.seat."+seat+".event."+event, eventBody(event, body), body.ID) } -func finding(kind, subject, words string, raised bool) { - key := meshconditionsclient.Key(meshconditionsclient.ScopeSeat, "distribution", kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) - if !raised { - _ = stdio.State(meshconditions.StateFindings).Delete(key) - return - } - f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], - Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeSeat, ID: "distribution"}, Severity: meshconditionsclient.Warning, - Summary: words, Headline: words, Explanation: words, Resolved: "cleared once it no longer holds", - Source: kind, Observer: "distribution", At: time.Now().UTC()} - if _, err := stdio.State(meshconditions.StateFindings).Put(key, f); err != nil { - logf("the finding %s could not be written: %v", key, err) - } -} - // learned are the proved callers whose build this process recorded as publishing through the seats. var learned sync.Map @@ -420,7 +405,7 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder. Input: obj(map[string]any{}), Run: func(map[string]any) (any, error) { front, _ := url.Parse("http://127.0.0.1:5100") - return holder.Health(&holder.OCI{Base: front}, finding) + return holder.Health(&holder.OCI{Base: front}, findings.Say) }}, stdio.Tool{Name: "registry_incoming", Description: "Open transfers of incoming and its fill; with drop, drops one by hand (a hand act, why required).", Input: obj(map[string]any{"drop": s("a transfer id to drop"), "why": s("why it is dropped")}), @@ -516,10 +501,30 @@ func tools(v *holder.Verbs, windows kept[holder.Window], callers func() (holder. return out } +// findings keeps the open findings of this bundle for its writer (design 47 G §689). +var findings = &holder.Findings{Logf: logf} + func main() { + // The tools bundle's findings go through the module's writer (design 47 G §689, A-16); it owns the transfers' kinds, the + // program's writer the driver's. + w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Module: holder.Self, State: stdio.State("findings"), Kinds: holder.TransferFindingKinds, Observe: findings.Observed, Logf: logf}) + if err != nil { + logf("%v", err) + os.Exit(1) + } + findings.Writer = w + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := w.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } + }() base, _ := url.Parse("http://127.0.0.1:5001") store := &holder.OCI{Base: base, Upstream: &holder.Upstream{}} - h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: finding, + h := &holder.Holder{Objects: parts.BusObjects(holder.IncomingObjects), Builder: holder.Reserved{Ask: stdio.Ask}, Store: store, Say: say, Finding: findings.Say, Transfers: parts.BusTransfers[holder.Opening](holder.TransfersState), Cap: parts.BusCap(holder.IncomingObjects), Record: recordAct, Idle: parts.BusArm(holder.IdleState)} files := &holder.Files{Container: or(os.Getenv("MESH_REGISTRY_CONTAINER"), "mesh-registry"), Run: holder.ExecRunner, UID: os.Getuid()} v := &holder.Verbs{Holder: h, Catalog: store, Holds: kept[holder.Hold]{state: "holds", key: holder.Hold.Key}, Mirrors: mirrors{}, diff --git a/modules/distribution/cmd/distribution/main.go b/modules/distribution/cmd/distribution/main.go index 8db8da6a4..4cedc92d9 100644 --- a/modules/distribution/cmd/distribution/main.go +++ b/modules/distribution/cmd/distribution/main.go @@ -20,7 +20,9 @@ import ( "syscall" "time" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/views" "github.com/nats-io/nats.go/jetstream" "distribution/internal/holder" @@ -62,9 +64,22 @@ func run(ctx context.Context, getenv func(string) string) error { return err } defer calls.Close() - findings := process.OptionalBucket(ctx, js, "distribution_findings", "a window held for seven days is said in the log only", logger.Printf) + // The program's findings go through the module's writer (design 47 G §689); it owns the driver's kinds, the tools + // bundle's writer the transfers'. + findings := &holder.Findings{Logf: logger.Printf} + w, err := views.NewWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Module: holder.Self, Node: cred.Node, State: calls.State("findings"), Kinds: holder.DriverFindingKinds, + JS: js, Conn: conn, Observe: findings.Observed, Logf: logger.Printf}) + if err != nil { + return err + } + findings.Writer = w + if err := w.Start(ctx); err != nil { + return err + } + defer w.Close() go drive(ctx, js, cred, &holder.Driver{Delivery: busDelivery{calls}, State: kvWindows{windows}, Callers: busCallers{calls}.Read, - Closed: sayClosed(js, cred), Finding: findingsKV(findings, logger.Printf)}, logger) + Closed: sayClosed(js, cred), Finding: findings.Say}, logger) } front := &holder.Front{Store: store, Window: &open, Logf: logger.Printf, Published: func(name, tag string) (bool, error) { diff --git a/modules/distribution/cmd/distribution/windows.go b/modules/distribution/cmd/distribution/windows.go index 15b166776..e7a06e185 100644 --- a/modules/distribution/cmd/distribution/windows.go +++ b/modules/distribution/cmd/distribution/windows.go @@ -18,8 +18,6 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/process" "distribution/internal/holder" @@ -173,30 +171,6 @@ func (k kvWindows) Publishing() (holder.Publishing, error) { return p, k.get(holder.PublishingKey, &p) } -// findingsKV writes the driver's findings into distribution's state findings, as the tools bundle writes its own. -func findingsKV(kv jetstream.KeyValue, logf func(string, ...any)) func(kind, subject, words string, raised bool) { - return func(kind, subject, words string, raised bool) { - if kv == nil { - return - } - key := meshconditionsclient.Key(meshconditionsclient.ScopeSeat, holder.Self, kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if !raised { - _ = kv.Delete(ctx, key) - return - } - f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], - Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeSeat, ID: holder.Self}, Severity: meshconditionsclient.Warning, - Summary: words, Headline: words, Explanation: words + "; registry_old_name or registry_direct_writes drops a retired node by hand", - Resolved: "cleared when the window closes", Source: kind, Observer: holder.Self, At: time.Now().UTC()} - b, _ := json.Marshal(f) - if _, err := kv.Put(ctx, key, b); err != nil { - logf("the finding %s could not be written: %v", key, err) - } - } -} - // sayClosed says a window's closing on distribution's own family distribution.events, once (its id is the window's). func sayClosed(js jetstream.JetStream, cred process.Account) func(holder.Closed) error { return func(c holder.Closed) error { diff --git a/modules/distribution/go.mod b/modules/distribution/go.mod index 4fbf8a54a..d5b6b1f8d 100644 --- a/modules/distribution/go.mod +++ b/modules/distribution/go.mod @@ -3,7 +3,6 @@ module distribution go 1.26.0 require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-delivery/contracts/go/clients/containerimageregistryclient v0.0.0-00010101000000-000000000000 diff --git a/modules/distribution/internal/holder/findings.go b/modules/distribution/internal/holder/findings.go new file mode 100644 index 000000000..b99472d9b --- /dev/null +++ b/modules/distribution/internal/holder/findings.go @@ -0,0 +1,77 @@ +package holder + +import ( + "context" + "sync" + "time" + + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + keyparts "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" +) + +// FindingKinds are the kinds the driver says (the program) and the transfers say (the tools bundle): each bundle's writer +// owns its own, since the two share one state `findings` (design 47 G §580, §689). +var ( + DriverFindingKinds = []string{FindingWindowHeld} + TransferFindingKinds = []string{"publish-refused", "incoming-full"} +) + +// FindingWriter is the module's writer (views.Writer) as the keeper writes and clears through it. +type FindingWriter interface { + Write(ctx context.Context, finding meshconditions.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error +} + +// Findings keeps a bundle's open findings and says them through the module's writer (design 47 G §689, A-14): its `Say` is +// the Finding callback of the Driver and the Holder, and its `Observed` is the writer's observe, so a statement covers +// what is open. +type Findings struct { + Writer FindingWriter + Logf func(string, ...any) + Now func() time.Time + + mu sync.Mutex + open map[string]meshconditions.Finding +} + +// Say raises (raised) or clears the finding of a kind about a subject. Its id is the seat's and the subject's, each escaped +// as a key's name is (design 47 G A-14); the kind stays the bundle's own word. +func (k *Findings) Say(kind, subject, words string, raised bool) { + id := keyparts.EscapeName(Self) + "." + keyparts.EscapeName(subject) + k.mu.Lock() + defer k.mu.Unlock() + key := views.FindingKey("seat", id, kind) + if !raised { + delete(k.open, key) + if err := k.Writer.Clear(context.Background(), "seat", id, kind, "it no longer holds"); err != nil { + k.Logf("the finding %s could not be cleared: %v", key, err) + } + return + } + at := time.Now() + if k.Now != nil { + at = k.Now() + } + f := meshconditions.Finding{Scope: "seat", ID: id, Kind: kind, Subject: meshconditions.Subject{Scope: "seat", ID: id}, + Severity: meshconditions.FindingSeverityWarning, Summary: words, Headline: words, Explanation: words, + Source: kind, Observer: Self, At: at.UTC()} + if k.open == nil { + k.open = map[string]meshconditions.Finding{} + } + k.open[key] = f + if err := k.Writer.Write(context.Background(), f); err != nil { + k.Logf("the finding %s could not be written: %v", key, err) + } +} + +// Observed is the findings open now, for the writer's statement (views.WriterConfig.Observe). +func (k *Findings) Observed(context.Context) (views.Observed[meshconditions.Finding], error) { + k.mu.Lock() + defer k.mu.Unlock() + var out views.Observed[meshconditions.Finding] + for _, f := range k.open { + out.Findings = append(out.Findings, f) + } + return out, nil +} diff --git a/modules/gitea/cmd/package-registry/findings.go b/modules/gitea/cmd/package-registry/findings.go new file mode 100644 index 000000000..544e699f1 --- /dev/null +++ b/modules/gitea/cmd/package-registry/findings.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "sync" + "time" + + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + keyparts "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" + + "gitea/internal/packageseats" +) + +// findingKinds are the kinds this bundle says: the token refusal and the transfers' (design 47 G §580, §689); the module's +// TypeScript bundle shares the state `findings` and owns its own. +var findingKinds = []string{PackagesTokenRefused, "publish-refused", "incoming-full"} + +// findingWriter is the module's writer (views.Writer) as the keeper writes and clears through it. +type findingWriter interface { + Write(ctx context.Context, finding meshconditions.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error +} + +// Findings keeps a bundle's open findings and says them through the module's writer (design 47 G §689, A-14): its `Say` is +// the Finding callback of the Driver and the Holder, and its `Observed` is the writer's observe, so a statement covers +// what is open. +type Findings struct { + Writer findingWriter + Logf func(string, ...any) + Now func() time.Time + + mu sync.Mutex + open map[string]meshconditions.Finding +} + +// Say raises (raised) or clears the finding of a kind about a subject. Its id is the seat's and the subject's, each escaped +// as a key's name is (design 47 G A-14); the kind stays the bundle's own word. +func (k *Findings) Say(kind, subject, words string, raised bool) { + id := keyparts.EscapeName(packageseats.Self) + "." + keyparts.EscapeName(subject) + k.mu.Lock() + defer k.mu.Unlock() + key := views.FindingKey("seat", id, kind) + if !raised { + delete(k.open, key) + if err := k.Writer.Clear(context.Background(), "seat", id, kind, "it no longer holds"); err != nil { + k.Logf("the finding %s could not be cleared: %v", key, err) + } + return + } + at := time.Now() + if k.Now != nil { + at = k.Now() + } + f := meshconditions.Finding{Scope: "seat", ID: id, Kind: kind, Subject: meshconditions.Subject{Scope: "seat", ID: id}, + Severity: meshconditions.FindingSeverityWarning, Summary: words, Headline: words, Explanation: words, + Source: kind, Observer: packageseats.Self, At: at.UTC()} + if k.open == nil { + k.open = map[string]meshconditions.Finding{} + } + k.open[key] = f + if err := k.Writer.Write(context.Background(), f); err != nil { + k.Logf("the finding %s could not be written: %v", key, err) + } +} + +// Observed is the findings open now, for the writer's statement (views.WriterConfig.Observe). +func (k *Findings) Observed(context.Context) (views.Observed[meshconditions.Finding], error) { + k.mu.Lock() + defer k.mu.Unlock() + var out views.Observed[meshconditions.Finding] + for _, f := range k.open { + out.Findings = append(out.Findings, f) + } + return out, nil +} diff --git a/modules/gitea/cmd/package-registry/forge.go b/modules/gitea/cmd/package-registry/forge.go index f752a4116..82fb0e69a 100644 --- a/modules/gitea/cmd/package-registry/forge.go +++ b/modules/gitea/cmd/package-registry/forge.go @@ -37,7 +37,7 @@ type Forge struct { // key in the module's findings state (packages-account.ts PACKAGES_TOKEN_REFUSED watches that key). const ( PackagesTokenRefused = "packages-token-refused" - PackagesTokenRefusedKey = "seat.gitea.packages-token-refused_write" + PackagesTokenRefusedKey = "seat.gitea.write.packages-token-refused" ) // wrote is a write's answer, said on Refused: a 401 raises the finding, a write the forge took clears it. @@ -86,7 +86,7 @@ func ForgeFromEnv() (*Forge, error) { } } f := &Forge{Client: c, Writer: &w, Owner: os.Getenv("MESH_NPM_OWNER"), Refused: func(raised bool) { - finding(PackagesTokenRefused, "write", "the forge refused the package seats' write token; the module mints it afresh", raised) + findings.Say(PackagesTokenRefused, "write", "the forge refused the package seats' write token; the module mints it afresh", raised) }} if f.Owner == "" { missing = append(missing, "MESH_NPM_OWNER") diff --git a/modules/gitea/cmd/package-registry/main.go b/modules/gitea/cmd/package-registry/main.go index 0869cfca1..ebcbed3ab 100644 --- a/modules/gitea/cmd/package-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -13,7 +13,9 @@ import ( "os" "strings" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" "gitea/internal/packageseats" ) @@ -23,7 +25,26 @@ import ( // of the module's seats as their verbs. The README names these two, and a test holds the two together. var ToolNames = []string{"go_packages", "go_retention", "go_incoming", "npm_packages", "npm_retention", "npm_incoming"} +// findings keeps this bundle's open findings for its writer (design 47 G §689). +var findings = &Findings{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, f+"\n", a...) }} + func main() { + // The bundle's findings go through the module's writer (design 47 G A-16). + w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Module: packageseats.Self, State: stdio.State("findings"), Kinds: findingKinds, Observe: findings.Observed}) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + findings.Writer = w + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := w.Start(context.Background()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }() forge := func() (*Forge, error) { return ForgeFromEnv() } holder := func() (*packageseats.Holder, error) { return busHolder(forge) } go followIdle(holder) diff --git a/modules/gitea/cmd/package-registry/seats.go b/modules/gitea/cmd/package-registry/seats.go index 4ec5770e8..336e473c7 100644 --- a/modules/gitea/cmd/package-registry/seats.go +++ b/modules/gitea/cmd/package-registry/seats.go @@ -22,14 +22,10 @@ import ( "io" "net/http" "net/url" - "os" "path" "strings" - "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" - meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" goreg "git.novox.be/novox/mesh-delivery/contracts/go/seats/gopackageregistry" npmreg "git.novox.be/novox/mesh-delivery/contracts/go/seats/npmpackageregistry" "git.novox.be/novox/mesh-delivery/sdk/parts" @@ -249,21 +245,6 @@ func sayOn(seat, event string, o packageseats.Outcome) error { return stdio.SeatPublish("mesh.seat."+seat+".event."+event, body, "") } -func finding(kind, subject, words string, raised bool) { - key := meshconditionsclient.Key(meshconditionsclient.ScopeSeat, packageseats.Self, kind+"."+strings.NewReplacer(".", "_", "@", "_", "/", "_").Replace(subject)) - if !raised { - _ = stdio.State(meshconditions.StateFindings).Delete(key) - return - } - f := meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: key, Kind: kind, Ref: key[strings.LastIndex(key, ".")+1:], - Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeSeat, ID: packageseats.Self}, Severity: meshconditionsclient.Warning, - Summary: words, Headline: words, Explanation: words, Resolved: "cleared once it no longer holds", - Source: kind, Observer: packageseats.Self, At: time.Now().UTC()} - if _, err := stdio.State(meshconditions.StateFindings).Put(key, f); err != nil { - fmt.Fprintf(os.Stderr, "the finding %s could not be written: %v\n", key, err) - } -} - // recordAct records a hand act through the seat mesh-hand-acts alone. func recordAct(q meshhandactsclient.Request) error { _, err := meshhandactsclient.Recorder{Ask: stdio.Ask}.Record(q) @@ -277,7 +258,7 @@ var boundCap = parts.BusCap(packageseats.IncomingObjects) func busHolder(forge func() (*Forge, error)) (*packageseats.Holder, error) { return &packageseats.Holder{Objects: parts.BusObjects(packageseats.IncomingObjects), Transfers: parts.BusTransfers[packageseats.Opening](packageseats.TransfersState), - Builder: packageseats.Reserved{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: finding, + Builder: packageseats.Reserved{Ask: stdio.Ask}, Store: seatStore{forge}, Say: say, Finding: findings.Say, Cap: boundCap, Record: recordAct, Idle: parts.BusArm(packageseats.IdleState)}, nil } diff --git a/modules/gitea/cmd/package-registry/seats_test.go b/modules/gitea/cmd/package-registry/seats_test.go index 6deb1ab47..13d4fbd0f 100644 --- a/modules/gitea/cmd/package-registry/seats_test.go +++ b/modules/gitea/cmd/package-registry/seats_test.go @@ -19,8 +19,10 @@ import ( "testing" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + keyparts "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" "gitea/internal/forgehttp" + "gitea/internal/packageseats" ) @@ -133,7 +135,7 @@ func TestThePackageSeatsStoreIntoTheForge(t *testing.T) { // and the next write the forge takes clears it (no timer looks at the token). var refused []bool forge.Refused = func(raised bool) { refused = append(refused, raised) } - if k := meshconditionsclient.Key(meshconditionsclient.ScopeSeat, packageseats.Self, PackagesTokenRefused+".write"); k != PackagesTokenRefusedKey { + if k := views.FindingKey("seat", keyparts.EscapeName(packageseats.Self)+".write", PackagesTokenRefused); k != PackagesTokenRefusedKey { t.Errorf("the refusal's key is %s, the watcher watches %s", k, PackagesTokenRefusedKey) } _ = os.WriteFile(writer.TokenFile, []byte("stale\n"), 0o600) diff --git a/modules/gitea/cmd/package-registry/whole_integration_test.go b/modules/gitea/cmd/package-registry/whole_integration_test.go index 5a52ce7a4..ec90b8933 100644 --- a/modules/gitea/cmd/package-registry/whole_integration_test.go +++ b/modules/gitea/cmd/package-registry/whole_integration_test.go @@ -127,7 +127,10 @@ func (p *player) handle(m wire) { p.answer(m.ID, map[string]any{"module": "m", "version": q.Version, "artifacts": []map[string]any{{"artifact": "m:a", "digest": d}}}, nil) case m.Method == "mesh/seat.publish": p.answer(m.ID, map[string]any{}, p.nc.Publish(a.Subject, a.Body)) - case m.Method == "mesh/publish": + case m.Method == "mesh/capabilities": + p.answer(m.ID, map[string]any{"words": []string{"put-sessions", "finding-calls"}}, nil) + case m.Method == "mesh/publish", m.Method == "mesh/subscribe": + // the module's finding writer emits its events and subscribes to findings-asked (design 47 G A-16); none is played p.answer(m.ID, map[string]any{}, nil) case strings.HasPrefix(m.Method, "mesh/objects."): p.objects(m, a.Objects, a.Name, a.Data) @@ -307,7 +310,7 @@ func TestThePackageSeatsWhole(t *testing.T) { return file } cmd := exec.CommandContext(ctx, bin) - cmd.Env = append(os.Environ(), "MESH_GITEA_URL="+srv.URL, "MESH_NPM_OWNER=acme", "MESH_GITEA_TOOLS_USER=mesh-tools", + cmd.Env = append(os.Environ(), "MESH_NODE=novox", "MESH_GITEA_URL="+srv.URL, "MESH_NPM_OWNER=acme", "MESH_GITEA_TOOLS_USER=mesh-tools", "MESH_GITEA_TOKEN_FILE="+kept("token", "reader", "read:package,read:user,write:issue,write:repository for mesh-tools"), "MESH_GITEA_PACKAGES_TOKEN_FILE="+kept("packages-token", "writer", "write:package for mesh-packages")) cmd.Stderr = os.Stderr diff --git a/modules/gitea/go.mod b/modules/gitea/go.mod index d84ba67cf..11e94c655 100644 --- a/modules/gitea/go.mod +++ b/modules/gitea/go.mod @@ -3,7 +3,6 @@ module gitea go 1.26.0 require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/provisioning/go/clients/meshprovisionerclient v0.0.0-00010101000000-000000000000 diff --git a/modules/gitea/packages-account.ts b/modules/gitea/packages-account.ts index ffbfab7b9..ac0ec296b 100644 --- a/modules/gitea/packages-account.ts +++ b/modules/gitea/packages-account.ts @@ -89,7 +89,7 @@ export async function keepPackagesToken(source: MintedToken, url: string, owner } /** The finding the Go bundle raises in the module's findings state when the forge refuses the write token (cmd/package-registry PackagesTokenRefusedKey). */ -export const PACKAGES_TOKEN_REFUSED = "seat.gitea.packages-token-refused_write"; +export const PACKAGES_TOKEN_REFUSED = "seat.gitea.write.packages-token-refused"; /** One change of the findings state, as its watch hands it. */ export interface FindingChange { readonly key: string; readonly op: "put" | "delete" } diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index 46872509f..a571e8538 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -73,14 +73,19 @@ func main() { if m.Node != "" { go m.Start(ctx) findings, err := newFindingsWriter() - if err == nil { - err = findings.Start(ctx) - } if err != nil { logf("the finding writer: %v", err) os.Exit(1) } manifestFindings = views.WriterFindings{Writer: findings} + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := findings.Start(ctx); err != nil { + logf("the finding writer: %v", err) + os.Exit(1) + } + }() } go startKeeper(ctx) // Every state it keeps, said on the bus before it serves (ADR 0495 §2; issue 1790). diff --git a/modules/mesh-build-go/go.mod b/modules/mesh-build-go/go.mod index 87a97efea..a3591514d 100644 --- a/modules/mesh-build-go/go.mod +++ b/modules/mesh-build-go/go.mod @@ -101,7 +101,7 @@ require git.novox.be/novox/mesh-catalog/modules/ssh-client/sshkeys v0.0.0-000101 replace git.novox.be/novox/mesh-catalog/modules/ssh-client/sshkeys => ../ssh-client/sshkeys require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-delivery/contracts/go/clients/gopackageregistryclient v0.0.0-00010101000000-000000000000 // indirect git.novox.be/novox/mesh-delivery/contracts/go/clients/npmpackageregistryclient v0.0.0-00010101000000-000000000000 // indirect diff --git a/modules/mesh-config/cmd/mesh-config/config.go b/modules/mesh-config/cmd/mesh-config/config.go index a745180e4..0b62393dd 100644 --- a/modules/mesh-config/cmd/mesh-config/config.go +++ b/modules/mesh-config/cmd/mesh-config/config.go @@ -86,6 +86,10 @@ type Config struct { Logf func(format string, a ...any) mu sync.Mutex + // Findings is the module's finding writer (design 47 G §689); nil, no finding is said. findMu guards the findings open. + Findings FindingsWriter + findMu sync.Mutex + open map[string]openFinding // withheld is why the newest list of predefined keys was not published, empty when it was (publish). withheld string diff --git a/modules/mesh-config/cmd/mesh-config/findings.go b/modules/mesh-config/cmd/mesh-config/findings.go index 3f9c1482d..42c4d7fd5 100644 --- a/modules/mesh-config/cmd/mesh-config/findings.go +++ b/modules/mesh-config/cmd/mesh-config/findings.go @@ -5,17 +5,19 @@ package main // is gone. The keeper raises and clears the condition of the same key (mesh-conditions' Observers and Homes). import ( + "context" "fmt" "slices" "strings" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "git.novox.be/novox/mesh-sdk/go/keys" + "git.novox.be/novox/mesh-sdk/go/views" ) // FindingsState is the state, meshconditions.StateFindings. -const FindingsState = meshconditions.StateFindings +const FindingsState = "findings" // The kinds of finding mesh-config writes. const ( @@ -31,89 +33,116 @@ const ( FindingRetireHeld = "retirement-held" ) -// findingKey is the key of a finding of kind about a module on a node: the condition's key. -func findingKey(kind, module, node, ref string) string { - id := module +// findingID is the id of a finding about a module on a node: the module, the node when there is one, and the ref +// (`..`), each escaped as a key's name is, since the kind is a fixed word (design 47 G A-14). +func findingID(module, node, ref string) string { + id := keys.EscapeName(module) if node != "" { - id += "." + node + id += "." + keys.EscapeName(node) } - return meshconditionsclient.Key(meshconditionsclient.ScopeModule, id, kind+"-"+ref) + return id + "." + keys.EscapeName(ref) } -func finding(kind, module, node, ref, summary, headline, explanation, needs, resolved, said string, at time.Time) meshconditions.FindingsByKeyValue { - id := module - if node != "" { - id += "." + node - } - return meshconditions.FindingsByKeyValue{V: meshconditionsclient.FindingVersion, Key: findingKey(kind, module, node, ref), Kind: kind, Ref: kind + "-" + ref, - Subject: meshconditions.Subject{Scope: meshconditionsclient.ScopeModule, ID: id, Machine: node}, - Severity: meshconditionsclient.Warning, Resolver: meshconditionsclient.ResolverOperator, Summary: summary, Headline: headline, - Explanation: explanation, Needs: needs, Resolved: resolved, +func finding(kind, module, node, ref, summary, headline, explanation, needs, said string, at time.Time) meshconditions.Finding { + id := findingID(module, node, ref) + return meshconditions.Finding{Scope: "module", ID: id, Kind: kind, + Subject: meshconditions.Subject{Scope: "module", ID: id, Machine: node}, + Severity: meshconditions.FindingSeverityWarning, Resolver: "operator", Summary: summary, Headline: headline, + Explanation: explanation, Needs: needs, Evidence: []meshconditions.Evidence{{At: at.UTC(), Said: said}}, Source: "config", Observer: Module, At: at.UTC()} } // renameRefused is the finding of a move mesh-config refused. -func renameRefused(module, to, from, why string, at time.Time) meshconditions.FindingsByKeyValue { +func renameRefused(module, to, from, why string, at time.Time) meshconditions.Finding { return finding(FindingRenameRefused, module, "", to, fmt.Sprintf("the rename of %s's config value %s to %s is refused: %s", module, from, to, why), fmt.Sprintf("%s's value %s is not moved to %s", module, from, to), fmt.Sprintf("The new version of %s names %s as the new name of its config value %s, and mesh-config refused to move the value: %s. Nothing was moved.", module, to, from, why), fmt.Sprintf("put the cause right, then move it with config_rename_again (module %s, to %s); the details name it.", module, to), - fmt.Sprintf("Resolved: %s's value %s is under %s", module, from, to), why, at) + why, at) } // windowHeld is the finding of a rename window held open past HeldFinding by a node. -func windowHeld(r Rename, node string, at time.Time) meshconditions.FindingsByKeyValue { +func windowHeld(r Rename, node string, at time.Time) meshconditions.Finding { return finding(FindingWindowHeld, r.Module, node, r.To, fmt.Sprintf("%s on %s has held the old config name %s open for %d days", r.Module, node, r.From, int(HeldFinding.Hours()/24)), fmt.Sprintf("%s on %s still runs the old config name", r.Module, node), fmt.Sprintf("The rename of %s's config value %s to %s is not complete on %s: its node-engine has not reported a build that reads %s (it may be away).", r.Module, r.From, r.To, node, r.To), fmt.Sprintf("bring %s back, or drop it from the window with config_rename_close.", node), - fmt.Sprintf("Resolved: %s reads %s on %s", r.Module, r.To, node), fmt.Sprintf("the window opened %s and %s has not reported a build reading %s", r.Nodes[node].Since.UTC().Format(time.RFC3339), node, r.To), at) } -// setFinding writes a finding when it differs from the one kept. -func (c *Config) setFinding(f meshconditions.FindingsByKeyValue) { - if err := meshconditionsclient.CheckFinding(f); err != nil { +// FindingsWriter is the module's finding writer (views.Writer), through which mesh-config writes and clears its findings +// (design 47 G §689). +type FindingsWriter interface { + Write(ctx context.Context, finding meshconditions.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error +} + +// openFinding is a finding written and not yet cleared, with the parts of its id. +type openFinding struct { + f meshconditions.Finding + module, ref string +} + +// setFinding writes a finding when it differs from the one written; the module and ref are the parts of its id. +func (c *Config) setFinding(module, ref string, f meshconditions.Finding) { + if c.Findings == nil { + return + } + if err := views.CheckFinding(f); err != nil { c.logf("[mesh-config] a finding is not valid: %v", err) return } - if e, err := c.Store.Get(FindingsState, f.Key); err == nil && e != nil { - if old, derr := meshconditions.DecodeFindingsByKeyValue(e.Value); derr == nil && old.Summary == f.Summary { - return // what it says has not changed - } + key := views.FindingKey(f.Scope, f.ID, f.Kind) + c.findMu.Lock() + defer c.findMu.Unlock() + if old, ok := c.open[key]; ok && old.f.Summary == f.Summary { + return // what it says has not changed } - if err := c.Store.Put(FindingsState, f.Key, f); err != nil { - c.logf("[mesh-config] the finding %s is not written: %v", f.Key, err) + if err := c.Findings.Write(context.Background(), f); err != nil { + c.logf("[mesh-config] the finding %s is not written: %v", key, err) + return + } + if c.open == nil { + c.open = map[string]openFinding{} + } + c.open[key] = openFinding{f: f, module: module, ref: ref} +} + +// clearFinding clears the findings of a kind and ref about a module, on whatever node but those in keep. +func (c *Config) clearFinding(kind, module, ref string, keep ...string) { + if c.Findings == nil { + return + } + c.findMu.Lock() + defer c.findMu.Unlock() + for k, o := range c.open { + if o.f.Kind != kind || o.ref != ref || o.module != module || + (o.f.Subject.Machine != "" && slices.Contains(keep, o.f.Subject.Machine)) { + continue + } + if err := c.Findings.Clear(context.Background(), o.f.Scope, o.f.ID, o.f.Kind, "the fault is gone"); err != nil { + c.logf("[mesh-config] the finding %s is not cleared: %v", k, err) + continue + } + delete(c.open, k) } } -// clearFinding deletes the findings of a kind and ref about a module, on whatever node but those in keep. -func (c *Config) clearFinding(kind, module, ref string, keep ...string) { - keys, err := c.Store.Keys(FindingsState) - if err != nil { - return - } - want := kind + "-" + ref - for _, k := range keys { - e, err := c.Store.Get(FindingsState, k) - if err != nil || e == nil { - continue - } - f, err := meshconditions.DecodeFindingsByKeyValue(e.Value) - if err != nil || f.Ref != want || !(f.Subject.ID == module || strings.HasPrefix(f.Subject.ID, module+".")) || - (f.Subject.Machine != "" && slices.Contains(keep, f.Subject.Machine)) { - continue - } - if err := c.Store.Delete(FindingsState, k); err != nil { - c.logf("[mesh-config] the finding %s is not cleared: %v", k, err) - } +// FindingsObserved is the findings open now, for the writer's statement (views.WriterConfig.Observe). +func (c *Config) FindingsObserved(context.Context) (views.Observed[meshconditions.Finding], error) { + c.findMu.Lock() + defer c.findMu.Unlock() + var out views.Observed[meshconditions.Finding] + for _, o := range c.open { + out.Findings = append(out.Findings, o.f) } + return out, nil } // retirementHeld is the finding of a retirement that waits seven days for nodes that still run a build reading the value. -func retirementHeld(e Retiring, at time.Time) meshconditions.FindingsByKeyValue { +func retirementHeld(e Retiring, at time.Time) meshconditions.Finding { node := "" if len(e.Nodes) > 0 { node = e.Nodes[0] @@ -123,17 +152,15 @@ func retirementHeld(e Retiring, at time.Time) meshconditions.FindingsByKeyValue fmt.Sprintf("%s's %s cannot be retired yet", e.Module, e.Key), fmt.Sprintf("No manifest reads %s's %s any more, but %s still run a build that does, so the operator is not asked to remove it.", e.Module, e.Key, strings.Join(e.Nodes, ", ")), "bring those nodes up to date, or cancel the retirement with config_retiring.", - fmt.Sprintf("Resolved: no node reads %s's %s", e.Module, e.Key), "retiring since "+e.Since.UTC().Format(time.RFC3339)+", still read on "+strings.Join(e.Nodes, ", "), at) } // retireUnanswered is the finding of a retirement asked twice and not answered. -func retireUnanswered(e Retiring, at time.Time) meshconditions.FindingsByKeyValue { +func retireUnanswered(e Retiring, at time.Time) meshconditions.Finding { return finding(FindingRetireUnanswered, e.Module, "", e.Key, fmt.Sprintf("the operator did not answer the ask to remove %s's %s, asked twice", e.Module, e.Key), fmt.Sprintf("%s's %s waits for an answer", e.Module, e.Key), fmt.Sprintf("No manifest reads %s's %s any more. The operator was asked twice whether to remove it and did not answer, so it stays.", e.Module, e.Key), "answer the ask again with config_retiring ask-again, or cancel it.", - fmt.Sprintf("Resolved: %s's %s is answered or read again", e.Module, e.Key), "asked "+e.Asked.UTC().Format(time.RFC3339)+" and once more", at) } diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index 16a839755..e45094a12 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -18,10 +18,11 @@ import ( "strings" "time" - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" ) // Seat is the role this module holds, once for the mesh, and Verb its one verb. @@ -120,13 +121,30 @@ func (runtimeAsks) Warranted(ask string) error { func main() { c := &Config{Store: runtimeStore{}, Logf: logf, Asks: runtimeAsks{}, Forge: toolForge{ask: stdio.Ask}, Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}} + // The module's one writer (design 47 G §689, A-16): the renames' and retirements' findings and the views reader's go through it. + w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Module: Module, State: stdio.State(FindingsState), Observe: c.FindingsObserved, + Kinds: []string{FindingRenameRefused, FindingWindowHeld, FindingRetireUnanswered, FindingRetireHeld}}) + if err != nil { + logf("%v", err) + os.Exit(1) + } + c.Findings = w moved := make(chan struct{}, 1) theViews = newOwnerViews(stdio.Ask, func() { select { case moved <- struct{}{}: default: } - }, meshconditionsclient.FindingWriter{State: stdio.State(FindingsState)}) + }, views.WriterFindings{Writer: w}) + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := w.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } + }() theViews.Start(context.Background()) go resolving(c, moved) go hearing(c) diff --git a/modules/mesh-config/cmd/mesh-config/renames.go b/modules/mesh-config/cmd/mesh-config/renames.go index a22de1506..f7338a38a 100644 --- a/modules/mesh-config/cmd/mesh-config/renames.go +++ b/modules/mesh-config/cmd/mesh-config/renames.go @@ -155,7 +155,7 @@ func (c *Config) rename(s *Sources, held map[string]Rename, trunk ModuleDeclarat nodes := nodesPlaced(s, module) var said []string refuse := func(why string) []string { - c.setFinding(renameRefused(module, to, from, why, now)) + c.setFinding(module, to, renameRefused(module, to, from, why, now)) if r, ok := held[key]; ok { s.Renames[key] = r } @@ -488,7 +488,7 @@ func (c *Config) sweepRenameFindings(s *Sources, now time.Time) { } c.clearFinding(FindingWindowHeld, r.Module, r.To, held...) for _, node := range held { - c.setFinding(windowHeld(r, node, now)) + c.setFinding(r.Module, r.To, windowHeld(r, node, now)) } } } diff --git a/modules/mesh-config/cmd/mesh-config/renames_test.go b/modules/mesh-config/cmd/mesh-config/renames_test.go index e4b5d3a27..7f7c6557d 100644 --- a/modules/mesh-config/cmd/mesh-config/renames_test.go +++ b/modules/mesh-config/cmd/mesh-config/renames_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -12,6 +13,7 @@ import ( meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" + "git.novox.be/novox/mesh-sdk/go/views" ) // fakeDelivery is the delivery parts as mesh-config asks them: each node's have, whether a node-engine states what it @@ -78,20 +80,20 @@ func aRenamingMesh(t *testing.T) (*Config, *memStore, *fakeDelivery, *time.Time) } clock := time.Date(2026, 11, 4, 10, 0, 0, 0, time.UTC) d := &fakeDelivery{haves: map[string]string{"g14.player": fp1, "novox.player": fp1}, notSays: map[string]bool{}} - c := &Config{Store: m, Now: func() time.Time { return clock }, Logf: t.Logf, Delivery: d} + c := &Config{Store: m, Now: func() time.Time { return clock }, Logf: t.Logf, Delivery: d, Findings: m} if _, err := c.Reconcile(); err != nil { t.Fatal(err) } return c, m, d, &clock } -func findingsOf(t *testing.T, m *memStore, kind string) []meshconditions.FindingsByKeyValue { +func findingsOf(t *testing.T, m *memStore, kind string) []meshconditions.Finding { t.Helper() - var out []meshconditions.FindingsByKeyValue + var out []meshconditions.Finding for _, k := range m.keysOf(FindingsState) { e, _ := m.Get(FindingsState, k) - f, err := meshconditions.DecodeFindingsByKeyValue(e.Value) - if err != nil { + var f meshconditions.Finding + if err := json.Unmarshal(e.Value, &f); err != nil { t.Fatalf("finding %s: %v", k, err) } if f.Kind == kind { @@ -435,3 +437,12 @@ func TestARetirementDeclinedUnansweredCancelledOrASecret(t *testing.T) { } _ = fmt.Sprint } + +// Write and Clear are the module's finding writer in a test: a finding is kept in its key and deleted as views.Writer does. +func (m *memStore) Write(_ context.Context, f meshconditions.Finding) error { + return m.Put(FindingsState, views.FindingKey(f.Scope, f.ID, f.Kind), f) +} + +func (m *memStore) Clear(_ context.Context, scope, id, kind, _ string) error { + return m.Delete(FindingsState, views.FindingKey(scope, id, kind)) +} diff --git a/modules/mesh-config/cmd/mesh-config/retiring.go b/modules/mesh-config/cmd/mesh-config/retiring.go index dc010dd74..6f6bd2c16 100644 --- a/modules/mesh-config/cmd/mesh-config/retiring.go +++ b/modules/mesh-config/cmd/mesh-config/retiring.go @@ -235,7 +235,7 @@ func (c *Config) advanceRetiring(s Sources, e *Retiring) []string { return []string{fmt.Sprintf("the retirement %s is not kept: %v", key, err)} } if now.Sub(e.Since) >= HeldFinding { - c.setFinding(retirementHeld(*e, now)) + c.setFinding(e.Module, e.Key, retirementHeld(*e, now)) } return []string{fmt.Sprintf("%s's %s is not read by any manifest, and %s still run a build that reads it", e.Module, e.Key, strings.Join(e.Nodes, ", "))} @@ -410,7 +410,7 @@ func (c *Config) decideRetirement(e *Retiring, now time.Time) []string { if w.Outcome != occ.OutcomeChosen { out := keep(RetireUnanswered, "the ask "+string(w.Outcome)+"; nothing is removed") if e.AskedAgain { - c.setFinding(retireUnanswered(*e, now)) + c.setFinding(e.Module, e.Key, retireUnanswered(*e, now)) } return out } diff --git a/modules/mesh-config/cmd/mesh-config/whole_integration_test.go b/modules/mesh-config/cmd/mesh-config/whole_integration_test.go index a8d253d6e..407535096 100644 --- a/modules/mesh-config/cmd/mesh-config/whole_integration_test.go +++ b/modules/mesh-config/cmd/mesh-config/whole_integration_test.go @@ -726,7 +726,8 @@ func TestConfigWholeRenamesAndRetiresAgainstTheBus(t *testing.T) { keys, _ := (func() ([]string, error) { kv, _ := p.kv(FindingsState); return kv.Keys(p.ctx) })() for _, k := range keys { raw, _ := p.get(FindingsState, k) - if f, err := meshconditions.DecodeFindingsByKeyValue(raw); err == nil && f.Kind == FindingRenameRefused && strings.Contains(f.Summary, "dock") { + var f meshconditions.Finding + if err := json.Unmarshal(raw, &f); err == nil && f.Kind == FindingRenameRefused && strings.Contains(f.Summary, "dock") { return true } } diff --git a/modules/mesh-config/go.mod b/modules/mesh-config/go.mod index fc6a03485..ced63103d 100644 --- a/modules/mesh-config/go.mod +++ b/modules/mesh-config/go.mod @@ -9,7 +9,6 @@ require ( git.novox.be/novox/mesh-contracts/go/seats/meshconfig v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/meshmoduleassigner v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/go/seats/nodeengine v0.0.0-00010101000000-000000000000 - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient v0.0.0-00010101000000-000000000000 diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 5220c92d1..8ffad9c06 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -246,10 +246,14 @@ func main() { } return e.Value, true, nil }) - if err := findings.Start(context.Background()); err != nil { - logf("%v", err) - os.Exit(1) - } + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := findings.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } + }() v.Start(context.Background()) h := &Holder{Store: Watching(s, ws), Asks: runtimeAsks{}, Now: time.Now, Logf: logf} h.Compose = FromCompositions(h.Store) // the node-engines' published compositions, from the copy (ADR 0593 §8) diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index c6dc697f9..171e66138 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -8,8 +8,9 @@ package main import ( - "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" + "context" "encoding/json" "errors" @@ -118,12 +119,25 @@ func watchLeavers(env *Env) bool { return true } if assignmentsView == nil { + // The vault's finding writer (design 47 G §689, A-16): it states the assignments view's findings and observes none of + // its own, and names no secret. + w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ + Module: Seat, State: stdio.State(FindingsState), + Observe: func(context.Context) (views.Observed[meshconditions.Finding], error) { + return views.Observed[meshconditions.Finding]{}, nil + }}) + if err == nil { + err = w.Start(context.Background()) + } + if err != nil { + return false + } assignmentsView = newAssignmentsView(stdio.Ask, func(key string) { if node, ok := leaverKey(key); ok { env.Vault.NodeLeft(node) } }, env.Vault.AssignmentsChanged, - meshconditionsclient.FindingWriter{State: stdio.State(FindingsState)}) + views.WriterFindings{Writer: w}) beginAssignmentsView(assignmentsView) } return true diff --git a/modules/mesh-vault/cmd/mesh-vault/vault_test.go b/modules/mesh-vault/cmd/mesh-vault/vault_test.go index 68def3bd5..d953d15db 100644 --- a/modules/mesh-vault/cmd/mesh-vault/vault_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/vault_test.go @@ -198,7 +198,7 @@ func TestNoBucketHoldsASecret(t *testing.T) { code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Watch(", "", 1)) code = []byte(strings.Replace(string(code), "stdio.State(WantedRead).Get(", "", 1)) // And its findings (ADR 0615 §6): the finding of a first view unanswered, written and deleted, never a secret. - code = []byte(strings.Replace(string(code), "meshconditionsclient.FindingWriter{State: stdio.State(FindingsState)}", "", 1)) + code = []byte(strings.Replace(string(code), "State: stdio.State(FindingsState)", "", 1)) // And its outcome entries (novox/hq ADR 0570 clauses 3, 6): a value's name and generation, never a value; and // one read of the seat's provisions, which hold none. for _, call := range []string{"Get(", "Put(", "Delete(", "Keys("} { diff --git a/modules/mesh-vault/go.mod b/modules/mesh-vault/go.mod index b09f9aeb8..b9bd08189 100644 --- a/modules/mesh-vault/go.mod +++ b/modules/mesh-vault/go.mod @@ -38,11 +38,10 @@ require ( ) require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-networking/modules/mesh-wireguard/go v0.0.0-00010101000000-000000000000 ) require ( - git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 // indirect + git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions v0.0.0-00010101000000-000000000000 git.novox.be/novox/mesh-networking/modules/mesh-connectivity-contracts/go/seats/theprivatenetwork v0.0.0-00010101000000-000000000000 // indirect ) diff --git a/modules/mesh-watcher/cmd/mesh-watcher/main.go b/modules/mesh-watcher/cmd/mesh-watcher/main.go index b4152ba21..d3a34095c 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/main.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/main.go @@ -84,10 +84,14 @@ func main() { } assignments.Findings = views.WriterFindings{Writer: findings} assignments.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) - if err := findings.Start(context.Background()); err != nil { - logf("%v", err) - os.Exit(1) - } + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := findings.Start(context.Background()); err != nil { + logf("%v", err) + os.Exit(1) + } + }() go run(w, l, node, assignments) if err := stdio.Serve("", append(tools(w, l), assignments.Tools("watcher")...)); err != nil { logf("%v", err) diff --git a/modules/minio/cmd/object-store/main.go b/modules/minio/cmd/object-store/main.go index 6b9f7712b..193c790bb 100644 --- a/modules/minio/cmd/object-store/main.go +++ b/modules/minio/cmd/object-store/main.go @@ -44,10 +44,14 @@ func main() { fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) os.Exit(1) } - if err := findings.Start(context.Background()); err != nil { - fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) - os.Exit(1) - } + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := findings.Start(context.Background()); err != nil { + fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) + os.Exit(1) + } + }() seats.Start(context.Background()) if err := stdio.Serve("", append(tools(s), seats.Tools("object_store")...)); err != nil { fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) diff --git a/modules/nfs-server/cmd/nfs-server/main.go b/modules/nfs-server/cmd/nfs-server/main.go index 17feebc89..3a85efeb4 100644 --- a/modules/nfs-server/cmd/nfs-server/main.go +++ b/modules/nfs-server/cmd/nfs-server/main.go @@ -63,10 +63,14 @@ func main() { seats.Findings = views.WriterFindings{Writer: w} seats.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) seats.Start(context.Background()) - if err := w.Start(context.Background()); err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } + // the runtime answers the writer's subscription only once Serve has its channel + go func() { + <-stdio.Served() + if err := w.Start(context.Background()); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + }() go ShareExports(m) if err := stdio.Serve("", append(Tools(m), seats.Tools("nfs_server")...)); err != nil { fmt.Fprintln(os.Stderr, err) -- 2.54.0 From 05fcebf9c5b8d31967ffda5ae593b9a3a995904f Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:58:08 +0100 Subject: [PATCH 092/243] COND A-22: the keeper holds findings as the contract's generic Finding and takes its own through the writer's wire form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G §434, §437, §439, A-3, A-22 (G-20) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-conditions/asks.go | 7 ++-- .../cmd/mesh-conditions/events.go | 41 ++++++++++-------- .../cmd/mesh-conditions/hold.go | 2 +- .../cmd/mesh-conditions/keeper.go | 18 ++++---- .../cmd/mesh-conditions/keeper_test.go | 36 +++++++++++++--- .../cmd/mesh-conditions/main.go | 42 ++++++++++++++----- .../cmd/mesh-conditions/model.go | 10 ++--- 7 files changed, 103 insertions(+), 53 deletions(-) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/asks.go b/modules/mesh-conditions/cmd/mesh-conditions/asks.go index b9bff6d23..f376a264e 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/asks.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/asks.go @@ -5,7 +5,6 @@ package main // `findings-unstated`, cleared by its next `findings-stated`. import ( - "encoding/json" "errors" "fmt" "sort" @@ -144,17 +143,17 @@ func (k *Keeper) AskPassed(bound string) error { return nil } now := k.Now().UTC() - f := seat.FindingsByKeyValue{V: cc.FindingVersion, Key: unstatedKey(observer, node), Kind: KindFindingsUnstated, + f := seat.Finding{Scope: cc.ScopeModule, ID: observer + "." + node, Kind: KindFindingsUnstated, Since: now, Subject: seat.Subject{Scope: cc.ScopeModule, ID: observer + "." + node, Machine: node}, Severity: cc.Warning, Summary: fmt.Sprintf("%s on %s stated no findings within %s of the ask", observer, node, k.askedWithin()), Headline: observer + " on " + node + " did not state its findings", Explanation: "The keeper asked " + observer + " on " + node + " to state its findings and heard nothing; its conditions stand as they were.", Source: "findings-asked", Observer: Module, At: now, Evidence: []seat.Evidence{{At: now, Said: "ask " + ask + " unanswered"}}} - raw, err := json.Marshal(f) + key, raw, err := storedFinding(f) if err != nil { return err } - return k.Finding(Module, f.Key, raw, false) + return k.Finding(Module, key, raw, false) } func unstatedKey(observer, node string) string { diff --git a/modules/mesh-conditions/cmd/mesh-conditions/events.go b/modules/mesh-conditions/cmd/mesh-conditions/events.go index f2be01310..458469f9f 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/events.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/events.go @@ -94,8 +94,7 @@ func (k *Keeper) FindingWritten(observer, node string, body []byte) error { // take holds one finding an observer says on a node, and raises or changes its condition. func (k *Keeper) take(observer, node string, f seat.Finding) error { - v := storedFinding(f) - value, err := json.Marshal(v) + key, value, err := storedFinding(f) if err != nil { return err } @@ -104,15 +103,15 @@ func (k *Keeper) take(observer, node string, f seat.Finding) error { if k.keyNode[observer] == nil { k.keyNode[observer] = map[string]string{} } - k.keyNode[observer][v.Key] = node - named := k.view(observer, node).Pending[v.Key] - delete(k.view(observer, node).Pending, v.Key) + k.keyNode[observer][key] = node + named := k.view(observer, node).Pending[key] + delete(k.view(observer, node).Pending, key) k.mu.Unlock() if !named.IsZero() { - _ = k.Store.Delete(BoundsState, pendingKey(observer, node, v.Key)) + _ = k.Store.Delete(BoundsState, pendingKey(observer, node, key)) } k.saveView(observer, node) - return k.Finding(observer, v.Key, value, false) + return k.Finding(observer, key, value, false) } // FindingCleared clears the condition of the finding an observer no longer observes. @@ -145,20 +144,26 @@ func (k *Keeper) FindingCleared(observer, node string, body []byte) error { return k.Finding(observer, key, nil, true) } -// storedFinding is the event's finding as the keeper holds one. -func storedFinding(f seat.Finding) seat.FindingsByKeyValue { +// storedFinding is the event's finding as the keeper holds one, with its key: the finding as the contract has it (design 47 +// G §437, A-22), under the key its scope, id and ref or kind make. +func storedFinding(f seat.Finding) (string, []byte, error) { + if f.Evidence == nil { + f.Evidence = []seat.Evidence{} + } + if f.Since.IsZero() { + f.Since = f.At + } + value, err := json.Marshal(f) + return findingKey(f), value, err +} + +// findingKey is the key a finding is held under: its scope, its id, and its ref where it has one, else its kind (ADR 0564). +func findingKey(f seat.Finding) string { token := f.Ref if token == "" { token = f.Kind } - evidence := f.Evidence - if evidence == nil { - evidence = []seat.Evidence{} - } - return seat.FindingsByKeyValue{V: cc.FindingVersion, Key: cc.Key(f.Scope, f.ID, token), Kind: f.Kind, Subject: f.Subject, - Severity: f.Severity, Summary: f.Summary, Headline: f.Headline, Explanation: f.Explanation, Needs: f.Needs, - Actions: f.Actions, Evidence: evidence, Source: f.Source, Resolver: f.Resolver, Observer: f.Observer, Owner: f.Owner, - Ref: f.Ref, Run: f.Run, WaitingOn: f.WaitingOn, At: f.At} + return cc.Key(f.Scope, f.ID, token) } // MachinePresence takes the watchman's word that a machine is away or back (design 47 G §389): an observer on a machine @@ -319,7 +324,7 @@ func (k *Keeper) FindingsStated(observer, node string, body []byte) error { v.Asks = map[string]time.Time{} named := map[string]bool{} for _, f := range s.Findings { - named[storedFinding(f).Key] = true + named[findingKey(f)] = true } oldPending := v.Pending v.Pending = map[string]time.Time{} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/hold.go b/modules/mesh-conditions/cmd/mesh-conditions/hold.go index 8c9247581..14d787778 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/hold.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/hold.go @@ -88,7 +88,7 @@ func (k *Keeper) motion(node, module string) (motion.Motion, motion.Want) { } // holdFor says whether key is held now, and as what. Called with mu held. -func (k *Keeper) holdFor(key string, f *seat.FindingsByKeyValue, open bool) (held, bool) { +func (k *Keeper) holdFor(key string, f *seat.Finding, open bool) (held, bool) { if open || f == nil { return held{}, false } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go index 4b600e93c..f1aeb4a3a 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper.go @@ -111,7 +111,7 @@ type Keeper struct { AskBound time.Duration mu sync.Mutex - findings map[string]map[string]seat.FindingsByKeyValue + findings map[string]map[string]seat.Finding blind map[string]string attempts map[string]meshhealers.AttemptsByAttemptValue wants map[string]motion.Want @@ -157,7 +157,7 @@ type Keeper struct { func (k *Keeper) init() { if k.findings == nil { - k.findings, k.blind, k.attempts = map[string]map[string]seat.FindingsByKeyValue{}, map[string]string{}, + k.findings, k.blind, k.attempts = map[string]map[string]seat.Finding{}, map[string]string{}, map[string]meshhealers.AttemptsByAttemptValue{} k.wants, k.haves, k.stuck, k.held, k.cleared = map[string]motion.Want{}, map[string]motion.Have{}, map[string]string{}, map[string]held{}, map[string]clearedMemo{} @@ -180,7 +180,7 @@ func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error k.mu.Lock() k.init() if k.findings[observer] == nil { - k.findings[observer] = map[string]seat.FindingsByKeyValue{} + k.findings[observer] = map[string]seat.Finding{} } if deleted { delete(k.findings[observer], key) @@ -191,9 +191,9 @@ func (k *Keeper) Finding(observer, key string, value []byte, deleted bool) error k.Logf("%s's finding %s is not one the keeper can raise, so it is left as it stood: %v", observer, key, err) return nil } - if f.Key != key { + if fk := findingKey(f); fk != key { k.mu.Unlock() - k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, f.Key) + k.Logf("%s's finding under %s names the key %s, so it is left as it stood", observer, key, fk) return nil } k.findings[observer][key] = f @@ -314,7 +314,7 @@ func (k *Keeper) settleAll(keys []string) error { // wanted is what the inputs say key should be: a finding of an observer. Nil when nothing says it; nothing is carried // from the controller's bucket (issue 1126). -func (k *Keeper) wanted(key string) *seat.FindingsByKeyValue { +func (k *Keeper) wanted(key string) *seat.Finding { observers := make([]string, 0, len(k.findings)) for o := range k.findings { observers = append(observers, o) @@ -390,7 +390,7 @@ func (k *Keeper) settleOnce(key string) error { } // derive is the condition key should be now, from what was open and what the inputs say. -func (k *Keeper) derive(key string, was Condition, open bool, f *seat.FindingsByKeyValue, now time.Time) Condition { +func (k *Keeper) derive(key string, was Condition, open bool, f *seat.Finding, now time.Time) Condition { var c Condition switch { case open: @@ -430,10 +430,10 @@ func (k *Keeper) derive(key string, was Condition, open bool, f *seat.FindingsBy // fromFinding is what the condition says of its finding: its words, subject, severity and source, and its evidence // newest first (clause 2). A healer's escalation is not undone by the finding saying a warning again. -func (k *Keeper) fromFinding(c *Condition, f seat.FindingsByKeyValue, now time.Time) { +func (k *Keeper) fromFinding(c *Condition, f seat.Finding, now time.Time) { escalated := c.Escalated() c.Kind, c.Subject, c.Summary, c.Source = f.Kind, f.Subject, f.Summary, f.Source - c.Headline, c.Explanation, c.Resolved, c.Needs, c.Actions = f.Headline, f.Explanation, f.Resolved, f.Needs, f.Actions + c.Headline, c.Explanation, c.Needs, c.Actions = f.Headline, f.Explanation, f.Needs, f.Actions if !(escalated && f.Severity != cc.Urgent) { c.Severity = f.Severity } diff --git a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go index 0578a1dd9..a4a3728a0 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/keeper_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "encoding/json" "fmt" nodeengine "git.novox.be/novox/mesh-contracts/go/seats/nodeengine" @@ -17,6 +18,7 @@ import ( seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/views" ) // fakeStore is the bus's states in memory, with revisions, as the runtime answers them. @@ -162,12 +164,11 @@ func (r *rig) open(key string) (Condition, bool) { } // encodeFinding writes a finding as an observer does: the seat's layout, checked. -func encodeFinding(f seat.FindingsByKeyValue) ([]byte, error) { - f.V = cc.FindingVersion +func encodeFinding(f seat.Finding) ([]byte, error) { if f.Evidence == nil { f.Evidence = []seat.Evidence{} } - if err := cc.CheckFinding(f); err != nil { + if err := views.CheckFinding(f); err != nil { return nil, err } return json.Marshal(f) @@ -176,14 +177,14 @@ func encodeFinding(f seat.FindingsByKeyValue) ([]byte, error) { // finding is an observer's finding as mesh-sdk's layout writes it (encodeFinding). func finding(t *testing.T, observer, scope, id, kind, source string, sev string, at time.Time) (string, []byte) { t.Helper() - f := seat.FindingsByKeyValue{Key: cc.Key(scope, id, kind), Kind: kind, Subject: seat.Subject{Scope: scope, ID: id}, + f := seat.Finding{Scope: scope, ID: id, Since: at, Kind: kind, Subject: seat.Subject{Scope: scope, ID: id}, Severity: sev, Summary: kind + " of " + id, Headline: kind, Explanation: "It is " + kind + ".", Source: source, Observer: observer, At: at, Evidence: []seat.Evidence{{At: at, Said: kind}}} raw, err := encodeFinding(f) if err != nil { t.Fatal(err) } - return f.Key, raw + return cc.Key(scope, id, kind), raw } func mustJSON(t *testing.T, v any) []byte { @@ -694,3 +695,28 @@ func TestTheKeeperSetsItsViewFromWhatAnObserverStates(t *testing.T) { } } } + +// The keeper's own finding goes in through the generic writer's wire form and is held as the contract's Finding, one that is +// not a finding the contract allows is refused, and a clear removes it (design 47 G A-22, §687). +func TestTheKeepersOwnFindingIsTheContractsFindingByItsWireForm(t *testing.T) { + r := newRig() + own := ownFindings{r.k} + wire := map[string]any{"kind": "view-unanswered", "scope": "module", "id": "mesh-conditions.assigner.wants", "severity": "warning", + "source": "views", "subject": map[string]any{"scope": "module", "id": Module}, "summary": "no first view", "at": r.now} + if err := own.WriteWire(context.Background(), wire); err != nil { + t.Fatal(err) + } + key := cc.Key("module", "mesh-conditions.assigner.wants", "view-unanswered") + if c, ok := r.open(key); !ok || c.Summary != "no first view" || c.Kind != "view-unanswered" { + t.Fatalf("the finding raised no condition: %+v %v", c, ok) + } + if err := own.WriteWire(context.Background(), map[string]any{"scope": "module", "id": "x", "kind": "a.b", "severity": "warning"}); err == nil { + t.Fatal("a kind with a dot was taken") + } + if err := own.Clear(context.Background(), "module", "mesh-conditions.assigner.wants", "view-unanswered", "no longer observed"); err != nil { + t.Fatal(err) + } + if _, ok := r.open(key); ok { + t.Fatal("the condition stands after its finding was cleared") + } +} diff --git a/modules/mesh-conditions/cmd/mesh-conditions/main.go b/modules/mesh-conditions/cmd/mesh-conditions/main.go index 90fd771f6..2a72170f7 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/main.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/main.go @@ -18,6 +18,7 @@ import ( "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshwantsclient" catseat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" @@ -170,24 +171,45 @@ func NewViews(k *Keeper) *views.Reader { return v } -// ownFindings takes the keeper's own finding straight into its picture of findings. +// ownFindings takes the keeper's own finding straight into its picture of findings. It is the WireWriter a views reader +// writes through (design 47 G §687, A-22): the finding's wire form, read as the contract's Finding, as an observer's is. type ownFindings struct{ k *Keeper } -func (o ownFindings) Put(f views.Finding) error { - v := meshconditionsclient.FindingFor(f) - if err := meshconditionsclient.CheckFinding(v); err != nil { - return err - } - value, err := json.Marshal(v) +func (o ownFindings) WriteWire(_ context.Context, wire map[string]any) error { + raw, err := json.Marshal(wire) if err != nil { return err } - return o.k.Finding(Module, v.Key, value, false) + var f seat.Finding + if err := json.Unmarshal(raw, &f); err != nil { + return fmt.Errorf("the keeper's own finding is not readable: %w", err) + } + f.Observer = Module + if f.At.IsZero() { + f.At = o.k.Now().UTC() + } + if f.Since.IsZero() { + f.Since = f.At + } + if f.Evidence == nil { + f.Evidence = []seat.Evidence{} + } + if err := views.CheckFinding(f); err != nil { + return err + } + key, value, err := storedFinding(f) + if err != nil { + return err + } + return o.k.Finding(Module, key, value, false) } -func (o ownFindings) Delete(f views.Finding) error { - return o.k.Finding(Module, meshconditionsclient.Key(meshconditionsclient.ScopeModule, f.ID, f.Kind), nil, true) + +func (o ownFindings) Clear(_ context.Context, scope, id, kind, _ string) error { + return o.k.Finding(Module, meshconditionsclient.Key(scope, id, kind), nil, true) } +func (ownFindings) ModuleNode() (string, string) { return Module, "" } + func main() { k := NewKeeper() v := NewViews(k) diff --git a/modules/mesh-conditions/cmd/mesh-conditions/model.go b/modules/mesh-conditions/cmd/mesh-conditions/model.go index 36d4d0247..cc229449a 100644 --- a/modules/mesh-conditions/cmd/mesh-conditions/model.go +++ b/modules/mesh-conditions/cmd/mesh-conditions/model.go @@ -13,6 +13,7 @@ import ( seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" meshhealers "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshhealers" + "git.novox.be/novox/mesh-sdk/go/views" cc "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshconditionsclient" ) @@ -213,8 +214,8 @@ func order(list []Condition) { // decodeFinding reads an observer's finding of the seat's state `findings`, refusing one the keeper could not make a // condition of. A finding written before `ref` was named has it as `token`. -func decodeFinding(data []byte) (seat.FindingsByKeyValue, error) { - var f seat.FindingsByKeyValue +func decodeFinding(data []byte) (seat.Finding, error) { + var f seat.Finding if err := json.Unmarshal(data, &f); err != nil { return f, fmt.Errorf("a finding is not readable: %w", err) } @@ -225,10 +226,7 @@ func decodeFinding(data []byte) (seat.FindingsByKeyValue, error) { _ = json.Unmarshal(data, &old) f.Ref = old.Token } - if f.V != cc.FindingVersion { - return f, fmt.Errorf("a finding of version %d, and this reads %d", f.V, cc.FindingVersion) - } - return f, cc.CheckFinding(f) + return f, views.CheckFinding(f) } // triedOf is the attempt as the condition carries it in `tried`. -- 2.54.0 From cd246777568b4f8ad584798c3e858e9f9e7e9e68 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 21:59:18 +0100 Subject: [PATCH 093/243] mesh-self-check and mesh-healers give the writer the findings-asked subject of the seat's contract, and the reader the writer itself Builds: design 47 G A-17, A-16 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- modules/mesh-healers/cmd/mesh-healers/healers_test.go | 2 +- modules/mesh-healers/cmd/mesh-healers/main.go | 3 ++- .../mesh-healers/cmd/mesh-healers/whole_integration_test.go | 3 +-- modules/mesh-self-check/cmd/mesh-self-check/main.go | 5 +++-- .../mesh-self-check/cmd/mesh-self-check/selfcheck_test.go | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/modules/mesh-healers/cmd/mesh-healers/healers_test.go b/modules/mesh-healers/cmd/mesh-healers/healers_test.go index 6bb3d37c1..8b1413991 100644 --- a/modules/mesh-healers/cmd/mesh-healers/healers_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/healers_test.go @@ -161,7 +161,7 @@ func (w *world) fresh() { }, } wr, err := views.NewWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ - Module: Module, Node: "ace", State: findingState{w.store}, Observe: w.h.Observe, Now: func() time.Time { return w.now }, + Module: Module, Node: "ace", Asked: "mesh.seat.mesh-conditions.event.findings-asked", State: findingState{w.store}, Observe: w.h.Observe, Now: func() time.Time { return w.now }, Publish: func(_ context.Context, subject string, _ []byte, _ string) error { w.events = append(w.events, subject) return nil diff --git a/modules/mesh-healers/cmd/mesh-healers/main.go b/modules/mesh-healers/cmd/mesh-healers/main.go index 969f0a6fe..55cafb836 100644 --- a/modules/mesh-healers/cmd/mesh-healers/main.go +++ b/modules/mesh-healers/cmd/mesh-healers/main.go @@ -14,6 +14,7 @@ import ( "os" "time" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -58,7 +59,7 @@ func main() { Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, Publish: stdio.SeatPublish} w, err := views.NewStdioWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ - Module: Module, State: stdio.State(FindingsState), Observe: h.Observe, Logf: logf}) + Module: Module, State: stdio.State(FindingsState), Observe: h.Observe, Asked: meshconditions.SubjectFindingsAsked, Logf: logf}) if err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go index 58cf1869e..cf6173115 100644 --- a/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go +++ b/modules/mesh-healers/cmd/mesh-healers/whole_integration_test.go @@ -30,7 +30,6 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/testbus" - "git.novox.be/novox/mesh-sdk/go/views" ) type wire struct { @@ -390,7 +389,7 @@ func TestTheHealersWholeHearTheKeeperAndPassOverWhatEnded(t *testing.T) { // The runtime's consumer holds an event until the bundle subscribes: delivered once it hears them. eventually(t, "the healers hearing the keeper's conditions", func() bool { return p.hears(RaisedEvent) && p.hears(ChangedEvent) }) // The writer hears the keeper's asks for findings, and states at its start the findings it observes: none. - eventually(t, "the writer hearing findings-asked", func() bool { return p.hears(views.FindingsAskedEvent) }) + eventually(t, "the writer hearing findings-asked", func() bool { return p.hears("mesh-conditions.findings-asked") }) if msg, err := stated.NextMsg(10 * time.Second); err != nil || !strings.Contains(string(msg.Data), `"findings":[]`) { t.Errorf("findings-stated at the start: %v", err) } diff --git a/modules/mesh-self-check/cmd/mesh-self-check/main.go b/modules/mesh-self-check/cmd/mesh-self-check/main.go index 985011397..75ef37b47 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/main.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/main.go @@ -17,6 +17,7 @@ import ( stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-contracts/health-and-repair/go/clients/meshhandactsclient" + meshconditions "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -69,7 +70,7 @@ func NewChecker() *Checker { Ask: func(key string, body any) (json.RawMessage, error) { return stdio.Ask(key, body) }, Publish: stdio.SeatPublish} w, err := views.NewStdioWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ - Module: Module, State: stdio.State(FindingsState), Observe: c.Observe, Logf: logf}) + Module: Module, State: stdio.State(FindingsState), Observe: c.Observe, Asked: meshconditions.SubjectFindingsAsked, Logf: logf}) if err != nil { logf("%v", err) os.Exit(1) @@ -83,7 +84,7 @@ func NewChecker() *Checker { // its owner's read tool, then followed by the owner's events. A first view unanswered past its bound is a finding of the // self-check's own (first-view-unanswered), written into its state findings. func newViews(w *FindingsWriter) *views.Reader { - r := &views.Reader{Module: Module, Logf: logf, Findings: views.WriterFindings{Writer: w}, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} + r := &views.Reader{Module: Module, Logf: logf, Findings: w, Acts: meshhandactsclient.Recorder{Ask: stdio.Ask}} r.Follow(assignerModule, placing.StateAssignments, AssignmentsTool) r.Follow(assignerModule, placing.StateSeatHolders, SeatHoldersTool) r.Follow(indexerModule, "modules", ModulesTool) diff --git a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go index 04922b800..74b4984d3 100644 --- a/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go +++ b/modules/mesh-self-check/cmd/mesh-self-check/selfcheck_test.go @@ -127,7 +127,7 @@ func checker(store *memStore, registry ...Probe) (*Checker, *[]string) { return nil }} w, err := views.NewWriter(views.WriterConfig[Finding, FindingCleared, FindingsStated, FindingsAsked]{ - Module: Module, Node: "novox", State: findingState{store}, Observe: c.Observe, Logf: c.Logf, Now: c.Now, + Module: Module, Node: "novox", Asked: "mesh.seat.mesh-conditions.event.findings-asked", State: findingState{store}, Observe: c.Observe, Logf: c.Logf, Now: c.Now, Publish: func(_ context.Context, subject string, _ []byte, _ string) error { store.events = append(store.events, subject) return nil -- 2.54.0 From 06ebd7e8cf49addbc0d89649117d352e5fd5c85a Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:03:09 +0100 Subject: [PATCH 094/243] G findings: the writers name the ask's subject from the contract; the readers take the writer whole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WriterConfig.Asked is the generated mesh-conditions contract's SubjectFindingsAsked (A-17), a reader's Findings is the module's writer itself (A-22, A-26), and mesh-build-go's single manifest ask writes and clears its first-view-unanswered finding through the writer with the contract's Finding (A-28). mesh-config and mesh-vault's own followers (A-28) are not yet moved onto a views.Reader: they do not compile at mesh-sdk 24f876a. Builds: design 47 G §687, §689, A-17, A-22, A-26, A-28 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/artifact-store-tools/main.go | 4 +- modules/claude-code/cmd/claude-code/main.go | 4 +- .../cmd/distribution-tools/main.go | 2 +- modules/distribution/cmd/distribution/main.go | 2 +- modules/gitea/cmd/package-registry/main.go | 2 +- .../cmd/mesh-build-go/compose.go | 39 ++++++++++++++----- .../cmd/mesh-build-go/compose_test.go | 12 ++++-- .../mesh-build-go/cmd/mesh-build-go/main.go | 3 +- modules/mesh-config/cmd/mesh-config/main.go | 4 +- .../cmd/mesh-provisioner/main.go | 4 +- modules/mesh-vault/cmd/mesh-vault/main.go | 4 +- modules/mesh-watcher/cmd/mesh-watcher/main.go | 4 +- modules/minio/cmd/object-store/main.go | 4 +- modules/nfs-server/cmd/nfs-server/main.go | 4 +- 14 files changed, 59 insertions(+), 33 deletions(-) diff --git a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go index 611ef1130..14394b18b 100644 --- a/modules/artifact-store-tools/cmd/artifact-store-tools/main.go +++ b/modules/artifact-store-tools/cmd/artifact-store-tools/main.go @@ -32,13 +32,13 @@ func main() { h.Acts = meshhandactsclient.Recorder{Ask: stdio.Ask} // The module's one writer (design 47 G §689, A-16): the holder's findings and the reader's go through it. w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused}, Observe: h.FindingsObserved}) + Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State("findings"), Kinds: []string{ImageMissing, SweepRefused}, Observe: h.FindingsObserved}) if err != nil { logf("%v", err) os.Exit(1) } h.Findings = w - v.Findings = views.WriterFindings{Writer: w} + v.Findings = w // the runtime answers the writer's subscription only once Serve has its channel go func() { <-stdio.Served() diff --git a/modules/claude-code/cmd/claude-code/main.go b/modules/claude-code/cmd/claude-code/main.go index 2ed3b5b03..df34d25ac 100644 --- a/modules/claude-code/cmd/claude-code/main.go +++ b/modules/claude-code/cmd/claude-code/main.go @@ -234,7 +234,7 @@ var meshViews *views.Reader // observes none of its own. func newFindingsWriter() (*views.Writer[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent], error) { return views.NewStdioWriter(views.WriterConfig[condseat.Finding, condseat.FindingCleared, condseat.FindingsStated, condseat.FindingsAskedEvent]{ - Module: "claude-code", State: stdio.State("findings"), + Asked: condseat.SubjectFindingsAsked, Module: "claude-code", State: stdio.State("findings"), Observe: func(context.Context) (views.Observed[condseat.Finding], error) { return views.Observed[condseat.Finding]{}, nil }}) @@ -503,7 +503,7 @@ func main() { say("%v", err) os.Exit(1) } - meshViews.Findings = views.WriterFindings{Writer: findings} + meshViews.Findings = findings // the runtime answers the writer's subscription only once Serve has its channel go func() { <-stdio.Served() diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index e43ca6b53..f99403064 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -508,7 +508,7 @@ func main() { // The tools bundle's findings go through the module's writer (design 47 G §689, A-16); it owns the transfers' kinds, the // program's writer the driver's. w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Module: holder.Self, State: stdio.State("findings"), Kinds: holder.TransferFindingKinds, Observe: findings.Observed, Logf: logf}) + Asked: meshconditions.SubjectFindingsAsked, Module: holder.Self, State: stdio.State("findings"), Kinds: holder.TransferFindingKinds, Observe: findings.Observed, Logf: logf}) if err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/distribution/cmd/distribution/main.go b/modules/distribution/cmd/distribution/main.go index 4cedc92d9..42e2aabc8 100644 --- a/modules/distribution/cmd/distribution/main.go +++ b/modules/distribution/cmd/distribution/main.go @@ -68,7 +68,7 @@ func run(ctx context.Context, getenv func(string) string) error { // bundle's writer the transfers'. findings := &holder.Findings{Logf: logger.Printf} w, err := views.NewWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Module: holder.Self, Node: cred.Node, State: calls.State("findings"), Kinds: holder.DriverFindingKinds, + Asked: meshconditions.SubjectFindingsAsked, Module: holder.Self, Node: cred.Node, State: calls.State("findings"), Kinds: holder.DriverFindingKinds, JS: js, Conn: conn, Observe: findings.Observed, Logf: logger.Printf}) if err != nil { return err diff --git a/modules/gitea/cmd/package-registry/main.go b/modules/gitea/cmd/package-registry/main.go index ebcbed3ab..e07e52e3c 100644 --- a/modules/gitea/cmd/package-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -31,7 +31,7 @@ var findings = &Findings{Logf: func(f string, a ...any) { fmt.Fprintf(os.Stderr, func main() { // The bundle's findings go through the module's writer (design 47 G A-16). w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Module: packageseats.Self, State: stdio.State("findings"), Kinds: findingKinds, Observe: findings.Observed}) + Asked: meshconditions.SubjectFindingsAsked, Module: packageseats.Self, State: stdio.State("findings"), Kinds: findingKinds, Observe: findings.Observed}) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose.go b/modules/mesh-build-go/cmd/mesh-build-go/compose.go index bae34801d..35afca719 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose.go @@ -30,6 +30,7 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/composer" stdio "git.novox.be/novox/mesh-sdk/go" + "git.novox.be/novox/mesh-sdk/go/keys" "git.novox.be/novox/mesh-sdk/go/views" ) @@ -37,6 +38,9 @@ import ( // module invokes it, module.json invokes). A manifest by its digest never changes and a compose ask is rare, so the // module asks for the one key it needs when it needs it, and keeps no picture of the indexer's whole state (ADR 0615). // indexerModule is the module that holds the seat mesh-catalog, whose manifests state a compose ask reads. +// findingsModule is the module whose findings these are. +const findingsModule = "mesh-build-go" + const indexerModule = "mesh-delivery-indexer" const ManifestsTool = indexerModule + ".indexer_read_manifests" @@ -88,31 +92,48 @@ func askManifest(key string) (json.RawMessage, error) { // manifestFindings is where askManifest writes its finding: the module's writer once main has started it (design 47 G §689, // A-16); outside a launch by the runtime, which has no node and no state, nothing is written. -var manifestFindings views.FindingWriter = noFindings{} +var manifestFindings findingWriter = noFindings{} + +// findingWriter is the module's writer (views.Writer) as askManifestVia writes and clears through it. +type findingWriter interface { + Write(ctx context.Context, finding seat.Finding) error + Clear(ctx context.Context, scope, id, kind, why string) error +} type noFindings struct{} -func (noFindings) Put(views.Finding) error { return nil } -func (noFindings) Delete(views.Finding) error { return nil } +func (noFindings) Write(context.Context, seat.Finding) error { return nil } +func (noFindings) Clear(context.Context, string, string, string, string) error { return nil } + +// unansweredManifest is the finding of an ask the indexer did not answer, laid out as mesh-sdk lays out a views reader's +// first-view-unanswered (design 47 G §687, A-28): the id is the view's name, escaped as a key's name is. +func unansweredManifest(err string, asked, at time.Time) seat.Finding { + id := keys.EscapeName(findingsModule + "." + indexerModule + ".manifests") + return seat.Finding{Kind: views.UnansweredKind, Scope: "module", ID: id, Severity: seat.FindingSeverityWarning, Source: "views", + Subject: seat.Subject{Scope: "module", ID: findingsModule, Machine: os.Getenv("MESH_NODE")}, + Summary: fmt.Sprintf("%s has no first view of %s's manifests: %s has not answered since %s (%s)", findingsModule, indexerModule, ManifestsTool, + asked.Format(time.RFC3339), strings.TrimSpace(err)), + Headline: fmt.Sprintf("%s waits for %s", findingsModule, indexerModule), At: at.UTC()} +} // newFindingsWriter is this module's finding writer: it states the finding of askManifest and observes none of its own. func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ - Module: "mesh-build-go", State: stdio.State("findings"), + Asked: seat.SubjectFindingsAsked, Module: findingsModule, State: stdio.State("findings"), Observe: func(context.Context) (views.Observed[seat.Finding], error) { return views.Observed[seat.Finding]{}, nil }}) } -func askManifestVia(ask func(string, any) (json.RawMessage, error), findings views.FindingWriter, now func() time.Time, key string) (json.RawMessage, error) { - st := views.Status{Owner: indexerModule, Tool: ManifestsTool, State: "manifests", Asked: now().UTC()} +func askManifestVia(ask func(string, any) (json.RawMessage, error), findings findingWriter, now func() time.Time, key string) (json.RawMessage, error) { + asked := now().UTC() raw, err := ask(ManifestsTool, map[string]any{"key": key}) + f := unansweredManifest("", asked, now()) if err != nil { - st.Error = err.Error() - _ = findings.Put(views.UnansweredFinding("mesh-build-go", st, now())) + _ = findings.Write(context.Background(), unansweredManifest(err.Error(), asked, now())) return nil, err } - _ = findings.Delete(views.UnansweredFinding("mesh-build-go", st, now())) + _ = findings.Clear(context.Background(), f.Scope, f.ID, f.Kind, "the indexer answered") var a struct { Found bool `json:"found"` Value json.RawMessage `json:"value"` diff --git a/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go b/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go index 722b5abc7..332f60d72 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/compose_test.go @@ -11,12 +11,12 @@ import ( "testing" "time" + seat "git.novox.be/novox/mesh-contracts/health-and-repair/go/seats/meshconditions" catalogclient "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" meshcat "git.novox.be/novox/mesh-delivery/contracts/go/seats/meshcatalog" sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" "git.novox.be/novox/mesh-delivery/sdk/composer" "git.novox.be/novox/mesh-sdk/go/forgekey" - "git.novox.be/novox/mesh-sdk/go/views" ) // nodeEngineComposer is the composer command of the node-engine's repository, whose Go module is at its root (ADR 0593 §7): @@ -187,8 +187,14 @@ func TestAnIndexerThatDoesNotAnswerIsAFinding(t *testing.T) { type mapFindings map[string][]byte -func (m mapFindings) Put(f views.Finding) error { m[f.ID+"."+f.Kind] = []byte(f.Summary); return nil } -func (m mapFindings) Delete(f views.Finding) error { delete(m, f.ID+"."+f.Kind); return nil } +func (m mapFindings) Write(_ context.Context, f seat.Finding) error { + m[f.ID+"."+f.Kind] = []byte(f.Summary) + return nil +} +func (m mapFindings) Clear(_ context.Context, _, id, kind, _ string) error { + delete(m, id+"."+kind) + return nil +} // ADR 0593 §7 (C9): a compose ask naming the node-engine's repository builds its composer command in the Go module at the // repository's root, under a key of that repository, commit and package, and runs the program over the request with its diff --git a/modules/mesh-build-go/cmd/mesh-build-go/main.go b/modules/mesh-build-go/cmd/mesh-build-go/main.go index a571e8538..8817fc799 100644 --- a/modules/mesh-build-go/cmd/mesh-build-go/main.go +++ b/modules/mesh-build-go/cmd/mesh-build-go/main.go @@ -17,7 +17,6 @@ import ( sdkstep "git.novox.be/novox/mesh-delivery/sdk/buildstep" stdio "git.novox.be/novox/mesh-sdk/go" "git.novox.be/novox/mesh-sdk/go/busstate" - "git.novox.be/novox/mesh-sdk/go/views" ) // The directories the module's manifest places outside its home (ADR 0466 §1): the workspaces, and the Go module @@ -77,7 +76,7 @@ func main() { logf("the finding writer: %v", err) os.Exit(1) } - manifestFindings = views.WriterFindings{Writer: findings} + manifestFindings = findings // the runtime answers the writer's subscription only once Serve has its channel go func() { <-stdio.Served() diff --git a/modules/mesh-config/cmd/mesh-config/main.go b/modules/mesh-config/cmd/mesh-config/main.go index e45094a12..44d1b28d0 100644 --- a/modules/mesh-config/cmd/mesh-config/main.go +++ b/modules/mesh-config/cmd/mesh-config/main.go @@ -123,7 +123,7 @@ func main() { Delivery: toolDelivery{ask: stdio.Ask}, Vault: toolVault{ask: stdio.Ask, recorder: recorder}} // The module's one writer (design 47 G §689, A-16): the renames' and retirements' findings and the views reader's go through it. w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Module: Module, State: stdio.State(FindingsState), Observe: c.FindingsObserved, + Asked: meshconditions.SubjectFindingsAsked, Module: Module, State: stdio.State(FindingsState), Observe: c.FindingsObserved, Kinds: []string{FindingRenameRefused, FindingWindowHeld, FindingRetireUnanswered, FindingRetireHeld}}) if err != nil { logf("%v", err) @@ -136,7 +136,7 @@ func main() { case moved <- struct{}{}: default: } - }, views.WriterFindings{Writer: w}) + }, w) // the runtime answers the writer's subscription only once Serve has its channel go func() { <-stdio.Served() diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go index 8ffad9c06..83f513caf 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/main.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/main.go @@ -219,7 +219,7 @@ func watchedOver(v *views.Reader) func(string, func(stdio.StateChange) error) er // observes none of its own. func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ - Module: "mesh-provisioner", State: stdio.State("findings"), + Asked: seat.SubjectFindingsAsked, Module: "mesh-provisioner", State: stdio.State("findings"), Observe: func(context.Context) (views.Observed[seat.Finding], error) { return views.Observed[seat.Finding]{}, nil }}) @@ -233,7 +233,7 @@ func main() { logf("%v", err) os.Exit(1) } - v.Findings = views.WriterFindings{Writer: findings} + v.Findings = findings // One copy of what it reads, each state by its watch (ADR 0580 clause 2); the indexer's and the assigner's by their // pictures, a manifest by its digest from the picture too. ws := stdio.WatchedStatesOver(watchedOver(v), func(state, key string) ([]byte, bool, error) { diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 171e66138..2026a44c4 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -122,7 +122,7 @@ func watchLeavers(env *Env) bool { // The vault's finding writer (design 47 G §689, A-16): it states the assignments view's findings and observes none of // its own, and names no secret. w, err := views.NewStdioWriter(views.WriterConfig[meshconditions.Finding, meshconditions.FindingCleared, meshconditions.FindingsStated, meshconditions.FindingsAskedEvent]{ - Module: Seat, State: stdio.State(FindingsState), + Asked: meshconditions.SubjectFindingsAsked, Module: Seat, State: stdio.State(FindingsState), Observe: func(context.Context) (views.Observed[meshconditions.Finding], error) { return views.Observed[meshconditions.Finding]{}, nil }}) @@ -137,7 +137,7 @@ func watchLeavers(env *Env) bool { env.Vault.NodeLeft(node) } }, env.Vault.AssignmentsChanged, - views.WriterFindings{Writer: w}) + w) beginAssignmentsView(assignmentsView) } return true diff --git a/modules/mesh-watcher/cmd/mesh-watcher/main.go b/modules/mesh-watcher/cmd/mesh-watcher/main.go index d3a34095c..9e659b913 100644 --- a/modules/mesh-watcher/cmd/mesh-watcher/main.go +++ b/modules/mesh-watcher/cmd/mesh-watcher/main.go @@ -82,7 +82,7 @@ func main() { logf("%v", err) os.Exit(1) } - assignments.Findings = views.WriterFindings{Writer: findings} + assignments.Findings = findings assignments.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) // the runtime answers the writer's subscription only once Serve has its channel go func() { @@ -103,7 +103,7 @@ func main() { // observes none of its own. func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ - Module: "mesh-watcher", State: stdio.State("findings"), + Asked: seat.SubjectFindingsAsked, Module: "mesh-watcher", State: stdio.State("findings"), Observe: func(context.Context) (views.Observed[seat.Finding], error) { return views.Observed[seat.Finding]{}, nil }}) diff --git a/modules/minio/cmd/object-store/main.go b/modules/minio/cmd/object-store/main.go index 193c790bb..e043b6639 100644 --- a/modules/minio/cmd/object-store/main.go +++ b/modules/minio/cmd/object-store/main.go @@ -36,7 +36,7 @@ func main() { fmt.Fprintf(os.Stderr, "[object-store] %v\n", err) os.Exit(1) } - seats.Findings = views.WriterFindings{Writer: findings} + seats.Findings = findings seats.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) s, err := newServer(os.Getenv(ConfigWord), os.Getenv(LinksWord), os.Getenv(ReceivesWord), func() (map[string][]byte, error) { return seats.Values(assignerModule + "." + mesh.StateSeatHolders) }) @@ -70,7 +70,7 @@ const SeatHoldersTool = assignerModule + ".assigner_read_seat_holders" // observes none of its own. func newFindingsWriter() (*views.Writer[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent], error) { return views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ - Module: "minio", State: stdio.State("findings"), + Asked: seat.SubjectFindingsAsked, Module: "minio", State: stdio.State("findings"), Observe: func(context.Context) (views.Observed[seat.Finding], error) { return views.Observed[seat.Finding]{}, nil }}) diff --git a/modules/nfs-server/cmd/nfs-server/main.go b/modules/nfs-server/cmd/nfs-server/main.go index 3a85efeb4..6b4d106df 100644 --- a/modules/nfs-server/cmd/nfs-server/main.go +++ b/modules/nfs-server/cmd/nfs-server/main.go @@ -52,7 +52,7 @@ func main() { } m := Here() w, err := views.NewStdioWriter(views.WriterConfig[seat.Finding, seat.FindingCleared, seat.FindingsStated, seat.FindingsAskedEvent]{ - Module: "nfs-server", State: stdio.State("findings"), + Asked: seat.SubjectFindingsAsked, Module: "nfs-server", State: stdio.State("findings"), Observe: func(context.Context) (views.Observed[seat.Finding], error) { return views.Observed[seat.Finding]{}, nil }}) @@ -60,7 +60,7 @@ func main() { fmt.Fprintln(os.Stderr, err) os.Exit(1) } - seats.Findings = views.WriterFindings{Writer: w} + seats.Findings = w seats.Follow(assignerModule, mesh.StateSeatHolders, SeatHoldersTool) seats.Start(context.Background()) // the runtime answers the writer's subscription only once Serve has its channel -- 2.54.0 From 1cc67aa9d7207282d982b0a6c85747efa9f45f3c Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:07:02 +0100 Subject: [PATCH 095/243] G findings: mesh-config's and mesh-vault's own followers go through a views.Reader holding the writer, adopted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds: design 47 G A-28, A-26, A-22, §687 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../mesh-config/cmd/mesh-config/ownerviews.go | 73 ++++++------------- .../cmd/mesh-config/ownerviews_test.go | 8 +- .../cmd/mesh-vault/assignmentsview.go | 59 ++++----------- .../cmd/mesh-vault/assignmentsview_test.go | 15 ++-- modules/mesh-vault/cmd/mesh-vault/main.go | 13 +++- 5 files changed, 63 insertions(+), 105 deletions(-) diff --git a/modules/mesh-config/cmd/mesh-config/ownerviews.go b/modules/mesh-config/cmd/mesh-config/ownerviews.go index 1eb15cf1f..2920b0c22 100644 --- a/modules/mesh-config/cmd/mesh-config/ownerviews.go +++ b/modules/mesh-config/cmd/mesh-config/ownerviews.go @@ -21,7 +21,6 @@ import ( "time" stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/views" ) @@ -49,19 +48,20 @@ var followed = []struct{ Read, Owner, State, Tool string }{ {WantsRead, assignerModule, "wants", AssignerWantsTool}, } -// ownerViews are the pictures, by the read name. +// ownerViews are the pictures, by the read name, followed through one views.Reader holding the module's writer (design 47 G +// A-28): each follower is adopted, so the reader raises and clears its findings and hears its owner's views event. type ownerViews struct { - by map[string]*views.Follower - // findings keeps the findings a first view unanswered writes. - findings views.FindingWriter + by map[string]*views.Follower + reader *views.Reader } -// newOwnerViews makes a follower for each state; changed is told when any picture changes. -func newOwnerViews(ask func(key string, body any) (json.RawMessage, error), changed func(), findings views.FindingWriter) *ownerViews { - o := &ownerViews{by: map[string]*views.Follower{}, findings: findings} +// newOwnerViews makes a follower for each state and adopts it; changed is told when any picture changes. findings is the +// module's writer, which the reader writes the first views' findings through. +func newOwnerViews(ask func(key string, body any) (json.RawMessage, error), changed func(), findings views.WireWriter) *ownerViews { + o := &ownerViews{by: map[string]*views.Follower{}, reader: &views.Reader{Module: Module, Findings: findings, Logf: logf}} for _, f := range followed { f := f - fol := &views.Follower{Owner: f.Owner, Tool: f.Owner + "." + f.Tool, State: f.State, Logf: logf, + fol := &views.Follower{Owner: f.Owner, Tool: f.Owner + "." + f.Tool, State: f.State, Ask: func(_ context.Context, after string) (views.Answer, error) { body := map[string]any{"values": true} if after != "" { @@ -78,54 +78,27 @@ func newOwnerViews(ask func(key string, body any) (json.RawMessage, error), chan return views.DecodeAnswer(answer) }, Changed: func(string, json.RawMessage, bool, time.Time) { changed() }} - fol.Raise = func(s views.Status) { - _ = findings.Put(views.UnansweredFinding(Module, s, time.Now())) + hear := func(ctx context.Context, body []byte) error { + ev, err := views.DecodeEvent(body) + if err != nil { + return err + } + if gap, why := fol.Apply(ev); gap { + logf("[mesh-config] a gap in %s's %s (%s): its first view is asked once more", f.Owner, f.State, why) + go func() { _ = fol.Start(ctx) }() + } + return nil } - fol.Clear = func(s views.Status) { - _ = findings.Delete(views.UnansweredFinding(Module, s, time.Now())) + if err := o.reader.Adopt(fol, f.Owner, views.EventName(f.State), hear); err != nil { + logf("[mesh-config] %v", err) } o.by[f.Read] = fol } return o } -// Start asks every first view until answered (each on its goroutine) and subscribes to the owners' views events. -func (o *ownerViews) Start(ctx context.Context) { - for _, fol := range o.by { - go func(f *views.Follower) { - if err := f.Start(ctx); err != nil { - logf("[mesh-config] the first view of %s's %s ended unanswered: %v", f.Owner, f.State, err) - } - }(fol) - } - for _, f := range followed { - f := f - pattern := f.Owner + "." + views.EventName(f.State) - go func() { - <-stdio.Served() - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - err := stdio.Subscribe(pattern, func(e stdio.Envelope) error { - ev, err := views.DecodeEvent(e.Body) - if err != nil { - logf("[mesh-config] %s is not a views event: %v", pattern, err) - return nil - } - fol := o.by[f.Read] - if gap, why := fol.Apply(ev); gap { - logf("[mesh-config] a gap in %s's %s (%s): its first view is asked once more", f.Owner, f.State, why) - go func() { _ = fol.Start(ctx) }() - } - return nil - }) - if err == nil { - return - } - logf("[mesh-config] does not hear %s yet (%v); asking again in %s", pattern, err, wait) - time.Sleep(wait) - } - }() - } -} +// Start subscribes the owners' views events and asks every first view until answered, each on its goroutine. +func (o *ownerViews) Start(ctx context.Context) { o.reader.Start(ctx) } // read is the picture of a read name, and whether the name is one followed. func (o *ownerViews) read(state string) (*views.Follower, bool) { diff --git a/modules/mesh-config/cmd/mesh-config/ownerviews_test.go b/modules/mesh-config/cmd/mesh-config/ownerviews_test.go index d7611930e..e688d25bb 100644 --- a/modules/mesh-config/cmd/mesh-config/ownerviews_test.go +++ b/modules/mesh-config/cmd/mesh-config/ownerviews_test.go @@ -36,6 +36,9 @@ func TestAFollowedStateIsReadFromItsOwnersFirstViewAndEvents(t *testing.T) { if _, followed, err := o.get(WantsRead, "want.g14.dunst"); !followed || err == nil || !strings.Contains(err.Error(), "not answered yet") { t.Fatalf("before the first view: followed %v, %v", followed, err) } + if !o.reader.Follows("mesh-delivery-assigner.wants") { + t.Error("the wants view is not one of the reader's (A-28: adopted)") + } if _, followed, _ := o.get(BuildsRead, "x"); followed { t.Error("the builder's builds is followed, and it has no owner tool") } @@ -78,5 +81,6 @@ func TestAFollowedStateIsReadFromItsOwnersFirstViewAndEvents(t *testing.T) { // nopFindings takes a finding and keeps none. type nopFindings struct{} -func (nopFindings) Put(views.Finding) error { return nil } -func (nopFindings) Delete(views.Finding) error { return nil } +func (nopFindings) WriteWire(context.Context, map[string]any) error { return nil } +func (nopFindings) Clear(_ context.Context, _, _, _, _ string) error { return nil } +func (nopFindings) ModuleNode() (string, string) { return Module, "" } diff --git a/modules/mesh-vault/cmd/mesh-vault/assignmentsview.go b/modules/mesh-vault/cmd/mesh-vault/assignmentsview.go index 644029f0f..c9f177d1c 100644 --- a/modules/mesh-vault/cmd/mesh-vault/assignmentsview.go +++ b/modules/mesh-vault/cmd/mesh-vault/assignmentsview.go @@ -14,8 +14,6 @@ import ( "strings" "time" - stdio "git.novox.be/novox/mesh-sdk/go" - "git.novox.be/novox/mesh-sdk/go/views" ) @@ -38,8 +36,7 @@ var assignmentsView *views.Follower // newAssignmentsView makes the follower: ask calls the tool, gone is told of a deleted key, changed of every change after // it is held (a first view's keys too). -func newAssignmentsView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), changed func(), - findings views.FindingWriter) *views.Follower { +func newAssignmentsView(ask func(key string, body any) (json.RawMessage, error), gone func(key string), changed func()) *views.Follower { return &views.Follower{Owner: assignerModule, Tool: AssignmentsTool, State: "assignments", Logf: say, Ask: func(_ context.Context, after string) (views.Answer, error) { body := map[string]any{"values": true} @@ -52,19 +49,6 @@ func newAssignmentsView(ask func(key string, body any) (json.RawMessage, error), } return views.DecodeAnswer(unwrapToolAnswer(raw)) }, - // Past its bound the vault's finding names the owner and the tool, and the condition keeper raises it (the - // answer deletes it); the vault keeps asking meanwhile. - Raise: func(s views.Status) { - say("the vault has no first view of the assigner's assignments: %s has not answered since %s (%s); it keeps asking", s.Tool, - s.Asked.Format(time.RFC3339), s.Error) - f := views.UnansweredFinding(Seat, s, time.Now()) - if err := findings.Put(f); err != nil { - say("the finding %s is not written: %v", f.ID, err) - } - }, - Clear: func(s views.Status) { - _ = findings.Delete(views.UnansweredFinding(Seat, s, time.Now())) - }, Changed: func(key string, _ json.RawMessage, deleted bool, _ time.Time) { if deleted { gone(key) @@ -88,35 +72,22 @@ func unwrapToolAnswer(raw json.RawMessage) []byte { return raw } -// beginAssignmentsView asks the first view until answered and hears the owner's events. -func beginAssignmentsView(f *views.Follower) { - go func() { - if err := f.Start(context.Background()); err != nil { - say("the first view of the assigner's assignments ended unanswered: %v", err) +// adoptAssignmentsView holds the follower in a views.Reader with the vault's writer (design 47 G A-28), so the reader writes +// and clears its findings (past its bound the finding names the owner and the tool, and the answer clears it; no secret). +func adoptAssignmentsView(f *views.Follower, findings views.WireWriter) (*views.Reader, error) { + r := &views.Reader{Module: Seat, Findings: findings, Logf: say} + hear := func(ctx context.Context, body []byte) error { + ev, err := views.DecodeEvent(body) + if err != nil { + return err } - }() - go func() { - <-stdio.Served() - for wait := 2 * time.Second; ; wait = min(wait*2, time.Minute) { - err := stdio.Subscribe(AssignmentsViewsEvent, func(e stdio.Envelope) error { - ev, err := views.DecodeEvent(e.Body) - if err != nil { - say("%s is not a views event: %v", AssignmentsViewsEvent, err) - return nil - } - if gap, why := f.Apply(ev); gap { - say("a gap in the assigner's assignments (%s): its first view is asked once more", why) - go func() { _ = f.Start(context.Background()) }() - } - return nil - }) - if err == nil { - return - } - say("does not hear %s yet (%v); asking again in %s", AssignmentsViewsEvent, err, wait) - time.Sleep(wait) + if gap, why := f.Apply(ev); gap { + say("a gap in the assigner's assignments (%s): its first view is asked once more", why) + go func() { _ = f.Start(ctx) }() } - }() + return nil + } + return r, r.Adopt(f, assignerModule, views.EventName("assignments"), hear) } // viewAssignments is the assignments reader over the picture: a key asked before the first view is answered is an error, so diff --git a/modules/mesh-vault/cmd/mesh-vault/assignmentsview_test.go b/modules/mesh-vault/cmd/mesh-vault/assignmentsview_test.go index f259e36e1..415d09913 100644 --- a/modules/mesh-vault/cmd/mesh-vault/assignmentsview_test.go +++ b/modules/mesh-vault/cmd/mesh-vault/assignmentsview_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "sync" "testing" "time" @@ -28,7 +29,10 @@ func TestAnUnansweredFirstViewOfTheAssignmentsIsAFindingTheAnswerClears(t *testi Values: map[string]json.RawMessage{"g14.node": json.RawMessage(`{}`)}}) return raw, nil } - f := newAssignmentsView(ask, func(string) {}, nil, heldFindings{&mu, held}) + f := newAssignmentsView(ask, func(string) {}, nil) + if _, err := adoptAssignmentsView(f, heldFindings{&mu, held}); err != nil { + t.Fatal(err) + } f.Bound, f.Wait = 30*time.Millisecond, 10*time.Millisecond reader := viewAssignments(func() *views.Follower { return f }) if _, err := reader.Keys(); err == nil { @@ -62,15 +66,16 @@ type heldFindings struct { held map[string]json.RawMessage } -func (h heldFindings) Put(f views.Finding) error { +func (h heldFindings) WriteWire(_ context.Context, f map[string]any) error { h.mu.Lock() defer h.mu.Unlock() - h.held[f.ID+"."+f.Kind] = json.RawMessage(f.Summary) + h.held[fmt.Sprint(f["id"], ".", f["kind"])] = nil return nil } -func (h heldFindings) Delete(f views.Finding) error { +func (h heldFindings) Clear(_ context.Context, _, id, kind, _ string) error { h.mu.Lock() defer h.mu.Unlock() - delete(h.held, f.ID+"."+f.Kind) + delete(h.held, id+"."+kind) return nil } +func (h heldFindings) ModuleNode() (string, string) { return Seat, "" } diff --git a/modules/mesh-vault/cmd/mesh-vault/main.go b/modules/mesh-vault/cmd/mesh-vault/main.go index 2026a44c4..3a93bebf9 100644 --- a/modules/mesh-vault/cmd/mesh-vault/main.go +++ b/modules/mesh-vault/cmd/mesh-vault/main.go @@ -132,13 +132,18 @@ func watchLeavers(env *Env) bool { if err != nil { return false } - assignmentsView = newAssignmentsView(stdio.Ask, func(key string) { + f := newAssignmentsView(stdio.Ask, func(key string) { if node, ok := leaverKey(key); ok { env.Vault.NodeLeft(node) } - }, env.Vault.AssignmentsChanged, - w) - beginAssignmentsView(assignmentsView) + }, env.Vault.AssignmentsChanged) + r, err := adoptAssignmentsView(f, w) + if err != nil { + say("the assignments view is not adopted: %v", err) + return false + } + assignmentsView = f + r.Start(context.Background()) } return true } -- 2.54.0 From 3215e5eeb92083d2f0266b92b7d6f0f5ddcb83b4 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:08:33 +0100 Subject: [PATCH 096/243] G findings: distribution and gitea's package registry delete their old finding keys once at start Builds: design 47 G A-31, A-14, A-22 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/distribution-tools/main.go | 4 ++ modules/distribution/cmd/distribution/main.go | 4 ++ .../distribution/internal/holder/findings.go | 26 +++++++++++ .../internal/holder/findings_test.go | 44 +++++++++++++++++++ .../gitea/cmd/package-registry/findings.go | 26 +++++++++++ .../cmd/package-registry/findings_test.go | 44 +++++++++++++++++++ modules/gitea/cmd/package-registry/main.go | 4 ++ 7 files changed, 152 insertions(+) create mode 100644 modules/distribution/internal/holder/findings_test.go create mode 100644 modules/gitea/cmd/package-registry/findings_test.go diff --git a/modules/distribution/cmd/distribution-tools/main.go b/modules/distribution/cmd/distribution-tools/main.go index f99403064..e3964a8c4 100644 --- a/modules/distribution/cmd/distribution-tools/main.go +++ b/modules/distribution/cmd/distribution-tools/main.go @@ -517,6 +517,10 @@ func main() { // the runtime answers the writer's subscription only once Serve has its channel go func() { <-stdio.Served() + // the keys of the older form end in no kind the writer owns, so it does not clear them (A-31) + if err := holder.DeleteOldFindings(stdio.State("findings"), holder.Self, holder.TransferFindingKinds); err != nil { + logf("the old findings are not deleted: %v", err) + } if err := w.Start(context.Background()); err != nil { logf("%v", err) os.Exit(1) diff --git a/modules/distribution/cmd/distribution/main.go b/modules/distribution/cmd/distribution/main.go index 42e2aabc8..7fb838063 100644 --- a/modules/distribution/cmd/distribution/main.go +++ b/modules/distribution/cmd/distribution/main.go @@ -74,6 +74,10 @@ func run(ctx context.Context, getenv func(string) string) error { return err } findings.Writer = w + // the keys of the older form end in no kind the writer owns, so it does not clear them (A-31) + if err := holder.DeleteOldFindings(calls.State("findings"), holder.Self, holder.DriverFindingKinds); err != nil { + logger.Printf("the old findings are not deleted: %v", err) + } if err := w.Start(ctx); err != nil { return err } diff --git a/modules/distribution/internal/holder/findings.go b/modules/distribution/internal/holder/findings.go index b99472d9b..6a4e51684 100644 --- a/modules/distribution/internal/holder/findings.go +++ b/modules/distribution/internal/holder/findings.go @@ -2,6 +2,7 @@ package holder import ( "context" + "strings" "sync" "time" @@ -75,3 +76,28 @@ func (k *Findings) Observed(context.Context) (views.Observed[meshconditions.Find } return out, nil } + +// DeleteOldFindings deletes, once at a module's start, the findings the keys of an older form hold (design 47 G A-31): the form +// `seat.._`, whose kind is no kind the writer owns in its key, so the writer's own first statement does not +// clear it. Only a key of one of kinds is deleted; the keys of the writer's form (`seat...`) are left. +func DeleteOldFindings(state views.FindingState, seat string, kinds []string) error { + keys, err := state.Keys() + if err != nil { + return err + } + for _, k := range keys { + rest, ok := strings.CutPrefix(k, "seat."+seat+".") + if !ok { + continue + } + for _, kind := range kinds { + if strings.HasPrefix(rest, kind+"_") { + if err := state.Delete(k); err != nil { + return err + } + break + } + } + } + return nil +} diff --git a/modules/distribution/internal/holder/findings_test.go b/modules/distribution/internal/holder/findings_test.go new file mode 100644 index 000000000..677910d1b --- /dev/null +++ b/modules/distribution/internal/holder/findings_test.go @@ -0,0 +1,44 @@ +package holder + +import ( + "sort" + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// A-31: the keys of the older form (`seat.._`) are deleted once; a key of the writer's form, another seat's +// and another kind's stay. +func TestTheOldFindingKeysAreDeletedOnce(t *testing.T) { + state := &oldKeysState{keys: map[string]bool{}} + for _, k := range []string{ + "seat.distribution.publish-refused_write", "seat.distribution.publish-refused_a_b_c", + "seat.distribution.sub.publish-refused", "seat.other.publish-refused_write", "seat.distribution.unrelated_write"} { + state.keys[k] = true + } + if err := DeleteOldFindings(state, "distribution", []string{"publish-refused", "incoming-full"}); err != nil { + t.Fatal(err) + } + var left []string + for k := range state.keys { + left = append(left, k) + } + sort.Strings(left) + if got, want := strings.Join(left, " "), "seat.distribution.sub.publish-refused seat.distribution.unrelated_write seat.other.publish-refused_write"; got != want { + t.Errorf("left %s, want %s", got, want) + } +} + +type oldKeysState struct{ keys map[string]bool } + +func (s *oldKeysState) Get(string) (*stdio.StateEntry, error) { return nil, nil } +func (s *oldKeysState) Put(string, any) (uint64, error) { return 0, nil } +func (s *oldKeysState) Delete(k string) error { delete(s.keys, k); return nil } +func (s *oldKeysState) Keys() ([]string, error) { + var out []string + for k := range s.keys { + out = append(out, k) + } + return out, nil +} diff --git a/modules/gitea/cmd/package-registry/findings.go b/modules/gitea/cmd/package-registry/findings.go index 544e699f1..b9ae01044 100644 --- a/modules/gitea/cmd/package-registry/findings.go +++ b/modules/gitea/cmd/package-registry/findings.go @@ -2,6 +2,7 @@ package main import ( "context" + "strings" "sync" "time" @@ -74,3 +75,28 @@ func (k *Findings) Observed(context.Context) (views.Observed[meshconditions.Find } return out, nil } + +// DeleteOldFindings deletes, once at a module's start, the findings the keys of an older form hold (design 47 G A-31): the form +// `seat.._`, whose kind is no kind the writer owns in its key, so the writer's own first statement does not +// clear it. Only a key of one of kinds is deleted; the keys of the writer's form (`seat...`) are left. +func DeleteOldFindings(state views.FindingState, seat string, kinds []string) error { + keys, err := state.Keys() + if err != nil { + return err + } + for _, k := range keys { + rest, ok := strings.CutPrefix(k, "seat."+seat+".") + if !ok { + continue + } + for _, kind := range kinds { + if strings.HasPrefix(rest, kind+"_") { + if err := state.Delete(k); err != nil { + return err + } + break + } + } + } + return nil +} diff --git a/modules/gitea/cmd/package-registry/findings_test.go b/modules/gitea/cmd/package-registry/findings_test.go new file mode 100644 index 000000000..df05ad213 --- /dev/null +++ b/modules/gitea/cmd/package-registry/findings_test.go @@ -0,0 +1,44 @@ +package main + +import ( + "sort" + "strings" + "testing" + + stdio "git.novox.be/novox/mesh-sdk/go" +) + +// A-31: the keys of the older form (`seat.._`) are deleted once; a key of the writer's form, another seat's +// and another kind's stay. +func TestTheOldFindingKeysAreDeletedOnce(t *testing.T) { + state := &oldKeysState{keys: map[string]bool{}} + for _, k := range []string{ + "seat.gitea.packages-token-refused_write", "seat.gitea.packages-token-refused_a_b_c", + "seat.gitea.sub.packages-token-refused", "seat.other.packages-token-refused_write", "seat.gitea.unrelated_write"} { + state.keys[k] = true + } + if err := DeleteOldFindings(state, "gitea", []string{"packages-token-refused", "publish-refused", "incoming-full"}); err != nil { + t.Fatal(err) + } + var left []string + for k := range state.keys { + left = append(left, k) + } + sort.Strings(left) + if got, want := strings.Join(left, " "), "seat.gitea.sub.packages-token-refused seat.gitea.unrelated_write seat.other.packages-token-refused_write"; got != want { + t.Errorf("left %s, want %s", got, want) + } +} + +type oldKeysState struct{ keys map[string]bool } + +func (s *oldKeysState) Get(string) (*stdio.StateEntry, error) { return nil, nil } +func (s *oldKeysState) Put(string, any) (uint64, error) { return 0, nil } +func (s *oldKeysState) Delete(k string) error { delete(s.keys, k); return nil } +func (s *oldKeysState) Keys() ([]string, error) { + var out []string + for k := range s.keys { + out = append(out, k) + } + return out, nil +} diff --git a/modules/gitea/cmd/package-registry/main.go b/modules/gitea/cmd/package-registry/main.go index e07e52e3c..fc26d39c7 100644 --- a/modules/gitea/cmd/package-registry/main.go +++ b/modules/gitea/cmd/package-registry/main.go @@ -40,6 +40,10 @@ func main() { // the runtime answers the writer's subscription only once Serve has its channel go func() { <-stdio.Served() + // the keys of the older form end in no kind the writer owns, so it does not clear them (A-31) + if err := DeleteOldFindings(stdio.State("findings"), packageseats.Self, findingKinds); err != nil { + fmt.Fprintln(os.Stderr, "the old findings are not deleted:", err) + } if err := w.Start(context.Background()); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) -- 2.54.0 From 8255640d00b3c46db8b4a51dbc1201f919a533d2 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:10:19 +0100 Subject: [PATCH 097/243] G findings: claude-code, mesh-provisioner, distribution, mesh-config and gitea declare the writer's three events MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The writer listens to mesh-conditions.findings-asked, so each also consumes it, as the modules that moved before. Builds: design 47 G A-30, §383, §686 Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/claude-code/config_test.go | 2 +- modules/claude-code/module.json | 17 ++++++++++++++- .../internal/holder/holder_test.go | 2 +- modules/distribution/module.json | 17 ++++++++++++++- modules/gitea/module.json | 17 ++++++++++++++- modules/mesh-config/module.json | 19 ++++++++++++++++- .../cmd/mesh-provisioner/provisioner_test.go | 8 ++++++- modules/mesh-provisioner/module.json | 21 +++++++++++++++++-- 8 files changed, 94 insertions(+), 9 deletions(-) diff --git a/modules/claude-code/cmd/claude-code/config_test.go b/modules/claude-code/cmd/claude-code/config_test.go index 5cdbc2238..948fb3c9b 100644 --- a/modules/claude-code/cmd/claude-code/config_test.go +++ b/modules/claude-code/cmd/claude-code/config_test.go @@ -279,7 +279,7 @@ func TestTheManifestListsEveryToolServed(t *testing.T) { // the controller, and no read of the indexer's or the assigner's state (ADR 0615 §1): their views are consumed. if !slices.Equal(m.Invokes, []string{"seat:mesh-hand-acts.warranted", "seat:mesh-hand-acts.hand-act", "seat:mesh-catalog.seat-definitions", ModulesTool, ManifestsTool, AssignmentsTool}) || !slices.Equal(m.Reads, []string{"claude-licence-manager.bindings"}) || - !slices.Equal(m.Consumes, []string{"mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.manifests", "mesh-delivery-assigner.views.assignments", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed"}) { + !slices.Equal(m.Consumes, []string{"mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.manifests", "mesh-delivery-assigner.views.assignments", "mesh-catalog.seat-definition-changed", "mesh-catalog.seat-definition-removed", "mesh-conditions.findings-asked"}) { t.Errorf("invokes = %v, reads = %v, consumes = %v", m.Invokes, m.Reads, m.Consumes) } if src, _ := os.ReadFile("proposals.go"); !strings.Contains(string(src), `"seat:mesh-hand-acts.warranted"`) || diff --git a/modules/claude-code/module.json b/modules/claude-code/module.json index 7bfbd71fd..5a0b83f2f 100644 --- a/modules/claude-code/module.json +++ b/modules/claude-code/module.json @@ -11,6 +11,20 @@ { "name": "gitea-read", "optional": true + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } } ], "binds": { @@ -35,7 +49,8 @@ "mesh-delivery-indexer.views.manifests", "mesh-delivery-assigner.views.assignments", "mesh-catalog.seat-definition-changed", - "mesh-catalog.seat-definition-removed" + "mesh-catalog.seat-definition-removed", + "mesh-conditions.findings-asked" ], "state": [ "servers", diff --git a/modules/distribution/internal/holder/holder_test.go b/modules/distribution/internal/holder/holder_test.go index c428e942b..332a3757e 100644 --- a/modules/distribution/internal/holder/holder_test.go +++ b/modules/distribution/internal/holder/holder_test.go @@ -865,7 +865,7 @@ func TestTheHolderOfTheTwoImageSeats(t *testing.T) { if _, ok := claims[OldSeat]; !ok { t.Error("the old seat mesh-artifact-store is no longer claimed while its window is open") } - if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed"}) { + if !reflect.DeepEqual(m.Consumes, []string{"mesh-delivery-view.delivered", "node-engine.manifest-reads-changed", "mesh-conditions.findings-asked"}) { t.Errorf("consumes = %v, want delivered and manifest-reads-changed (the windows' wakes)", m.Consumes) } // E14: the registry container, on the loopback, is listed before the program, so the port is free when it starts. diff --git a/modules/distribution/module.json b/modules/distribution/module.json index 04b974bbb..0642df247 100644 --- a/modules/distribution/module.json +++ b/modules/distribution/module.json @@ -67,7 +67,8 @@ ], "consumes": [ "mesh-delivery-view.delivered", - "node-engine.manifest-reads-changed" + "node-engine.manifest-reads-changed", + "mesh-conditions.findings-asked" ], "config": { "callers": { @@ -132,6 +133,20 @@ "max-age-days": 7, "max-mib": 16 } + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } } ], "capabilities": [ diff --git a/modules/gitea/module.json b/modules/gitea/module.json index 5255a55f8..dd8276509 100644 --- a/modules/gitea/module.json +++ b/modules/gitea/module.json @@ -34,6 +34,20 @@ "options": { "history": 1 } + }, + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } } ], "contributes": { @@ -657,6 +671,7 @@ "git.branch-moved", "git.pull-updated", "git.pull-closed", - "git.pull-merged" + "git.pull-merged", + "mesh-conditions.findings-asked" ] } diff --git a/modules/mesh-config/module.json b/modules/mesh-config/module.json index 5e65b3fc8..5c772339f 100644 --- a/modules/mesh-config/module.json +++ b/modules/mesh-config/module.json @@ -33,7 +33,8 @@ "mesh-delivery-indexer.views.predefined", "mesh-delivery-indexer.views.config-candidates", "mesh-delivery-assigner.views.assignments", - "mesh-delivery-assigner.views.wants" + "mesh-delivery-assigner.views.wants", + "mesh-conditions.findings-asked" ], "invokes": [ "seat:mesh-hand-acts.hand-act", @@ -124,5 +125,21 @@ ], "retention": "history" } + ], + "requires": [ + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } ] } diff --git a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go index 886324ebb..fd6cbc2d8 100644 --- a/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go +++ b/modules/mesh-provisioner/cmd/mesh-provisioner/provisioner_test.go @@ -290,6 +290,12 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { if err != nil { t.Fatal(err) } + // the writer's three events are declared (design 47 G A-30) + for _, ev := range []string{"finding-written.*", "finding-cleared.*", "findings-stated.*"} { + if !strings.Contains(string(raw), `"`+ev+`"`) { + t.Errorf("the manifest does not declare %s", ev) + } + } var m struct { Module string `json:"module"` RunsAs string `json:"runs-as"` @@ -364,7 +370,7 @@ func TestTheManifestDeclaresWhatTheCodeKeeps(t *testing.T) { } if !slices.Equal(m.Invokes, []string{AssignmentsTool, WantsTool, ModulesTool, ManifestsTool, "seat:mesh-hand-acts.hand-act"}) || !slices.Equal(m.Consumes, []string{"mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", - "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.manifests"}) { + "mesh-delivery-indexer.views.modules", "mesh-delivery-indexer.views.manifests", "mesh-conditions.findings-asked"}) { t.Errorf("invokes %v, consumes %v", m.Invokes, m.Consumes) } // It composes nothing (ADR 0593 §8): no config value is read to make a plan. diff --git a/modules/mesh-provisioner/module.json b/modules/mesh-provisioner/module.json index c35a14b64..fb3326069 100644 --- a/modules/mesh-provisioner/module.json +++ b/modules/mesh-provisioner/module.json @@ -40,7 +40,8 @@ "mesh-delivery-assigner.views.assignments", "mesh-delivery-assigner.views.wants", "mesh-delivery-indexer.views.modules", - "mesh-delivery-indexer.views.manifests" + "mesh-delivery-indexer.views.manifests", + "mesh-conditions.findings-asked" ], "tools": [ "provisioner_provisions", @@ -105,5 +106,21 @@ ] } ] - } + }, + "requires": [ + { + "name": "finding-events", + "storage": "stream", + "template": "events", + "options": { + "events": [ + "finding-written.*", + "finding-cleared.*", + "findings-stated.*" + ], + "max-age-days": 7, + "max-mib": 16 + } + } + ] } -- 2.54.0 From 7cc4a33cc57dcd8bfa06724c1c86203f8e65dce3 Mon Sep 17 00:00:00 2001 From: jochen Date: Wed, 11 Nov 2026 22:10:42 +0100 Subject: [PATCH 098/243] =?UTF-8?q?GATE=20A-24,=20=C2=A7172,=20=C2=A7431,?= =?UTF-8?q?=20=C2=A7592,=20=C2=A7593,=20=C2=A7613:=20the=20gate=20keeps=20?= =?UTF-8?q?no=20merge=20states;=20deliveries,=20show,=20groups=20and=20sta?= =?UTF-8?q?lled=20answer=20the=20delivery=20records?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removes holder, table, approval, ready, onestatus, pullkey, checks, group_merge, the effects, the forge reads, the teller's and the view's reads, kvOperator, kvPulls, delivery_merge_again and their tests. deliveries, show, groups and stalled answer the records as kept (stalled: a record open past a day, and an act owed unanswered past 15 minutes, read from the record's acts); the switch reads the old section deliveries and says approval-carried for each unperformed merges key. The manifest drops pulls, asks, reads, the forge, teller and view consumes and invokes, and the events group and merged. Builds: design 47 G §172, §431, §592, §593, §613, A-24, A-35 (G-3, G-9, G-16, G-17, G-20) Claude-Session: https://claude.ai/code/session_01JG73KEq4zoDfM9WtYpcQgZ --- .../cmd/mesh-delivery-gate/adr350_test.go | 32 - .../cmd/mesh-delivery-gate/adr505_test.go | 221 ---- .../cmd/mesh-delivery-gate/adr661_test.go | 155 --- .../cmd/mesh-delivery-gate/approval.go | 835 ------------ .../cmd/mesh-delivery-gate/approval_test.go | 523 -------- .../cmd/mesh-delivery-gate/bridge_test.go | 311 ----- .../cmd/mesh-delivery-gate/bus_test.go | 416 +----- .../mesh-delivery-gate/checked_shim_test.go | 142 -- .../cmd/mesh-delivery-gate/checkerbus_test.go | 60 - .../cmd/mesh-delivery-gate/checks.go | 283 ---- .../cmd/mesh-delivery-gate/checks_test.go | 226 ---- .../cmd/mesh-delivery-gate/checktime.go | 34 - .../cmd/mesh-delivery-gate/conditions.go | 59 +- .../cmd/mesh-delivery-gate/configheld.go | 100 -- .../cmd/mesh-delivery-gate/effects.go | 394 ------ .../cmd/mesh-delivery-gate/effects_test.go | 62 - .../cmd/mesh-delivery-gate/events.go | 110 -- .../cmd/mesh-delivery-gate/events_test.go | 175 --- .../cmd/mesh-delivery-gate/fakes_test.go | 445 ------- .../cmd/mesh-delivery-gate/gate.go | 88 ++ .../cmd/mesh-delivery-gate/gitseat.go | 153 --- .../cmd/mesh-delivery-gate/gitseat_test.go | 117 -- .../cmd/mesh-delivery-gate/group_merge.go | 396 ------ .../cmd/mesh-delivery-gate/group_test.go | 157 --- .../cmd/mesh-delivery-gate/groupsaid_test.go | 35 - .../cmd/mesh-delivery-gate/helpers_test.go | 49 - .../cmd/mesh-delivery-gate/holder.go | 1168 ----------------- .../cmd/mesh-delivery-gate/holder_test.go | 399 ------ .../cmd/mesh-delivery-gate/issue1083_test.go | 52 - .../cmd/mesh-delivery-gate/issue1107_test.go | 63 - .../cmd/mesh-delivery-gate/issue1114_test.go | 82 -- .../cmd/mesh-delivery-gate/issue1275_test.go | 742 ----------- .../issue1275_whole_test.go | 332 ----- .../cmd/mesh-delivery-gate/issue1348_test.go | 246 ---- .../cmd/mesh-delivery-gate/issue1349_test.go | 116 -- .../cmd/mesh-delivery-gate/issue1726_test.go | 123 -- .../issue1726_trunk_test.go | 156 --- .../cmd/mesh-delivery-gate/issue326_test.go | 70 - .../cmd/mesh-delivery-gate/issue341_test.go | 226 ---- .../cmd/mesh-delivery-gate/issue403_test.go | 69 - .../cmd/mesh-delivery-gate/issue407_test.go | 105 -- .../cmd/mesh-delivery-gate/issue438_test.go | 183 --- .../cmd/mesh-delivery-gate/issue441_test.go | 234 ---- .../cmd/mesh-delivery-gate/issue443_test.go | 66 - .../cmd/mesh-delivery-gate/issue451_test.go | 48 - .../cmd/mesh-delivery-gate/issue783_test.go | 130 -- .../cmd/mesh-delivery-gate/issue888_test.go | 165 --- .../cmd/mesh-delivery-gate/issue902_test.go | 65 - .../cmd/mesh-delivery-gate/issuelinks.go | 106 -- .../cmd/mesh-delivery-gate/keeps.go | 2 +- .../cmd/mesh-delivery-gate/main.go | 196 +-- .../cmd/mesh-delivery-gate/manifest_test.go | 63 +- .../cmd/mesh-delivery-gate/merges.go | 704 +--------- .../cmd/mesh-delivery-gate/model.go | 362 +---- .../cmd/mesh-delivery-gate/onestatus.go | 403 ------ .../mesh-delivery-gate/onestatus_bus_test.go | 79 -- .../cmd/mesh-delivery-gate/onestatus_test.go | 186 --- .../cmd/mesh-delivery-gate/operator_test.go | 121 -- .../cmd/mesh-delivery-gate/owed.go | 125 +- .../cmd/mesh-delivery-gate/owed_test.go | 129 -- .../cmd/mesh-delivery-gate/owedended.go | 154 +-- .../cmd/mesh-delivery-gate/ownerreads.go | 107 -- .../cmd/mesh-delivery-gate/ownerreads_test.go | 79 -- .../cmd/mesh-delivery-gate/ports.go | 287 ---- .../cmd/mesh-delivery-gate/pullkey.go | 157 --- .../cmd/mesh-delivery-gate/pullkey_test.go | 165 --- .../cmd/mesh-delivery-gate/reached.go | 100 -- .../cmd/mesh-delivery-gate/reached_test.go | 74 -- .../cmd/mesh-delivery-gate/react.go | 16 +- .../cmd/mesh-delivery-gate/ready.go | 677 ---------- .../cmd/mesh-delivery-gate/ready_test.go | 315 ----- .../cmd/mesh-delivery-gate/record_test.go | 163 --- .../cmd/mesh-delivery-gate/recordevents.go | 39 - .../cmd/mesh-delivery-gate/recordfill.go | 55 - .../cmd/mesh-delivery-gate/recordtools.go | 9 + .../cmd/mesh-delivery-gate/settings.go | 59 +- .../cmd/mesh-delivery-gate/switch.go | 193 ++- .../cmd/mesh-delivery-gate/table.go | 383 ------ .../cmd/mesh-delivery-gate/table_test.go | 237 ---- .../cmd/mesh-delivery-gate/timers_test.go | 90 -- .../cmd/mesh-delivery-gate/unchecked.go | 381 ------ .../unchecked_events_test.go | 97 -- .../cmd/mesh-delivery-gate/unchecked_test.go | 80 -- .../cmd/mesh-delivery-gate/verbs.go | 493 +++---- .../cmd/mesh-delivery-gate/waiting_checks.go | 67 - .../cmd/mesh-delivery-gate/writes.go | 114 -- .../cmd/mesh-delivery-gate/writes_test.go | 46 - .../mesh-delivery-gate/files/settings.json | 2 - modules/mesh-delivery-gate/module.json | 56 +- 89 files changed, 619 insertions(+), 16970 deletions(-) delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/bridge_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checked_shim_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checkerbus_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checks.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checks_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/checktime.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/configheld.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/effects.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/effects_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/events_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/fakes_test.go create mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gate.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/gitseat_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_merge.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/group_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/groupsaid_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/helpers_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/holder_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1083_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1107_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1114_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1275_whole_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1348_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1349_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue1726_trunk_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue326_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue341_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue403_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue407_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue438_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue441_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue443_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue451_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue783_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue888_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issue902_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/issuelinks.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_bus_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/onestatus_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/operator_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/owed_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ownerreads_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/pullkey.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/pullkey_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/reached_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/ready_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/table_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/timers_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked_events_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/unchecked_test.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/waiting_checks.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes.go delete mode 100644 modules/mesh-delivery-gate/cmd/mesh-delivery-gate/writes_test.go diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go deleted file mode 100644 index d00273c48..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr350_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "testing" - "time" -) - -const ( - cat = "novox/mesh-catalog" - headCC = "cc00000000000001" - headDu = "dd00000000000002" -) - -var atCC = time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC) - -// A pull request closed unmerged is closed. Since issue 1318 (ADR 0503 §3) the controller is no longer asked to let go -// of what its checks built, and since ADR 0539 §17 the gate asks it nothing at all. -func TestAPullRequestClosedUnmergedIsClosedAndAsksNoOne(t *testing.T) { - w := newWorldOver(t, newMemStore(), newFakeForge(), atCC) - cc := pr(cat, 101, headCC, "feat/claude-code", "modules/claude-code/module.json") - w.h.PullUpdated(cc) - w.h.PullClosed(cc) - if s := w.state(IDOf(cat, headCC)); s != Closed { - t.Fatalf("closed unmerged it is %s", s) - } - du := pr(cat, 102, headDu, "feat/dunst", "modules/dunst/module.json") - w.h.PullUpdated(du) - w.h.PullClosed(du) - if s := w.state(IDOf(cat, headDu)); s != Closed { - t.Fatalf("a closed delivery is %s", s) - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go deleted file mode 100644 index b3c0355a0..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr505_test.go +++ /dev/null @@ -1,221 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "strings" - "testing" - "time" -) - -// The tests of novox/hq ADR 0505 "How it is checked" for the gate: a refusal the gate is already answering is no -// stall yet (§4), and a delivery whose cure is its author's new head is its author's for two hours (§5). - -// ADR 0505 §4, §5: the two bounds are settings with their decided defaults. -func TestTheBoundsAreSettingsWithTheirDefaults(t *testing.T) { - read := func(body string) (Settings, error) { - p := filepath.Join(t.TempDir(), "settings.json") - if err := os.WriteFile(p, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - return readSettings(p) - } - h := &Holder{} - if h.refusalFloor() != 10*time.Minute || h.authorBound() != 2*time.Hour { - t.Fatalf("no settings: floor %s, author %s", h.refusalFloor(), h.authorBound()) - } - s, err := read(`{"refusal-stall-floor-minutes": 5, "author-new-head-minutes": 60}`) - if err != nil { - t.Fatal(err) - } - h.Settings = func() Settings { return s } - if h.refusalFloor() != 5*time.Minute || h.authorBound() != time.Hour { - t.Fatalf("set: floor %s, author %s", h.refusalFloor(), h.authorBound()) - } - s, _ = read(`{}`) - if h.refusalFloor() != 10*time.Minute || h.authorBound() != 2*time.Hour { - t.Fatalf("empty file: floor %s, author %s", h.refusalFloor(), h.authorBound()) - } - if _, err := read(`{"author-new-head-minutes": 0}`); err == nil { - t.Fatalf("a bound of zero minutes was taken") - } -} - -// ADR 0505 §4: a repository's last measured check time is its newest verdict's (`took`, as mesh-controller -// cmd/mesh-builder main.go writes it: a duration rounded to the second); none, and the floor alone holds. -func TestTheLastMeasuredCheckTimeIsTheNewestVerdictsOfItsRepository(t *testing.T) { - h := &Holder{} - h.init() - at := time.Date(2026, 10, 22, 10, 0, 0, 0, time.UTC) - h.deliveries["novox/mesh-catalog@a"] = &Delivery{ID: "novox/mesh-catalog@a", Repository: "novox/mesh-catalog", - Check: &Verdict{At: at, Took: "4m0s"}} - h.deliveries["novox/mesh-catalog@b"] = &Delivery{ID: "novox/mesh-catalog@b", Repository: "novox/mesh-catalog", - Check: &Verdict{At: at.Add(time.Hour), Took: "7m30s"}} - h.deliveries["novox/hq@c"] = &Delivery{ID: "novox/hq@c", Repository: "novox/hq", - Check: &Verdict{At: at.Add(2 * time.Hour), Took: "1m0s"}} - if got := h.lastCheckTime("novox/mesh-catalog"); got != 7*time.Minute+30*time.Second { - t.Fatalf("the newer of two: %s", got) - } - if got := h.answeredRefusalBound("novox/mesh-catalog"); got != 15*time.Minute { - t.Fatalf("twice 7m30s: %s", got) - } - if got := h.answeredRefusalBound("novox/hq"); got != 10*time.Minute { - t.Fatalf("twice one minute is under the floor: %s", got) - } - if got := h.answeredRefusalBound("novox/mesh-sdk"); got != 10*time.Minute { - t.Fatalf("no verdict: %s", got) - } -} - -// refusedForAMovedTrunk is a lone delivery approved and refused by the merge tool because its trunk moved and its check -// was asked again (gitea merge.ts:147, errAskedAgain). -func refusedForAMovedTrunk(t *testing.T) (*world, string) { - t.Helper() - w := announced(t) - id := IDOf("novox/app", readyHead) - hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) - w.forge.refuse = errAskedAgain - approve(w) - if len(w.forge.merges) != 1 || w.delivery(id).Ready.RefusedSince == nil { - t.Fatalf("the merge was not asked and refused: %v %+v", w.forge.merges, w.delivery(id).Ready) - } - return w, id -} - -func linesFor(w *world, id string) []StalledLine { - var out []StalledLine - for _, l := range w.h.Stalled() { - if l.ID == id { - out = append(out, l) - } - } - return out -} - -// ADR 0505 §4: with no answer, the refusal is said at twice the repository's last check time, at least ten minutes. -func TestATrunkMovedRefusalIsStalledAtTwiceTheLastCheckTimeAtLeastTheFloor(t *testing.T) { - w, id := refusedForAMovedTrunk(t) - w.later(9 * time.Minute) - if got := linesFor(w, id); len(got) != 0 { - t.Fatalf("said before the floor: %+v", got) - } - w.later(2 * time.Minute) - if got := linesFor(w, id); len(got) != 1 { - t.Fatalf("not said past the floor of ten minutes: %+v", got) - } - - w, id = refusedForAMovedTrunk(t) - w.h.mu.Lock() - w.h.deliveries[id].Check.Took = "7m0s" // its repository's last check ran seven minutes: the bound is fourteen - w.h.mu.Unlock() - w.later(11 * time.Minute) - if got := linesFor(w, id); len(got) != 0 { - t.Fatalf("said at eleven minutes, under twice the last check time: %+v", got) - } - w.later(4 * time.Minute) - if got := linesFor(w, id); len(got) != 1 { - t.Fatalf("not said past twice the last check time: %+v", got) - } -} - -// ADR 0505 §4: any other refusal is said at once, as issue 1115 has it. -func TestAnyOtherRefusalIsStalledAtOnce(t *testing.T) { - w := announced(t) - id := IDOf("novox/app", readyHead) - hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) - w.forge.refuse = refusedWords("novox/app#7") - approve(w) - if got := linesFor(w, id); len(got) != 1 { - t.Fatalf("a missing status refusal is not said at once: %+v", got) - } -} - -// rejectedBeforeItsAnswer is a delivery approved and then rejected: its check failed after the ask (as in events_test.go), -// its pull request opened by author as the forge's record says, when given. -func rejectedBeforeItsAnswer(t *testing.T, author string) (*world, string) { - t.Helper() - w := newWorld(t) - w.op.auto = false - p := pr("novox/app", 7, readyHead, "feat/x", "src/a.go") - p.Author = author - w.h.PullUpdated(p) - w.settleAll() - id := IDOf("novox/app", readyHead) - hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "pass", "t1t1t1t1t1t1")) - // Judged again (the table's recheck, which the checker's verdict on a new trunk makes): proposed once more. - w.h.mu.Lock() - d := w.h.deliveries[id] - if _, err := Apply(d, EvRecheck, true, Facts{Now: w.h.Now(), Why: "a second look", By: "tester"}); err != nil { - w.h.mu.Unlock() - t.Fatal(err) - } - d.Check = nil - w.h.mu.Unlock() - hear(t, w, shimCheckedKey, judgedAt("novox/app", 7, readyHead, "fail", "t1t1t1t1t1t1")) - w.settleAll() - // A check that fails before the operator answers withdraws the ask (novox/hq ADR 0661 §5, ADR 0441 §1). - if open := w.op.open(); len(open) != 0 { - t.Fatalf("the ask stands after its check failed: %v", open) - } - w.h.React() - if w.state(id) != Rejected { - t.Fatalf("not rejected: %s", w.state(id)) - } - return w, id -} - -// ADR 0505 §5: a rejected delivery is told on its pull request at once, and reaches the operator only after two hours -// with no new head, in words naming its author (its branch, and the session that pushed it) and needing a new head — -// never "stop it", never "the operator's". -func TestARejectedDeliveryIsItsAuthorsForTwoHours(t *testing.T) { - w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries") - said := false - for _, c := range w.forge.comments { - said = said || strings.Contains(c, "not merged") - } - if !said { - t.Fatalf("its author was not told on the pull request at once: %v", w.forge.comments) - } - w.later(2*time.Hour - time.Minute) - if got := linesFor(w, id); len(got) != 0 { - t.Fatalf("the operator's before two hours: %+v", got) - } - w.later(2 * time.Minute) - got := linesFor(w, id) - if len(got) != 1 { - t.Fatalf("not the operator's after two hours with no new head: %+v", got) - } - l := got[0] - if l.Needs != NeedsNewHead || l.Author != "the author of feat/x (quiet-deliveries)" || l.Branch != "feat/x" { - t.Fatalf("the line does not name its author and the new head it needs: %+v", l) - } - for _, never := range []string{"operator's", "stop it"} { - if strings.Contains(l.Says+l.H2, never) { - t.Fatalf("the line says %q: %+v", never, l) - } - } - if !strings.Contains(l.Says, "needs a new head from the author of feat/x (quiet-deliveries)") { - t.Fatalf("the words do not say a new head is needed from its author: %q", l.Says) - } -} - -// ADR 0505 §5: with no author in the forge's record, the branch alone names its author. -func TestARejectedDeliveryWithNoAuthorIsNamedByItsBranch(t *testing.T) { - w, id := rejectedBeforeItsAnswer(t, "") - w.later(2*time.Hour + time.Minute) - if got := linesFor(w, id); len(got) != 1 || got[0].Author != "the author of feat/x" { - t.Fatalf("not named by its branch: %+v", got) - } -} - -// ADR 0505 §5: a new head within the two hours ends it: the rejected head is never the operator's. -func TestANewHeadWithinTwoHoursKeepsTheRejectedHeadFromTheOperator(t *testing.T) { - w, id := rejectedBeforeItsAnswer(t, "quiet-deliveries") - w.later(time.Hour) - w.h.PullUpdated(pr("novox/app", 7, pushed, "feat/x", "src/a.go")) - w.settleAll() - w.later(2 * time.Hour) - if got := linesFor(w, id); len(got) != 0 { - t.Fatalf("the superseded head reached the operator: %+v", got) - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go deleted file mode 100644 index 19e7d353e..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/adr661_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package main - -import ( - "encoding/json" - "reflect" - "strings" - "testing" - "time" -) - -// No check gates a merge (novox/hq ADR 0661), the gate's part: no approved merge waits for another delivery's running check, -// and no turn or stalled line holds it (§1); the merge judges the operator's Approve alone, so an Approve given before a trunk -// move merges onto the new trunk with no wait for its carry (§3); a failure after the Approve is said once on the pull -// request and as a stalled line naming the merge it follows, and refuses nothing (§5). - -const otherHead = "c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3" - -// approvedLone is a lone delivery into oneRepo whose check passed, approved by the operator (the world approves at once). -func approvedLone(w *world) string { - w.t.Helper() - p := pr(oneRepo, 1, oneHead, "feat/one") - requiresCheck(w, p) - w.h.PullUpdated(p) - putCheckerVerdict(w, "pass", headOf(oneRepo, 1, oneHead)) - w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk)) - return IDOf(oneRepo, oneHead) -} - -// sayingFailed is the forge's word that oneHead's check failed on the trunk commit given (a carry's line). -func sayingFailed(trunk string) ForgeStatusEvent { - e := checkStatus(oneRepo, oneHead, "failure", trunk) - e.Description = "failed " + oneHead[:12] + " on " + trunk[:12] + " | a test failed" - return e -} - -func lineOf(w *world, id string) *StalledLine { - for _, l := range w.h.Stalled() { - if l.ID == id { - return &l - } - } - return nil -} - -func TestNoCheckGatesAMergeAtTheGate(t *testing.T) { - for name, run := range map[string]func(t *testing.T){ - // §1: a running check of other heads on the same repository holds nothing, and no stalled line is raised for it. - "an approved merge into a repository under a running check merges at once": func(t *testing.T) { - w := newWorld(t) - putCheckerVerdict(w, "running", headOf(oneRepo, 9, otherHead)) - id := approvedLone(w) - w.settleAll() - if w.op.count() != 1 || len(w.forge.merges) != 1 { - t.Fatalf("asked %d times, merges %v: want asked once and merged at once (%s)", w.op.count(), w.forge.merges, w.logs()) - } - w.h.PullMerged(merged(oneRepo, 1, oneHead, strings.Repeat("c", 40))) // the seat git says the merge too - w.later(time.Hour) - w.settleAll() - if l := lineOf(w, id); l != nil { - t.Fatalf("a stalled line for a merge a running check once held: %+v", l) - } - }, - "the delivery's own running verdict does not hold it": func(t *testing.T) { - w := newWorld(t) - w.op.auto = false - id := approvedLone(w) - w.settleAll() - putCheckerVerdict(w, "running", headOf(oneRepo, 1, oneHead)) // its own heads judged again (a carry) - for _, k := range w.op.open() { - w.op.answer(k, OptionApprove) - } - w.settleAll() - if d := w.delivery(id); len(w.forge.merges) != 1 || d.Ready == nil { - t.Fatalf("merges %v, run %+v: its own check held it", w.forge.merges, d.Ready) - } - }, - // §3 (and ADR 0548 §8): the ask stands through a trunk move; the Approve given then merges onto the new trunk on the - // verdict judged against the old one, with no wait for its carry and no second ask. - "an Approve given after a trunk move merges at once, its carry after": func(t *testing.T) { - w := newWorld(t) - w.op.auto = false - id := approvedLone(w) - w.settleAll() - if w.op.count() != 1 || len(w.forge.merges) != 0 { - t.Fatalf("asked %d times, merges %v: want asked and not merged yet", w.op.count(), w.forge.merges) - } - w.h.PullMerged(merged(oneRepo, 5, otherHead, oneTrunk2)) // a merge at the forge moves the trunk - w.settleAll() - for _, k := range w.op.open() { - w.op.answer(k, OptionApprove) - } - w.settleAll() - d := w.delivery(id) - if len(w.forge.merges) != 1 || w.op.count() != 1 || d.State != Ready && d.State != Merged { - t.Fatalf("merges %v, asked %d times, %s (waits %q): want merged at once on the old verdict (%s)", w.forge.merges, - w.op.count(), d.State, d.Ready.Waits, w.logs()) - } - }, - // §5: a failure after the Approve, a carry's, is said once on the pull request and as one stalled line naming the merge; - // it refuses nothing, and a later pass clears the line. - "a failure after the Approve is said once and refuses nothing": func(t *testing.T) { - w := newWorld(t) - id := approvedLone(w) - w.settleAll() - if len(w.forge.merges) != 1 { - t.Fatalf("merges %v: want merged at once", w.forge.merges) - } - before := len(w.forge.comments) - w.h.CommitStatus(sayingFailed(oneTrunk2)) - w.settleAll() - w.h.CommitStatus(sayingFailed(oneTrunk2)) - w.settleAll() - said := 0 - for _, c := range w.forge.comments[before:] { - if strings.Contains(c, "failed after the Approve") { - said++ - } - } - if said != 1 { - t.Fatalf("said %d times on the pull request, want once: %v", said, w.forge.comments[before:]) - } - l := lineOf(w, id) - if l == nil || !strings.Contains(l.Says, "after the operator's Approve") || !strings.Contains(l.Says, oneHead[:12]) { - t.Fatalf("the stalled line = %+v", l) - } - if d := w.delivery(id); d.State == Rejected || len(w.forge.merges) != 1 { - t.Fatalf("%s, merges %v: the failure refused the approved merge", d.State, w.forge.merges) - } - w.h.CommitStatus(checkStatus(oneRepo, oneHead, "success", oneTrunk2)) - w.settleAll() - if l := lineOf(w, id); l != nil { - t.Fatalf("a later pass left the stalled line: %+v", l) - } - }, - } { - t.Run(name, run) - } -} - -// Issue 1794: the gate keeps no mark of an own ask of a check, so no verdict of another check of the head can clear one while -// the gate's own is in flight: since issue 1318 the gate asks no check, only the checker's verdict at the heads is read, and a -// run holds no field that marks an ask of one. -func TestTheGateAsksNoCheckOfItsOwnSoNoMarkOfOneCanBeClearedByAnother(t *testing.T) { - typ := reflect.TypeOf(ReadyRun{}) - for i := 0; i < typ.NumField(); i++ { - if f := typ.Field(i).Name; strings.HasPrefix(f, "OwnAsk") || strings.HasPrefix(f, "OwnCheck") { - t.Errorf("ReadyRun holds %s: a mark of a check the gate asked itself, which it no longer asks", f) - } - } - for _, key := range []string{"own_asked_at", "own_ask"} { - if raw, _ := json.Marshal(ReadyRun{}); strings.Contains(string(raw), key) { - t.Errorf("a run is kept with %s", key) - } - } -} diff --git a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go b/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go deleted file mode 100644 index 9b4c5718f..000000000 --- a/modules/mesh-delivery-gate/cmd/mesh-delivery-gate/approval.go +++ /dev/null @@ -1,835 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - "time" - - occ "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/clients/operatorchannelclient" - oc "git.novox.be/novox/mesh-contracts/operator-and-conversation/go/seats/operatorchannel" - mdc "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshdeliveryclient" - stdio "git.novox.be/novox/mesh-sdk/go" -) - -// A merge into main waits for the operator's Approve (novox/hq ADR 0398 §3, issue 948, ADR 0421). -// -// This owner writes the ask into its bucket asks (novox/hq ADR 0437 §6) as soon as a pull request's checks pass: -// Approve or Decline, each option binding the act "merge these heads". The teller publishes it to the operator on the -// router and, on a warrant that holds the ask (its asker, the ask's digest, an option it offered, in time, and the act -// that option bound), writes the answer into its bucket answers. This owner reads the answer and acts on it only while -// the heads of the key it was written under are still its group's heads; it asks the router nothing. -// -// - Asked once per head. A group is asked once, for all its members' heads together, by the first member that reaches -// this point; the Approve covers every member at those heads. -// - A push after the ask ends it, said once on the pull request: the new head is asked on its own. -// - A Decline merges nothing and is said once on the pull request. -// - An ask that ended without a choice (expired, cancelled, refused by the router) is asked again when the router says so. -// - The ask, its state and the warrant's words are kept on the delivery's ReadyRun and shown by `show`. -// -// The terminal Approve (ADR 0398 §3) is issue 953; today the proving channel is the operator's phone. - -// Module is this module's name: the one the bus names its buckets and events under (novox/hq ADR 0437 §3). Seat is -// the seat it claims, whose name stays `mesh-delivery` (ADR 0437 §11). -const ( - Module = "mesh-delivery-gate" - Seat = "mesh-delivery" -) - -// The two answers, both at the level approve. -const ( - OptionApprove = "approve" - OptionDecline = "decline" -) - -// AskLasts is how long the operator has to answer: a day less a margin, under the router's bound for an approval. -const AskLasts = occ.ApproveLasts - 10*time.Minute - -// Operator is how this owner asks the operator: through the keys of its bucket asks and the teller's answers. kvOperator in -// the running module. -type Operator interface { - // Ask writes the ask under its key (mdc.MergeAskKey); it writes nothing when that key holds the same ask already. - Ask(key string, a Ask) error - // Cancel takes the ask at a key away, which the teller reads as the ask withdrawn. - Cancel(key string) error - // Where is the teller's answer to the ask at a key, or nil when there is none. - Where(key string) (*Answer, error) -} - -// askGroup is the name a delivery's ask and merge are kept under: its group's, or, for a lone pull request, a group of one -// under its branch name (novox/hq ADR 0437 §6). -func askGroup(d *Delivery) string { - switch { - case d.Group != "": - return d.Group - case d.Branch != "": - return d.Branch - } - return d.ID -} - -// headsString is the heads an act names, as an act's value: "=" for each, sorted, joined by commas. -func headsString(heads map[string]string) string { - var out []string - for id, sha := range heads { - out = append(out, id+"="+sha) - } - sort.Strings(out) - return strings.Join(out, ",") -} - -// mergeAct is what an answer binds: this ask, the answer, and the heads it covers. -func mergeAct(ask, option string, heads map[string]string) occ.Act { - return occ.Act{"verb": Seat + ".merge", "ask": ask, "option": option, "heads": headsString(heads)} -} - -var unshownPath = regexp.MustCompile(`[/\\@~$]|\d+\.\d+\.\d+|[A-Za-z0-9+/=_]{24,}|(?i)\b[a-z0-9-]+\.[a-z]{2,12}\b`) - -// phone is a line the operator's phone may be shown: what the router refuses in an ask's words (a path, an address, a -// host-like name, a long unbroken word) is left out, and the line is cut to n characters. -func phone(s string, n int) string { - s = strings.Join(strings.Fields(unshownPath.ReplaceAllString(s, "…")), " ") - if r := []rune(s); len(r) > n { - return string(r[:n-1]) + "…" - } - return s -} - -// issuesSaid is the issues linked to the deliveries' pull requests in the issue tracker, as the ask names them. -func issuesSaid(ds []*Delivery) string { - seen := map[int]bool{} - var out []string - for _, d := range ds { - if d.IssuesUnread != "" { - out = append(out, pullSaid(d)+"'s unread (the issue tracker did not answer)") - } - for _, n := range d.Issues { - if !seen[n] { - seen[n] = true - out = append(out, "issue "+strconv.Itoa(n)) - } - } - } - if len(out) == 0 { - return "none linked" - } - return strings.Join(out, ", ") -} - -func repoName(repository string) string { - _, name, ok := strings.Cut(repository, "/") - if !ok { - return repository - } - return name -} - -func pullSaid(d *Delivery) string { return fmt.Sprintf("%s#%d", repoName(d.Repository), d.Number) } - -// touchesSaid is the modules the delivery's check says its change touches, for the ask; the gate plans no delivery. -func touchesSaid(d *Delivery) string { - if d.Check == nil || len(d.Check.Modules) == 0 { - return "no module the check named" - } - return strings.Join(d.Check.Modules, ", ") -} - -// checkClause is the check's state as the ask's summary says it after the pull request's or the group's name: sep and -// "passed its check", or ": its check has no verdict yet: it is running (since 21:05 UTC)" and the like. -func checkClause(c checkSaid, sep string) string { - if c.passed() { - return sep + c.Words - } - return ": " + c.Words -} - -// approveCaveat is what an Approve does beyond the merge while the check has not passed: it merges although it failed (the check gates -// no merge, check-gates-merge false). Empty once it passed; an ask is made only with a verdict. -func approveCaveat(c checkSaid) string { - switch c.Key { - case checkSaidPass: - return "" - case checkSaidFail: - return " although its check failed" - } - return "" -} - -// mergeHeadline opens the ask's headline: "Merge into main: ", and "Check failed - merge into main: " once the check failed, -// since a merge is not gated on the check and the headline is the phone's only warning (issue 1166). A passed or still -// running check keeps the plain headline. The prefix leads, so phone's cut of a long headline never drops it. -func mergeHeadline(c checkSaid) string { - if c.Key == checkSaidFail { - return "Check failed \u2014 merge into main: " - } - return "Merge into main: " -} - -// composeMergeAsk is the ask for a delivery, or for a group's members together. -// check is the state of the check at the heads it covers (checkAt): the ask says "passed its check" only when it did, and -// otherwise that Approve merges before the check ends, or although it failed (issue 1733). -func composeMergeAsk(id string, d *Delivery, group string, members []*Delivery, heads map[string]string, check checkSaid, - now time.Time) oc.Ask { - var a oc.Ask - a.ID, a.Who, a.Expires = id, occ.Operator, occ.FormatTime(now.Add(AskLasts)) - a.OnExpiry = "nothing is merged; the ask is made again" - a.About = "delivery." + phone(repoName(d.Repository), 40) + "." + strconv.Itoa(d.Number) - a.About = strings.ReplaceAll(a.About, " ", "-") - var reaches, details []string - for _, m := range members { - touches := touchesSaid(m) - reaches = append(reaches, touches) - details = append(details, fmt.Sprintf("%s %q at %s: issues: %s; touches: %s", - pullSaid(m), m.Title, shortOf(m.Commit, 12), issuesSaid([]*Delivery{m}), touches)) - } - if group == "" { - a.Headline = phone(mergeHeadline(check)+pullSaid(d)+" "+d.Title, occ.HeadlineLength) - summary := fmt.Sprintf("%s %s%s. Approve merges it into main at %s%s; Decline leaves it unmerged.", - pullSaid(d), d.Title, checkClause(check, " "), shortOf(d.Commit, 12), approveCaveat(check)) - a.Explanation = strings.Join([]string{ - phone(summary, occ.SummaryLength), - phone("Issues: "+issuesSaid(members), occ.PointLength), - phone("Touches: "+reaches[0], occ.PointLength), - }, "\n") - } else { - var names []string - for _, m := range members { - names = append(names, pullSaid(m)) - } - a.Headline = phone(mergeHeadline(check)+"group "+group+" ("+strings.Join(names, ", ")+")", occ.HeadlineLength) - caveat := approveCaveat(check) - if caveat != "" { - caveat = "," + caveat - } - summary := fmt.Sprintf("The group %s, %d pull requests%s. Approve merges all of them into main in one pass, at the heads named%s; Decline leaves them unmerged.", - group, len(members), checkClause(check, ", "), caveat) - a.Explanation = strings.Join([]string{ - phone(summary, occ.SummaryLength), - phone("Issues: "+issuesSaid(members), occ.PointLength), - phone("Touches: "+strings.Join(reaches, "; "), occ.PointLength), - }, "\n") - } - a.Details = phone(strings.Join(details, "\n"), 1500) - if len(details) > 0 { // keep the lines apart: phone joined them - a.Details = strings.Join(func() []string { - var out []string - for _, l := range details { - out = append(out, phone(l, 400)) - } - return out - }(), "\n") - } - for _, o := range []struct{ id, label, does string }{ - {OptionApprove, "Approve", "the pull request is merged into main at the head named (a group: all its members, in one pass)"}, - {OptionDecline, "Decline", "nothing is merged"}, - } { - binds, _ := occ.ActDigest(mergeAct(id, o.id, heads)) - a.Options = append(a.Options, oc.Option{ID: o.id, Label: o.label, Does: o.does, Level: occ.Approve, Binds: binds}) - } - return a -} - -// askMembers are the deliveries one ask covers: the group's members not yet merged, by id, or the delivery. -func (h *Holder) askMembers(d *Delivery) []*Delivery { - g := h.groups[d.Group] - if d.Group == "" || g == nil { - return []*Delivery{d} - } - var out []*Delivery - seen := map[string]bool{} - ids := append([]string(nil), g.Members...) - sort.Strings(ids) - for _, id := range ids { - if o := h.deliveries[id]; o != nil && !seen[id] && o.MergedAs == "" && !o.State.Final() { - seen[id] = true - out = append(out, o) - } - } - if len(out) == 0 { - return []*Delivery{d} - } - return out -} - -// covering is the ready run holding an ask (pending or approved) that covers d at its head: its own, or a group -// sibling's, whose ask named d's head too. -func (h *Holder) covering(d *Delivery) *ReadyRun { - has := func(o *Delivery) *ReadyRun { - if o != nil && o.Ready != nil && (o.Ready.AskID != "" || o.Ready.Approved != nil) && o.Ready.Heads[d.ID] == d.Commit { - return o.Ready - } - return nil - } - if r := has(d); r != nil { - return r - } - if g := h.groups[d.Group]; d.Group != "" && g != nil { - for _, id := range g.Members { - if r := has(h.deliveries[id]); r != nil { - return r - } - } - } - return nil -} - -func headsEqual(a, b map[string]string) bool { - if len(a) != len(b) { - return false - } - for k, v := range a { - if b[k] != v { - return false - } - } - return true -} - -// headsHold is whether an ask's heads are still the heads of d's group: every member now at the head asked, and every -// head asked that is no member now a member already merged. -func (h *Holder) headsHold(d *Delivery, c *ReadyRun) bool { - now := map[string]bool{} - for _, m := range h.askMembers(d) { - now[m.ID] = true - if c.Heads[m.ID] != m.Commit { - return false - } - } - for id := range c.Heads { - if !now[id] { - if o := h.deliveries[id]; o == nil || o.MergedAs == "" { - return false - } - } - } - return true -} - -// endStaleAsks ends every ask whose heads are no longer its group's: a push to any member ends the approval (ADR 0398 -// §3), and says so once on each pull request it covered (ADR 0421 §6). An ask for a head a newer one replaced ends too. -// A Decline stands, and is not ended. An ask none of whose deliveries can still merge by it is taken away after -// (endAsksOfFinished). Under the lock; the cancels run with the look's asks. -func (h *Holder) endStaleAsks() { - ending := map[string]bool{} - for _, d := range h.deliveries { - r := d.Ready - if r == nil || r.Merged != nil || d.MergedAs != "" || r.declined() || (r.AskID == "" && r.Approved == nil) || len(r.Heads) == 0 { - continue - } - id, before := r.AskID, len(h.cancels) - switch { - case d.State.Final(): - h.cancels = append(h.cancels, h.dropAsk(r.AskID, "its head "+shortOf(d.Commit, 12)+" is no longer the pull request's head, or the pull request closed")) - case !h.headsHold(d, r): - h.cancels = append(h.cancels, h.dropAsk(r.AskID, "a head it covers moved on: the heads it covers are no longer the group's heads")) - case r.Approved == nil && r.AskID != "" && r.AskCheck != "" && h.checkMoved(d, r.AskCheck): - // The check gates the ask, so it was made on a pass: once that pass is gone (a failure, or a check of these heads - // running again), the ask not yet answered is withdrawn in words that say so (novox/hq ADR 0661 §5, ADR 0441 §1). - h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, "the check at its heads is now: "+h.checkAt(h.askMembers(d)).Words)) - case r.Approved == nil && r.AskID != "" && !r.AskedAt.IsZero() && !h.Now().Before(r.AskedAt.Add(AskLasts)): - // The ask lapsed at the router with nobody's answer, and nothing in the gate's inputs says so: it is taken away and - // written again, which the teller asks afresh (ADR 0441 §1). - h.cancels = append(h.cancels, h.withdrawAsk(r.AskID, "nobody answered in "+AskLasts.String())) - } - if id != "" && (len(h.cancels) > before || h.withdrawn[id]) { - ending[id] = true - } - } - h.endAsksOfFinished(ending) -} - -// endAsksOfFinished takes away every ask that stands in the gate's asks while none of the deliveries it was asked for can still -// be merged by it: each merged (on the operator's Approve or by another path) or ended. The ask is about a merge, so it ends -// once that merge cannot follow from it, however the deliveries got there (issue 774: on 2026-10-31 two pull requests the lead -// merged once their checks were green left their asks standing for a day, and the teller's asks held two of the router's three -// places, so a newer ask was refused for want of room). It reads what the gate keeps (AskedKey, Written), so an ask whose -// taking away failed is taken away again at the next look. An ask another end of this look takes away (ending, withdrawn) is -// left to it. Nothing is said on a pull request: its merge or its end said it. Under the lock. -func (h *Holder) endAsksOfFinished(ending map[string]bool) { - standing, live := map[string]bool{}, map[string]bool{} - for _, d := range h.deliveries { - r := d.Ready - if r == nil || r.AskedKey == "" { - continue - } - if r.Written { - standing[r.AskedKey] = true - } - if d.MergedAs == "" && !d.State.Final() { - live[r.AskedKey] = true - } - } - keys := make([]string, 0, len(standing)) - for key := range standing { - if !live[key] && !ending[key] && !h.withdrawn[key] { - keys = append(keys, key) - } - } - sort.Strings(keys) - for _, key := range keys { - for _, d := range h.deliveries { - if r := d.Ready; r != nil && r.AskID == key && r.Approved == nil { - r.AskID, r.Ask, r.Heads = "", nil, nil - h.keep(d) - } - } - h.Logf("[mesh-delivery] the operator's ask %s is taken away: every pull request it was asked for is merged or ended", key) - if h.Operator == nil { - continue - } - h.cancels = append(h.cancels, func() { - if err := h.Operator.Cancel(key); err != nil { - h.Logf("[mesh-delivery] the operator's ask %s could not be taken away, and is taken away at the next look (%v)", key, err) - return - } - h.mu.Lock() - h.markWritten(key, false) - h.mu.Unlock() - }) - } -} - -// checkMoved is whether the check at the heads of d's ask reached a verdict its pending ask does not say: it passed or failed -// since, or the pass it said is gone. Under the lock. -func (h *Holder) checkMoved(d *Delivery, said string) bool { - now := h.checkAt(h.askMembers(d)).Key - if now == said { - return false - } - return now == checkSaidPass || now == checkSaidFail || said == checkSaidPass || said == checkSaidFail -} - -// approval is what a delivery that may merge waits on for the operator: empty once its head is approved, and the ask -// to make when none is made yet. Under the lock. -func (h *Holder) approval(d *Delivery, now time.Time) (string, func()) { - r := d.Ready - c := h.covering(d) - if c != nil { - if c.Approved != nil { - r.Approved, r.Warrant, r.AskID, r.Ask, r.Heads = c.Approved, c.Warrant, c.AskID, c.Ask, c.Heads - return "", nil - } - if !c.AskedAt.IsZero() { - h.lookAt(c.AskedAt.Add(AskLasts), "the operator's Approve, unanswered until the ask lapses") - } - return "the operator's Approve is asked (" + c.AskID + ")", nil - } - if r.Refused != "" { - return "the operator's answer was refused here: " + r.Refused, nil - } - if h.Operator == nil { - return "the operator cannot be asked from here", nil - } - members := h.askMembers(d) - if waits := h.writeWaits(d, members, now); waits != "" { - return waits, nil // asked only once the forge account can carry the merge out (writes.go, issue 1945) - } - if read := h.issuesToRead(members); read != nil { - return "the issues linked to its pull request are read from the issue tracker", read - } - group := "" - if len(members) > 1 { - group = d.Group - } - heads := map[string]string{} - var covered []mdc.Head - for _, m := range members { - heads[m.ID] = m.Commit - covered = append(covered, mdc.Head{Repository: m.Repository, Number: m.Number, Head: m.Commit}) - } - name := askGroup(d) - id, err := mdc.MergeAskKey(name, covered) - if err != nil { - return "the operator cannot be asked: " + err.Error(), nil - } - r.Asks++ - // Each asking at these heads has its own router id (mdc.AskingID): the router keeps an ask by its id and drops a second ask - // under it as a redelivery, so an ask written again at unchanged heads (a lapse, or a status changing) is - // the next asking, with a new id (review round 1, G2; issue 1370). - asking := 1 - for _, m := range members { - if m.Ready != nil && m.Ready.AskedKey == id { - asking = max(asking, m.Ready.Asking+1) - } - } - check := h.checkAt(members) - ask := composeMergeAsk(mdc.AskingID(id, asking), d, group, members, heads, check, now) - taken := []*Delivery{} - var reads []memberRead - for _, m := range members { - h.runOf(m, now) - m.Ready.AskID, m.Ready.Ask, m.Ready.Heads, m.Ready.AskedAt, m.Ready.Asks = id, &ask, heads, now, max(m.Ready.Asks, r.Asks) - m.Ready.AskedKey, m.Ready.Asking, m.Ready.AskCheck = id, asking, check.Key - m.Ready.Declined, m.Ready.AnsweredAsk = nil, "" // a new ask: an answer to an earlier one does not stand on it - h.keep(m) - taken = append(taken, m) - reads = append(reads, readOf(m, m.Commit)) - } - h.Logf("[mesh-delivery] %s (%s): the operator is asked to Approve its merge into main (%s, ask %d)", - d.ID, check.Words, id, r.Asks) - gid := "" - if group != "" { - gid = d.Group - } - return "the operator's Approve is asked (" + id + ")", func() { - // A group is not asked while a member's required status is not a success at its head (issue 1275): the group - // says which, and is asked once every member may merge. - var holdMember, holdSays string - var readErr error - for _, m := range reads { - if gid == "" { - break - } - says, forge, err := h.notASuccess(m) - if err != nil { - readErr = err - break - } - if says != "" && !forge { // the forge's merge answer is weighed at the merge, where it is asked again (group_merge.go) - holdMember, holdSays = m.id, says - break - } - } - undo := func() { - for _, m := range taken { - if m.Ready != nil && m.Ready.AskID == id { - m.Ready.AskID, m.Ready.Ask, m.Ready.Heads = "", nil, nil - h.keep(m) - } - } - } - // An earlier asking of this key may still stand in the bucket: not written again, it is taken away. - stale := func() { - h.mu.Lock() - standing := h.standing(id) - h.mu.Unlock() - if standing { - if err := h.Operator.Cancel(id); err != nil { - h.Logf("[mesh-delivery] the operator's ask %s could not be withdrawn (%v)", id, err) - return - } - h.mu.Lock() - h.markWritten(id, false) - h.mu.Unlock() - } - } - // Nothing is asked that the forge account could not merge (writes.go, issue 1945): said at the next look. - if says := h.cannotWrite(reads); says != "" { - stale() - h.mu.Lock() - defer h.mu.Unlock() - undo() - h.Logf("[mesh-delivery] %s (%s): the operator is not asked: %s", d.ID, check.Words, says) - h.lookAt(h.Now(), "an ask not made: the forge account cannot write a repository it covers") - return - } - if readErr != nil || holdSays != "" { - stale() - h.mu.Lock() - defer h.mu.Unlock() - undo() - if g := h.groups[gid]; g != nil && holdSays != "" { - if g.Hold == nil || g.Hold.Kind != "unasked" || g.Hold.Says != holdSays || g.Hold.Member != holdMember { - h.Logf("[mesh-delivery] group %s is not asked for its Approve: %s waits on a required status: %s", gid, holdMember, holdSays) - g.Hold = &GroupHold{Kind: "unasked", Member: holdMember, Says: holdSays, Since: h.Now()} - h.keepGroup(g) - } - } - if readErr != nil { - h.Logf("[mesh-delivery] group %s: the required statuses cannot be read, so the operator is not asked yet: %v", gid, readErr) - } - h.again("a group's Approve held for a required status") - return - } - if g := func() *Group { h.mu.Lock(); defer h.mu.Unlock(); return h.groups[gid] }(); g != nil && g.Hold != nil && g.Hold.Kind == "unasked" { - h.mu.Lock() - g.Hold = nil - h.keepGroup(g) - h.mu.Unlock() - } - toAsk, err := gateAsk(name, covered, asking, ask) - if err == nil { - err = h.Operator.Ask(id, toAsk) - } - if err == nil { - h.mu.Lock() - h.markWritten(id, true) - h.mu.Unlock() - } else { - h.mu.Lock() - defer h.mu.Unlock() - undo() - h.again("the operator's Approve, which could not be asked") - h.Logf("[mesh-delivery] the operator could not be asked about %s (%v); asked again in %s", d.ID, err, askAgainAfter) - } - } -} - -// dropAsk forgets a pending ask on every delivery it covers and takes it back from the router; under the lock. What it -// says is said once on each pull request it covered (ADR 0421 §6): the Approve is ended, and why. -func (h *Holder) dropAsk(id, why string) func() { - var said []func() - seen := map[string]bool{} - var covered []*Delivery - for _, d := range h.deliveries { - if d.Ready != nil && d.Ready.AskID == id { - covered = append(covered, d) - } - } - sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) - for _, d := range covered { - d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil - if d.Ready.Merged == nil { - d.Ready.Approved, d.Ready.Warrant = nil, "" - } - h.keep(d) - if k := fmt.Sprintf("%s#%d", d.Repository, d.Number); !seen[k] && d.Number != 0 { - seen[k] = true - said = append(said, h.sayOn(d, "mesh-delivery: the operator's Approve of this pull request's merge into main is ended: "+why+ - ". Nothing is merged on it. The operator is asked again once the checks pass at the new head.")) - } - } - h.Logf("[mesh-delivery] the operator's ask %s is cancelled: %s", id, why) - return func() { - for _, s := range said { - s() - } - if h.Operator == nil { - return - } - if err := h.Operator.Cancel(id); err != nil { - h.Logf("[mesh-delivery] the operator's ask %s could not be cancelled (%v)", id, err) - return - } - h.mu.Lock() - h.markWritten(id, false) - h.mu.Unlock() - } -} - -// withdrawAsk takes back an ask nobody answered, on every delivery it covers, so the next look asks again with what is -// known now (issue 1079). Unlike dropAsk it ends no Approve and says nothing on a pull request: there was none. Under the -// lock; the router's cancel runs with the look's asks. -func (h *Holder) withdrawAsk(id, why string) func() { - for _, d := range h.deliveries { - if d.Ready != nil && d.Ready.AskID == id && d.Ready.Approved == nil { - d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil - h.keep(d) - } - } - // The key is not taken away: the next look writes it again with the next asking, if the group may be asked, and takes it away - // only if it may not (readyAsks, withdrawnKeys), so the key is never absent between the two. - h.withdrawn[id] = true - h.Logf("[mesh-delivery] the operator's ask %s is withdrawn to be asked again: %s", id, why) - return func() {} -} - -// withdrawnKeys are, after a look, the closures that take away the asks withdrawn and not written again by it. Under the lock. -func (h *Holder) withdrawnKeys() []func() { - var out []func() - for key := range h.withdrawn { - delete(h.withdrawn, key) - again := false - for _, d := range h.deliveries { - again = again || (d.Ready != nil && d.Ready.AskID == key) - } - if again || h.Operator == nil { - continue - } - key := key - out = append(out, func() { - if err := h.Operator.Cancel(key); err != nil { - h.Logf("[mesh-delivery] the operator's ask %s could not be withdrawn (%v)", key, err) - return - } - h.mu.Lock() - h.markWritten(key, false) - h.mu.Unlock() - }) - } - return out -} - -// pendingAsks are the asks waiting for an answer, by id; under the lock. -func (h *Holder) pendingAsks() []string { - seen := map[string]bool{} - var out []string - for _, d := range h.deliveries { - if r := d.Ready; r != nil && r.AskID != "" && r.Approved == nil && !r.declined() && !seen[r.AskID] { - seen[r.AskID] = true - out = append(out, r.AskID) - } - } - sort.Strings(out) - return out -} - -// hearAsks reads the teller's answer to every pending ask and applies each that is there. -func (h *Holder) hearAsks() { - if h.Operator == nil { - return - } - h.mu.Lock() - keys := h.pendingAsks() - h.mu.Unlock() - for _, key := range keys { - answer, err := h.Operator.Where(key) - if err != nil { - h.Logf("[mesh-delivery] the teller's answer to the ask %s cannot be read: %v", key, err) - continue - } - if answer == nil { - continue - } - h.mu.Lock() - said := h.answered(key, *answer) - h.mu.Unlock() - for _, s := range said { - s() - } - } -} - -// answered applies the teller's answer to the ask at a key to the deliveries it covers; under the lock. The teller has -// held the warrant to the ask (novox/hq ADR 0437 §6); this owner acts on the answer only if the key it was written under is -// still the key of the heads asked: an answer to an ask taken away, or to other heads, merges nothing (ADR 0441 §1). It -// returns what to say on the pull requests. -func (h *Holder) answered(key string, ans Answer) []func() { - var covered []*Delivery - for _, d := range h.deliveries { - if d.Ready != nil && d.Ready.AskID == key { - covered = append(covered, d) - } - } - if len(covered) == 0 { - h.Logf("[mesh-delivery] the answer to the ask %s names no ask that is asked now: nothing is merged on it", key) - return nil - } - sort.Slice(covered, func(i, j int) bool { return covered[i].ID < covered[j].ID }) - first, now := covered[0].Ready, h.Now() - refuse := func(why string) { - h.Logf("[mesh-delivery] REFUSED the answer to the ask %s: %s", key, why) - for _, d := range covered { - d.Ready.AskID, d.Ready.Ask, d.Ready.Heads = "", nil, nil - d.Ready.Refused = "was answered, and the answer does not hold here: " + clip(oneLine(why), 300) - h.keep(d) - } - } - var heads []mdc.Head - for id, sha := range first.Heads { - m := h.deliveries[id] - if m == nil { - refuse("the head " + id + " it covers is no delivery of this owner") - return nil - } - heads = append(heads, mdc.Head{Repository: m.Repository, Number: m.Number, Head: sha}) - } - if want, err := mdc.MergeAskKey(askGroup(covered[0]), heads); err != nil || want != key { - refuse(fmt.Sprintf("its key %s is not the key %s of the heads asked", key, want)) - return nil - } - // The answer is to the asking asked now: the teller names the router id it was given in the warrant's id (`@