diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go index 2a802eb9..9c88735b 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go @@ -62,3 +62,31 @@ func (f forgeReads) File(_ context.Context, repository, path, ref string) (strin } return *a.File.Content, nil } + +// commitTreesTool lists a commit's files under directories (mesh-catalog modules/gitea): {files: {directory: {path: blob}}}. +const commitTreesTool = "gitea.gitea_commit_trees" + +// Files lists every file under each directory at a commit, relative to it; a directory the commit lacks wraps +// check.ErrNotFound. +func (f forgeReads) Files(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) { + owner, repo := ownerRepo(repository) + raw, err := f.ask(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": commit, "files": dirs}) + if err != nil { + if strings.Contains(err.Error(), "lacks") || notFoundStatus.MatchString(err.Error()) { + return nil, fmt.Errorf("%w: %v", check.ErrNotFound, err) + } + return nil, err + } + var a struct { + Files map[string]map[string]string `json:"files"` + } + if err := json.Unmarshal(raw, &a); err != nil { + return nil, fmt.Errorf("the answer of %s is not {files}: %v", commitTreesTool, err) + } + for _, d := range dirs { + if _, ok := a.Files[d]; !ok { + return nil, fmt.Errorf("%w: the answer of %s lists no files of %q", check.ErrNotFound, commitTreesTool, d) + } + } + return a.Files, nil +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index 79b22f2d..6675c671 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -46,6 +46,7 @@ type judgeDeps struct { // published is the indexer's packages state as it is followed (packages.go). published *publishedPackages forge check.Forge + trees check.Trees announce check.Announcer builds check.BuildStates // ends waits on the builder's builds of a candidate before the check composes (ADR 0603 §10); nil composes without waiting. @@ -230,7 +231,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, KnownPackages: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, SeatDefinitions: d.seatDefs, SeatList: d.seats, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, KnownPackages: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Dirs: d.repos, Predefined: d.predefined, Tags: d.tags, Renames: d.renames, SeatDefinitions: d.seatDefs, SeatList: d.seats, Trees: d.trees, Logf: d.logf} if d.nodes != nil && d.builds != nil && d.repos != nil { ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index e10b7f85..2687bedd 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -290,7 +290,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ch.Judge = newJudge(judgeDeps{ asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks, kept: kvStore{buckets[check.CheckerKeptBucket]}, previews: kvStore{buckets[check.PreviewsBucket]}, candidates: tryCandidates{candidates}, - repos: repos, published: published, builds: buildStates, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf, + repos: repos, published: published, builds: buildStates, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, trees: forgeReads{call: caller.Call}, logf: logger.Printf, announce: announce, compositions: compositions, tags: tags, nodes: openNodeNames(ctx, js, logger.Printf), predefined: keptPredefined{predefined}, diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index ee9887ee..8b64dee4 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1071,8 +1071,12 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's // group (issue 1921): the same judge, the same verdicts. try bool + // heads has the change touch web-contracts too, a contracts module whose head defines mesh-catalog with a verb of a + // part it includes: the judges get that definition over the list's (ADR 0636 §4, ADR 0633 §6). + heads bool }{ {name: "the seat list read is the request's Seats", seats: "read"}, + {name: "the members' heads' definitions over the list's, for the trunk's manifest validator too", seats: "read", indexer: "ok", heads: true}, {name: "the seat list not read judges nothing", seats: "unread", fails: "the change's manifests cannot be judged: the seat list is not read (the seat list is held from the bootstrap"}, {name: "a seat definition that does not parse", unparsed: "seats/web-api/seat.json: unexpected end of JSON input", @@ -1214,6 +1218,16 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { b.fetch.Changed = []string{JudgeDir + "/judge/judge.go"} } p.Modules = []string{"web"} + if row.heads { + p.Modules = append(p.Modules, "web-contracts") + b.ex.Forge = headFiles{ + repoA + " modules/web-contracts/module.json " + head1: `{"module": "web-contracts", "build": {"artifacts": [{"name": "go", "kind": "package", "language": "go", "generated-from": "."}]}}`, + repoA + " modules/web-contracts/seats/mesh-catalog/seat.json " + head1: `{"format":1,"seat":"mesh-catalog","scope":"mesh","grain":"whole","domain":"Delivery","decision":"ADR 1","includes":["lists"],` + + `"verbs":[{"name":"modules","input":{"type":"object"},"answer":{"type":"object"}}]}`, + repoA + " modules/web-contracts/parts/lists.json " + head1: `{"format":1,"part":"lists","verbs":[{"name":"seat-definitions","input":{"type":"object"},"answer":{"type":"object"}}]}`, + } + b.ex.Trees = headTrees{repoA + " modules/web-contracts/seats " + head1: {"mesh-catalog/seat.json": "1", "mesh-catalog/fixtures/a.json": "2"}} + } p.Fps = map[string]meshcatalog.CandidatesByGroupValuePullsItemFpsValue{"web": {Fp2: "src2:" + strings.Repeat("f", 64), Manifest: web}} // Issue 1921: the judge is built at the group's heads too, also a member that publishes no package yet. cs := caseOf(t, p, pullOf("novox/mesh-networking", 9, head2)) @@ -1284,7 +1298,25 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { req.Cases[0].Manifests["mail"].Digest != strings.Repeat("a", 64) { t.Fatalf("%s: the request %+v: %v", row.name, req, err) } - if row.seats == "read" && (len(req.Seats) != 2 || string(req.Seats[0]) != `{"seat":"a"}` || string(req.Seats[1]) != `{"seat":"b"}`) { + if row.heads { + var got []string + for _, raw := range req.Seats { + var d struct { + Seat string `json:"seat"` + Verbs []struct { + Name string `json:"name"` + } `json:"verbs"` + } + _ = json.Unmarshal(raw, &d) + for _, v := range d.Verbs { + d.Seat += "." + v.Name + } + got = append(got, d.Seat) + } + if want := []string{"a", "b", "mesh-catalog.seat-definitions.modules"}; !slices.Equal(got, want) { + t.Errorf("%s: the trunk's manifest validator's seats %v, want %v: the list's with the head's definition, its part resolved", row.name, got, want) + } + } else if row.seats == "read" && (len(req.Seats) != 2 || string(req.Seats[0]) != `{"seat":"a"}` || string(req.Seats[1]) != `{"seat":"b"}`) { t.Errorf("%s: the request's seats %s, want the two definitions the reader holds", row.name, req.Seats) } if row.seats == "" && len(req.Seats) != 0 { @@ -1311,6 +1343,21 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // fakeSeatList is the seat list reader as a test gives it: two definitions once read, else the reader's words. type fakeSeatList struct{ read bool } +// headTrees are the forge's listings of a directory at a commit, by " ". +type headTrees map[string]map[string]string + +func (h headTrees) Files(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) { + out := map[string]map[string]string{} + for _, d := range dirs { + files, ok := h[repository+" "+d+" "+commit] + if !ok { + return nil, ErrNotFound + } + out[d] = files + } + return out, nil +} + func (f fakeSeatList) Definitions() ([]json.RawMessage, error) { if !f.read { return nil, errors.New("the seat list is held from the bootstrap (2 seats) and nothing new is acted on") diff --git a/modules/mesh-delivery-checker/internal/check/headseats.go b/modules/mesh-delivery-checker/internal/check/headseats.go new file mode 100644 index 00000000..0b00ed1c --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/headseats.go @@ -0,0 +1,176 @@ +package check + +// The seat definitions of the members' heads (novox/hq ADR 0636 §4, ADR 0633 §6 as its insight of 2026-11-05 (2) says): the +// manifest validator inside a check is the one reader that reads them. A manifest of a group may serve or invoke what the +// group's contracts modules add to a seat (a verb, an event), which the seat list the checker follows (the trunks') does not +// hold yet; judged without them, by the trunk's manifest validator above all, the group's new indexer would be refused for +// serving its own seat's new verb, and so would every module that calls it. The definitions are read from each member's head +// as the indexer reads a revision's (the contracts module's `seats//seat.json` with the parts it includes), through the +// forge, and put over the list's in the request's Seats. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "path" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/contracts/go/clients/meshcatalogclient" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/seatdef" +) + +// Trees lists the files of a repository under directories at a commit (the forge's commit trees): {directory: {path +// relative to it: blob id}}. A directory the commit lacks is an error wrapping ErrNotFound. +type Trees interface { + Files(ctx context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) +} + +// headSeats are the definitions, resolved, that the contracts modules the case's members touch hold at their heads, by +// seat; none where the checker lists no trees. A head's definition that cannot be read or does not parse is an error: the +// manifests cannot be judged by the group's seats then. +func (st *state) headSeats(ctx context.Context) (map[string]json.RawMessage, error) { + if st.j.Trees == nil || st.j.Forge == nil { + return nil, nil + } + out := map[string]json.RawMessage{} + for _, p := range st.c.Pulls { + for _, module := range p.Modules { + defs, ok, err := st.contractsDir(ctx, p, module) + if err != nil { + return nil, err + } + if !ok { + continue + } + got, err := st.definitionsAt(ctx, p, module, defs) + if err != nil { + return nil, err + } + maps.Copy(out, got) + } + } + return out, nil +} + +// contractsDir is the directory a touched module's definitions live in at its member's head (its package artifact's +// generated-from, under the module's directory), and whether the module is a contracts module at all. +func (st *state) contractsDir(ctx context.Context, p Pull, module string) (string, bool, error) { + var m catalogue.Manifest + dir, known := "", false + if st.j.Dirs != nil { + dir, known = st.j.Dirs.ModuleDir(module) + } + if digest := p.Fps[module].Manifest; known && digest != "" && st.j.Read != nil { + var err error + if m, err = st.j.Read.Manifest(ctx, digest); err != nil { + return "", false, fmt.Errorf("the manifest of %s at the head of %s cannot be read: %w", module, pullKey(p), err) + } + } else { + var found bool + var err error + if m, dir, found, err = st.manifestAtHead(ctx, p, module); err != nil || !found { + return "", false, err + } + } + if m.Build == nil { + return "", false, nil + } + for _, a := range m.Build.Artifacts { + if a.GeneratedFrom == "" { + continue + } + base := strings.Trim(dir, "/") + if base == meshcatalogclient.RootPath { + base = "" + } + return path.Clean(path.Join(base, a.GeneratedFrom)), true, nil + } + return "", false, nil +} + +// definitionsAt reads every seat a contracts module defines at the member's head, each resolved with its parts. +func (st *state) definitionsAt(ctx context.Context, p Pull, module, defs string) (map[string]json.RawMessage, error) { + under := func(rel string) string { + if defs == "." || defs == "" { + return rel + } + return defs + "/" + rel + } + seatsDir := under("seats") + listed, err := st.j.Trees.Files(ctx, p.Repository, p.Head, []string{seatsDir}) + if errors.Is(err, ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("the seats of %s at the head of %s cannot be listed%s: %w", module, pullKey(p), unheardWords(err), err) + } + read := func(rel string) ([]byte, error) { + text, err := st.j.Forge.File(ctx, p.Repository, under(rel), p.Head) + if err != nil { + return nil, fmt.Errorf("%s of %s at the head of %s cannot be read%s: %w", under(rel), module, pullKey(p), unheardWords(err), err) + } + return []byte(text), nil + } + var seats []string + for file := range listed[seatsDir] { + if seat, ok := strings.CutSuffix(file, "/seat.json"); ok && seat != "" && !strings.Contains(seat, "/") { + seats = append(seats, seat) + } + } + out := map[string]json.RawMessage{} + parts := map[string]seatdef.Part{} + for _, seat := range slices.Sorted(slices.Values(seats)) { + raw, err := read("seats/" + seat + "/seat.json") + if err != nil { + return nil, err + } + d, err := seatdef.Parse(raw) + if err != nil { + return nil, fmt.Errorf("the seat definition of %s (module %s) at the head of %s does not parse: %w", seat, module, pullKey(p), err) + } + for _, name := range d.Includes { + if _, ok := parts[name]; ok { + continue + } + praw, err := read("parts/" + name + ".json") + if err != nil { + return nil, err + } + if parts[name], err = seatdef.ParsePart(praw); err != nil { + return nil, fmt.Errorf("the part %s of %s at the head of %s does not parse: %w", name, module, pullKey(p), err) + } + } + if d, err = seatdef.Resolve(d, parts); err != nil { + return nil, fmt.Errorf("the seat definition of %s (module %s) at the head of %s does not resolve: %w", seat, module, pullKey(p), err) + } + if out[d.Seat], err = json.Marshal(d); err != nil { + return nil, fmt.Errorf("the definition of %s cannot be encoded: %w", d.Seat, err) + } + } + return out, nil +} + +// overSeats is the list's definitions with each of the heads' put in place of the one of its seat, or added, sorted by seat. +func overSeats(list []json.RawMessage, heads map[string]json.RawMessage) []json.RawMessage { + if len(heads) == 0 { + return list + } + by := map[string]json.RawMessage{} + for _, raw := range list { + var d struct { + Seat string `json:"seat"` + } + _ = json.Unmarshal(raw, &d) + by[d.Seat] = raw + } + maps.Copy(by, heads) + out := make([]json.RawMessage, 0, len(by)) + for _, seat := range slices.Sorted(maps.Keys(by)) { + out = append(out, by[seat]) + } + return out +} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 1c0ffd1c..c4326d2e 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -143,6 +143,9 @@ type Examiner struct { // SeatList gives the seat definitions the change's manifests are judged against, as the seat list reader holds them // (manifests_judge.go, novox/hq ADR 0636 §1); nil gives the judge none. SeatList SeatList + // Trees lists a member's files at its head, so the seat definitions of the contracts modules a case touches are read + // at their heads and put over SeatList's (headseats.go, novox/hq ADR 0636 §4); nil reads none. + Trees Trees // Renames asks the seat mesh-config's verb renames for a change that brings or drops a `renamed-from` (renames.go, // design 32); nil fails such a change, saying it cannot ask. Renames Renames diff --git a/modules/mesh-delivery-checker/internal/check/manifests_judge.go b/modules/mesh-delivery-checker/internal/check/manifests_judge.go index 0c0f2974..da9c5e6d 100644 --- a/modules/mesh-delivery-checker/internal/check/manifests_judge.go +++ b/modules/mesh-delivery-checker/internal/check/manifests_judge.go @@ -139,6 +139,12 @@ func (st *state) judgeManifests(ctx context.Context) *meshdeliverychecker.Verdic } req.Seats = seats } + // The members' heads' definitions over the list's (ADR 0636 §4): the trunk's manifest validator of a new indexer as well. + heads, err := st.headSeats(ctx) + if err != nil { + return fail(goModule, "the change's manifests cannot be judged: "+err.Error()) + } + req.Seats = overSeats(req.Seats, heads) cv, trunkCase, v := st.runJudge(ctx, req, touched, commit, at, st.packagesOf(DeliveryRepository)) if v != nil { return v diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 39b6911b..93921a91 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -125,7 +125,8 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 // build module's tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4). // The forge's reads for a mesh-sdk pin (ADR 0484 clause 9), and of the node seat node-engine its one verb // compose-candidate, asked of every placed node (ADR 0593 §7), never the seat's other verbs. - if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches", + // The forge's commit trees list a contracts module's seats at a member's head (ADR 0636 §4, headseats.go). + if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "gitea.gitea_list_branches", "mesh-delivery-assigner.assigner_read_assignments", "mesh-delivery-indexer.indexer_read_candidates", "mesh-delivery-indexer.indexer_read_manifests", "mesh-delivery-indexer.indexer_read_modules", "mesh-delivery-indexer.indexer_read_packages", "mesh-delivery-indexer.indexer_read_predefined", "mesh-delivery-indexer.indexer_read_seat_definitions", diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 9fb77e6c..6626e7d8 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -73,6 +73,7 @@ "checker_verdicts" ], "invokes": [ + "gitea.gitea_commit_trees", "gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file",