From 31ff708f8895baf2d5c0c499ea1ac7ad8e4643b4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 17:52:53 +0100 Subject: [PATCH 01/13] =?UTF-8?q?Judge=20a=20successor=20indexer's=20manif?= =?UTF-8?q?est=20with=20the=20seat=20definitions=20its=20revision=20carrie?= =?UTF-8?q?s,=20not=20the=20built-in=20list=20(issue=201966,=20ADR=200633?= =?UTF-8?q?=20=C2=A71,=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The take-in's dry run reads the contracts modules' definitions of the revision's merges and puts them over the seats while it judges; cmd/mesh-judge puts the request's seats over its own. A refusal of the named indexer's manifest is said on take-in.pending and in indexer_pending. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-indexer/main.go | 2 +- .../cmd/mesh-delivery-indexer/named.go | 11 +- .../cmd/mesh-judge/main.go | 14 +- .../internal/index/candidates.go | 4 + .../internal/index/pending.go | 55 ++++-- .../internal/index/pending_test.go | 76 +++++++- .../internal/index/seatsof.go | 154 +++++++++++++++ .../internal/index/verdicts.go | 4 + modules/mesh-delivery-indexer/judge/judge.go | 19 +- .../mesh-delivery-indexer/judge/judge_test.go | 38 ++++ .../mesh-delivery-indexer/judge/seatfiles.go | 179 ++++++++++++++++++ .../mesh-delivery-indexer/takein/takein.go | 6 + 12 files changed, 545 insertions(+), 17 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/seatsof.go create mode 100644 modules/mesh-delivery-indexer/judge/seatfiles.go diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 7f21ad2a..b925b5b2 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -347,7 +347,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), - ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), + ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call), OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}, Message: messageVia(calls.Call), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 8d60e4b9..8822cb8e 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -120,6 +120,12 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st return goFileVia(call, "go.mod") } +// readFileVia reads one file of a repository at a commit by its path (gitea_get_file); a file the commit does not hold is +// none. The seat definitions of a revision are read with it (seatsof.go). +func readFileVia(call toolCaller) func(ctx context.Context, repository, sha, path string) ([]byte, error) { + return goFileVia(call, "") +} + // goFileVia reads one Go file of a module directory at a commit through the forge's get-file tool; a file the commit does // not hold is none. func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) { @@ -129,7 +135,10 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor return nil, fmt.Errorf("%q is not /", repository) } p := name - if dir != "" && dir != delivery.RootPath { + switch { + case name == "": // readFileVia: dir is the file's whole path + p = dir + case dir != "" && dir != delivery.RootPath: p = dir + "/" + name } raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha}) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go b/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go index 8cd30382..5f7be950 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-judge/main.go @@ -46,7 +46,19 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fmt.Fprintln(stderr, "mesh-judge:", err) return 1 } - v, err := judge.Request(r) + // The request's seat definitions (ADR 0636 §4): the composer's request type of this build does not decode them. + var seats struct { + Seats []json.RawMessage `json:"seats"` + } + if err := json.Unmarshal(raw, &seats); err != nil { + fmt.Fprintln(stderr, "mesh-judge:", err) + return 1 + } + files := make([][]byte, len(seats.Seats)) + for i, s := range seats.Seats { + files[i] = s + } + v, err := judge.RequestWith(r, files) if err != nil { fmt.Fprintln(stderr, "mesh-judge:", err) return 1 diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 381ce794..72acca96 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -82,6 +82,9 @@ type Indexer struct { // repository it replaces by a directory (fingerprint.ReplacedSiblings), which the version line takes by their // content; nil reads none. GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error) + // ReadFile reads one file of a repository at a commit (gitea.gitea_get_file); a file the commit does not hold is nil. + // With Files it reads the seat definitions a revision carries (seatsof.go); nil reads none. + ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error) // Now is the clock of a taken value; nil is time.Now. Now func() time.Time Builds *Builds @@ -119,6 +122,7 @@ type Indexer struct { atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished) // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. OwnBuild func(ctx context.Context) (string, error) + underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) judgeMu sync.Mutex // one judge pass at a time (verdicts.go) ownMu sync.Mutex // guards ownFP ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index be323c74..39189ff2 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -15,6 +15,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" + "mesh-delivery-indexer/judge" + "git.novox.be/novox/mesh-delivery/sdk/graph" ) @@ -70,6 +72,7 @@ type Pending struct { Ref *PendingRef `json:"ref,omitempty"` Forge json.RawMessage `json:"forge,omitempty"` Lost string `json:"lost,omitempty"` + Refused string `json:"refused,omitempty"` Group string `json:"group,omitempty"` Ended bool `json:"ended,omitempty"` Since time.Time `json:"since"` @@ -688,7 +691,7 @@ func (ix *Indexer) dry() *dryRun { d := &dryRun{} x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains, - ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil} + ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, ReadFile: ix.ReadFile, underSeats: true} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) @@ -721,16 +724,29 @@ func (ix *Indexer) dry() *dryRun { // is left out, as its take-in will drop it. func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) { d := ix.dry() - for _, e := range ready { - if e.Covered { - continue - } - if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil { - if errors.Is(err, ErrUnreadable) { + // The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer + // it names is not refused for a verb or an event the built-in list lacks. + seats, err := ix.revisionSeats(ctx, ready) + if err != nil { + return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err) + } + var applyErr error + judge.WithSeats(seats, func() { + for _, e := range ready { + if e.Covered { continue } - return "", d, fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err) + if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil { + if errors.Is(err, ErrUnreadable) { + continue + } + applyErr = fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err) + return + } } + }) + if applyErr != nil { + return "", d, applyErr } key, _ := delivery.ModulesKey(delivery.IndexerModule) value, _, found, err := d.ix.Modules.Get(ctx, key) @@ -915,24 +931,33 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own return fmt.Errorf("the indexer's key of the module graph: %w", err) } } + // Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the + // revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost). + refused := "" if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found { if m, err := delivery.DecodeModule(value); err == nil && m.Manifest != "" { if vk, err := delivery.VerdictsKey(m.Manifest); err == nil { if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) { return fmt.Errorf("the verdict on the indexer's manifest: %w", err) } + if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok { + if v, err := delivery.DecodeManifestVerdict(vv); err == nil && v.Refused() { + refused = strings.Join(v.Problems, "; ") + } + } } } } t := revisionWait(takein.TakeInPending, ready, names) + t.Refused = refused var ended, unmarked []PendingEntry said, changed := false, false for _, e := range ready { switch { case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending: said = true - changed = changed || e.Value.WaitsFor != names - if e.Value.WaitsFor != names { + changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused + if e.Value.WaitsFor != names || e.Value.Refused != refused { unmarked = append(unmarked, e) } case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending: @@ -947,13 +972,16 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own } if !said || changed { ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own) + if refused != "" { + ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused) + } if !ix.sayTakeIn(ctx, t) { - ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own }) + ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused }) return nil } unmarked = ready } - ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own }) + ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused }) return nil } @@ -1015,6 +1043,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait { if e.Value.Lost != "" { pw.Lost = append(pw.Lost, e.Key) } + if e.Value.Refused != "" { + pw.Refused = e.Value.Refused + } if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) { pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt } diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 7e6456fc..3d2ea4d9 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -12,6 +12,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" "git.novox.be/novox/mesh-sdk/go/buscompose" + "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" ) @@ -188,6 +189,63 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) t.Errorf("said %v, want %v", b.events(), want) } }) + t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) { + // A successor's manifest invokes a verb of the node-engine's seat that the built-in list lacks and the revision's + // definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending; + // accepted with them, and the revision waits for the build it names. + const verb, contracts = "manifest-reads", "modules/seat-contracts" + for _, row := range []struct { + name string + defined bool + refused bool + }{{"definitions the revision does not carry", false, true}, {"definitions the revision carries", true, false}} { + b := newTakeInBed(t) + f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2") + f.Manifests = map[string]string{} + for dir, raw := range b.base.Manifests { + f.Manifests[dir] = raw + } + dir := "modules/" + delivery.IndexerModule + f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+delivery.IndexerModule+`"`, + `"module": "`+delivery.IndexerModule+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1) + if row.defined { + f.Trees[contracts] = "tree-seat-contracts" + f.Locks = map[string]map[string]string{} + for d, l := range b.base.Locks { + f.Locks[d] = l + } + f.Locks[contracts] = map[string]string{} + f.Manifests[contracts] = `{"module":"seat-contracts","version":"1","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}` + b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) { + if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) { + t.Errorf("listed %v of %s at %s", dirs, repository, commit) + } + return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil + } + b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { + switch p { + case contracts + "/seats/node-engine/seat.json": + return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil + case contracts + "/parts/reads.json": + return []byte(`{"format":1,"part":"reads","verbs":[{"name":"` + verb + `"}]}`), nil + } + return nil, nil + } + } + b.arrive(t, 2, f) + waits := PendingWaits(readPending(t, b)) + refused := "" + if len(b.said) > 0 { + refused = b.said[0].Refused + } + if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) { + t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused) + } + if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) { + t.Errorf("%s: the seats are not put back after the dry run", row.name) + } + } + }) t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) { b := newTakeInBed(t) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) @@ -520,7 +578,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) named := map[string]bool{} for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", - "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn"} { + "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) @@ -540,3 +598,19 @@ func mustReadPending(t *testing.T, b *takeInBed) []PendingEntry { } return entries } + +// judgeSeatsHave says whether the catalogue's seat list holds the verb on the seat now. +func judgeSeatsHave(seat, verb string) bool { + s, _ := catalogue.SeatNamed(seat) + return slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb }) +} + +// readPending is the bed's pending state as indexer_pending reads it. +func readPending(t *testing.T, b *takeInBed) []PendingEntry { + t.Helper() + entries, err := ReadPending(context.Background(), b.pending, nil) + if err != nil { + t.Fatal(err) + } + return entries +} diff --git a/modules/mesh-delivery-indexer/internal/index/seatsof.go b/modules/mesh-delivery-indexer/internal/index/seatsof.go new file mode 100644 index 00000000..526e6e3b --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/seatsof.go @@ -0,0 +1,154 @@ +package index + +// The seat definitions a revision carries (novox/hq ADR 0633 §1, ADR 0635 §3, ADR 0636 §4): when a take-in works out which +// indexer a revision names, the manifests it writes are judged against the seats of that revision, never against the +// list this program was built with, since the indexer the revision names may serve or invoke what the revision's +// definitions add (a verb of a seat, an event). They are read from the merges' trees: each contracts module a merge's +// manifests name (a package artifact with `generated-from`) holds `seats//seat.json` and the `parts/` they +// include, read through the forge's listing and get-file tools. A delivery group's merges are one revision, so a +// group's members' definitions are read together. + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "path" + "slices" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + + "mesh-delivery-indexer/judge" +) + +// definitionsDir is the directory of a contracts module's definitions that the module.json at dir names, from the first +// package artifact naming `generated-from`: ok is false for a module that is no contracts module. A manifest that does +// not decode is none here: the judge says why it is refused. +func definitionsDir(dir, manifest string) (string, bool) { + var m struct { + Build struct { + Artifacts []struct { + GeneratedFrom string `json:"generated-from"` + } `json:"artifacts"` + } `json:"build"` + } + if json.Unmarshal([]byte(manifest), &m) != nil { + return "", false + } + for _, a := range m.Build.Artifacts { + if a.GeneratedFrom == "" { + continue + } + base := dir + if base == "." || base == "/" { + base = "" + } + return path.Clean(path.Join(base, a.GeneratedFrom)), true + } + return "", false +} + +// revisionSeats are the seats of the definitions the ready merges carry, read at their merge commits, the newest merge +// of a repository governing; none where the indexer has no forge reader. A read the forge does not answer is an error +// (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it +// would say it unparsed. +func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) { + if ix.ReadFile == nil || ix.Files == nil { + return nil, nil + } + var defs []judge.Definition + for _, e := range ready { + if e.Covered || e.Forge.Merge == "" { + continue + } + for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) { + defsDir, ok := definitionsDir(dir, e.Forge.Manifests[dir]) + if !ok { + continue + } + got, err := ix.readDefinitions(ctx, e.Forge.Repository, e.Forge.Merge, defsDir) + if err != nil { + return nil, err + } + defs = append(defs, got...) + } + } + seats, err := judge.Seats(defs) + if err != nil { + return nil, err + } + return seats, nil +} + +// readDefinitions reads the definitions under a contracts module's definitions directory at a commit, resolved with their +// parts. +func (ix *Indexer) readDefinitions(ctx context.Context, repository, commit, defsDir string) ([]judge.Definition, error) { + file := func(p string) ([]byte, error) { + if defsDir != "." && defsDir != "" { + p = defsDir + "/" + p + } + raw, err := ix.ReadFile(ctx, repository, commit, p) + if err != nil { + return nil, fmt.Errorf("reading %s of %s at %.12s: %w", p, repository, commit, err) + } + return raw, nil + } + dir := path.Join(defsDir, "seats") + if defsDir == "." || defsDir == "" { + dir = "seats" + } + listed, err := ix.Files(ctx, repository, commit, []string{dir}) + if err != nil { + return nil, fmt.Errorf("listing %s of %s at %.12s: %w", dir, repository, commit, err) + } + parts := map[string]judge.Part{} + var out []judge.Definition + for _, p := range slices.Sorted(maps.Keys(listed[dir])) { + seat, ok := strings.CutSuffix(p, "/seat.json") + if !ok || seat == "" || strings.Contains(seat, "/") { + continue + } + raw, err := file("seats/" + seat + "/seat.json") + if err != nil { + return nil, err + } + d, err := judge.ParseDefinition(raw) + if err == nil && d.Seat != seat { + err = fmt.Errorf("it names the seat %q, not %q", d.Seat, seat) + } + for _, name := range d.Includes { + if err != nil { + break + } + if _, have := parts[name]; have { + continue + } + praw, perr := file("parts/" + name + ".json") + if perr != nil { + return nil, perr + } + var part judge.Part + if praw == nil { + perr = errors.New("the commit holds no such part") + } else { + part, perr = judge.ParsePart(praw) + } + if perr != nil { + err = fmt.Errorf("its part %s: %w", name, perr) + break + } + parts[name] = part + } + if err == nil { + d, err = d.Resolve(parts) + } + if err != nil { + ix.logf("take-in: the seat definition seats/%s of %s at %.12s does not parse, so it is left out: %v", seat, repository, commit, err) + continue + } + out = append(out, d) + } + return out, nil +} diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 1263a8e1..5eabc0a2 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -61,6 +61,10 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, over ma // modules another is rewriting. ix.judgeMu.Lock() defer ix.judgeMu.Unlock() + if !ix.underSeats { + // The seat list stands still for this pass: a take-in's dry run may put its revision's seats over it (seatsof.go). + defer judge.HoldSeats()() + } var out Judged if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil { return out, nil diff --git a/modules/mesh-delivery-indexer/judge/judge.go b/modules/mesh-delivery-indexer/judge/judge.go index b41c6ba0..c144cf02 100644 --- a/modules/mesh-delivery-indexer/judge/judge.go +++ b/modules/mesh-delivery-indexer/judge/judge.go @@ -204,7 +204,24 @@ const Unnamed = "(between manifests)" // each case's manifests as one catalogue (Shelf), every manifest accepted or refused with its problems, one per line; the // problems no module's removal ends under Between. The verdict's SDK names this judge (Identity). A manifest left as a // digest, or a case named twice, is a request it cannot run. -func Request(r composer.Request) (composer.Verdict, error) { +func Request(r composer.Request) (composer.Verdict, error) { return RequestWith(r, nil) } + +// RequestWith is Request judging against the seat definitions the request gives (ADR 0636 §4; the request's `seats`, +// which the mesh-sdk this judge is built with does not decode: cmd/mesh-judge reads them from the request's bytes), put +// over the seats this program was built with, never the built-in list alone: inside a delivery group's check they are +// the members' heads' definitions, so a manifest that serves or invokes what the group's definitions add is judged by +// them. None is Request. +func RequestWith(r composer.Request, seats [][]byte) (composer.Verdict, error) { + over, err := SeatsOfFiles(seats) + if err != nil { + return composer.Verdict{}, fmt.Errorf("a seat definition of the request: %w", err) + } + var v composer.Verdict + WithSeats(over, func() { v, err = request(r) }) + return v, err +} + +func request(r composer.Request) (composer.Verdict, error) { v := composer.Verdict{V: composer.Version, SDK: Identity()} seen := map[string]bool{} counts := trunkOldSections(r) diff --git a/modules/mesh-delivery-indexer/judge/judge_test.go b/modules/mesh-delivery-indexer/judge/judge_test.go index 27caf578..b57cfe4d 100644 --- a/modules/mesh-delivery-indexer/judge/judge_test.go +++ b/modules/mesh-delivery-indexer/judge/judge_test.go @@ -96,6 +96,7 @@ func TestTheJudge(t *testing.T) { "a": {Raw: json.RawMessage(`{"module":"a","version":"1"}`)}, "b": {Raw: json.RawMessage(`{"module":"b","version":"1","requires":["amqp"]}`)}, }}}} + was := catalogue.Seats() v, err := Request(r) if err != nil || v.V != composer.Version || len(v.Cases) != 1 || v.SDK != Identity() || !v.Cases[0].Manifests["a"].Accepted || v.Cases[0].Manifests["b"].Accepted || !strings.Contains(v.Cases[0].Manifests["b"].Refused, `requires "amqp"`) { @@ -122,6 +123,43 @@ func TestTheJudge(t *testing.T) { !got["newer"].Accepted || !got["user"].Accepted || !got["mesh-grants"].Accepted { t.Errorf("the change's old sections: %+v", got) } + // ADR 0633 §6, ADR 0636 §4: a successor's manifest is judged by the seat definitions its request carries, put over the + // built-in seats: a verb the built-in list lacks is refused without them and accepted with them (the request's bytes, + // as cmd/mesh-judge reads them), and the seats stand as before once the request is judged. + succ := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change", Manifests: map[string]composer.Manifest{ + "successor": {Raw: json.RawMessage(`{"module":"successor","version":"1","invokes":["seat:node-engine.manifest-reads"]}`)}}}}} + defs := [][]byte{[]byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","includes":["reads"],"verbs":[{"name":"manifest-reads"}]}`)} + for _, row := range []struct { + name string + seats [][]byte + want string // a word of the refusal; empty: accepted + }{ + {"without definitions", nil, `serves no verb "manifest-reads"`}, + {"a definition with the verb", [][]byte{[]byte(strings.Replace(string(defs[0]), `"includes":["reads"],`, "", 1))}, ""}, + {"a definition naming a part it includes", defs, "includes"}, + {"a definition of no seat", [][]byte{[]byte(`{"format":1}`)}, "names its seat"}, + } { + sv, err := RequestWith(succ, row.seats) + switch { + case row.want != "" && err != nil: + if !strings.Contains(err.Error(), row.want) { + t.Errorf("%s: %v, want %q", row.name, err, row.want) + } + case err != nil: + t.Errorf("%s: %v", row.name, err) + case (row.want == "") != sv.Cases[0].Manifests["successor"].Accepted || + !strings.Contains(sv.Cases[0].Manifests["successor"].Refused, row.want): + t.Errorf("%s: %+v, want %q", row.name, sv.Cases[0].Manifests, row.want) + } + if _, ok := catalogue.SeatNamed("node-engine"); !ok || len(catalogue.Seats()) != len(was) { + t.Fatalf("%s: the seats are not put back", row.name) + } + } + parts := map[string]Part{"reads": {Verbs: []DefVerb{{Name: "manifest-reads"}}}} + d, _ := ParseDefinition(defs[0]) + if d, err := d.Resolve(parts); err != nil || len(d.Verbs) != 2 || d.Verbs[0].Name != "manifest-reads" { + t.Errorf("a resolved definition: %+v %v", d, err) + } r.Cases[0].Manifests["c"] = composer.Manifest{Digest: strings.Repeat("a", 64)} if _, err := Request(r); err == nil { t.Fatal("a manifest left as a digest was judged") diff --git a/modules/mesh-delivery-indexer/judge/seatfiles.go b/modules/mesh-delivery-indexer/judge/seatfiles.go new file mode 100644 index 00000000..384baace --- /dev/null +++ b/modules/mesh-delivery-indexer/judge/seatfiles.go @@ -0,0 +1,179 @@ +package judge + +// The seat definitions a revision carries (novox/hq ADR 0633 §1, §6; ADR 0635, ADR 0636 §4): a manifest is judged +// against the seats of the revision it belongs to, never against the list this program was built with. A definition is +// the contracts module's `seats//seat.json` (design 26 *A seat's definition*), with the parts it includes from +// `parts/.json`. This reader takes of it only what a manifest is judged by (the seat's scope, what it delivers, +// its verbs and events) and reads the rest leniently: the format's one strict reader (mesh-sdk seatdef) is not in the +// mesh-sdk this judge is built with, and the indexer of the revision keeps the definitions whole. + +import ( + "encoding/json" + "fmt" + "slices" + "sync" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Definition is the part of one seat's definition a manifest is judged by. +type Definition struct { + Format int `json:"format"` + Seat string `json:"seat"` + Scope string `json:"scope"` + Decision string `json:"decision"` + HolderDeclares bool `json:"holder-declares"` + Delivers string `json:"delivers"` + Replicated bool `json:"replicated"` + PulledByKind bool `json:"pulled-by-kind"` + Yields bool `json:"yields"` + HeldByTheEngine bool `json:"held-by-the-engine"` + Includes []string `json:"includes"` + Verbs []DefVerb `json:"verbs"` + Accepts []DefName `json:"accepts"` + Events []DefName `json:"events"` +} + +// DefVerb is a verb of a definition. +type DefVerb struct { + Name string `json:"name"` + Description string `json:"description"` + Optional bool `json:"optional"` + Replaces []string `json:"replaces"` +} + +// DefName is an accepted message or an event of a definition. +type DefName struct { + Name string `json:"name"` +} + +// Part is a part a definition includes: verbs and events written once for the seats that share them. +type Part struct { + Part string `json:"part"` + Verbs []DefVerb `json:"verbs"` + Events []DefName `json:"events"` +} + +// ParseDefinition reads one definition; a file that is not one names a seat and a scope. +func ParseDefinition(raw []byte) (Definition, error) { + var d Definition + if err := json.Unmarshal(raw, &d); err != nil { + return Definition{}, fmt.Errorf("a seat definition is not JSON: %w", err) + } + if d.Seat == "" || (d.Scope != catalogue.ScopeMesh && d.Scope != catalogue.ScopeNode) { + return Definition{}, fmt.Errorf("a seat definition names its seat and its scope (mesh or node), not %q and %q", d.Seat, d.Scope) + } + return d, nil +} + +// ParsePart reads one part. +func ParsePart(raw []byte) (Part, error) { + var p Part + if err := json.Unmarshal(raw, &p); err != nil { + return Part{}, fmt.Errorf("a part of a seat definition is not JSON: %w", err) + } + return p, nil +} + +// Resolve is the definition with the verbs and events of the parts it includes put first, as the indexer keeps it. +func (d Definition) Resolve(parts map[string]Part) (Definition, error) { + out := d + out.Includes, out.Verbs, out.Events = nil, nil, nil + for _, name := range d.Includes { + p, ok := parts[name] + if !ok { + return Definition{}, fmt.Errorf("the seat %s includes the part %q, which is not given", d.Seat, name) + } + out.Verbs = append(out.Verbs, p.Verbs...) + out.Events = append(out.Events, p.Events...) + } + out.Verbs = append(out.Verbs, d.Verbs...) + out.Events = append(out.Events, d.Events...) + return out, nil +} + +// Seats are the seats of resolved definitions, in the catalogue's types; a seat its holder declares itself is left out. +// A definition still naming parts it includes is refused: resolve it first. +func Seats(defs []Definition) ([]catalogue.Seat, error) { + var out []catalogue.Seat + for _, d := range defs { + if d.HolderDeclares { + continue + } + if len(d.Includes) > 0 { + return nil, fmt.Errorf("the seat %s names parts it includes: resolve it first", d.Seat) + } + s := catalogue.Seat{Name: d.Seat, Scope: d.Scope, Delivers: d.Delivers, Replicated: d.Replicated, + PulledByKind: d.PulledByKind, Yields: d.Yields, HeldByTheEngine: d.HeldByTheEngine, Decision: d.Decision} + for _, a := range d.Accepts { + s.Accepts = append(s.Accepts, a.Name) + } + for _, e := range d.Events { + s.Emits = append(s.Emits, e.Name) + } + for _, v := range d.Verbs { + s.Serves = append(s.Serves, catalogue.Verb{Name: v.Name, Description: v.Description, Optional: v.Optional, Replaces: v.Replaces}) + } + out = append(out, s) + } + return out, nil +} + +// SeatsOfFiles are the seats of resolved definitions given as files, as a request carries them. +func SeatsOfFiles(raws [][]byte) ([]catalogue.Seat, error) { + defs := make([]Definition, 0, len(raws)) + for _, raw := range raws { + d, err := ParseDefinition(raw) + if err != nil { + return nil, err + } + defs = append(defs, d) + } + return Seats(defs) +} + +// merged is base with each seat of over put in place of the seat of its name, or added: a definition is a seat's whole +// contract. A verb the base seat holds keeps its input and output (a definition read here carries none). +func merged(base, over []catalogue.Seat) []catalogue.Seat { + out := slices.Clone(base) + for _, s := range over { + at := slices.IndexFunc(out, func(b catalogue.Seat) bool { return b.Name == s.Name }) + if at < 0 { + out = append(out, s) + continue + } + for i, v := range s.Serves { + if j := slices.IndexFunc(out[at].Serves, func(b catalogue.Verb) bool { return b.Name == v.Name }); j >= 0 { + s.Serves[i].Input, s.Serves[i].Output = out[at].Serves[j].Input, out[at].Serves[j].Output + } + } + out[at] = s + } + return out +} + +// seatsMu keeps the seat list the catalogue holds (a list of the process) still while one pass judges with another: +// WithSeats takes it whole; a judge pass of the running indexer that uses the list as it stands takes it shared. +var seatsMu sync.RWMutex + +// HoldSeats keeps the catalogue's seat list as it stands until the returned function is called, so a pass that judges +// with it never sees another's (WithSeats). The one pass inside WithSeats does not take it. +func HoldSeats() func() { + seatsMu.RLock() + return seatsMu.RUnlock +} + +// WithSeats runs fn with seats put over the catalogue's seat list (merged), and puts the list back after. Nothing +// else judges meanwhile (HoldSeats). fn must not call HoldSeats. No seats runs fn as it is, without the exclusion. +func WithSeats(seats []catalogue.Seat, fn func()) { + if len(seats) == 0 { + fn() + return + } + seatsMu.Lock() + defer seatsMu.Unlock() + was := catalogue.Seats() + catalogue.UseSeats(merged(was, seats)) + defer catalogue.UseSeats(was) + fn() +} diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index d5ce9866..3e76e4ba 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -136,6 +136,10 @@ type TakeIn struct { // Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the // revision waits until indexer_drop_pending drops them (issue 1585). Lost []string `json:"lost,omitempty"` + // Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with + // the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for + // an indexer whose manifest is accepted, and says why here, as it says what it lost. + Refused string `json:"refused,omitempty"` } func (t TakeIn) check() error { @@ -235,6 +239,8 @@ type PendingWait struct { SaidAt time.Time `json:"said-at,omitzero"` // Lost are the keys of its merges whose message the forge bridge's stream no longer holds (TakeIn.Lost). Lost []string `json:"lost,omitempty"` + // Refused is why the manifest of the indexer it names was refused (TakeIn.Refused). + Refused string `json:"refused,omitempty"` } // PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read. From 55936c1cd7d37bffe0eb77b6c3ae493b3368fd75 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:05:24 +0100 Subject: [PATCH 02/13] Try a delivery group: members at exact commits, judged by the code of a group's check, every failure named (issue 1921) A try names several repositories, each the exact commit asked. A whole try (not narrowed to a level or tests) runs the group check's own path: the manifest validator, the module graph and the composition on every placed node, reading the indexer's candidate of the try's group. A try with no such candidate is an error that says so, never a pass. A try's candidate is no check: the checker leaves it alone and owes the forge nothing. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-checker-tools/main.go | 52 +++-- .../mesh-delivery-checker-tools/main_test.go | 28 +++ .../mesh-delivery-checker/integration_test.go | 27 ++- .../cmd/mesh-delivery-checker/judge.go | 7 +- .../cmd/mesh-delivery-checker/main.go | 5 +- .../mesh-delivery-checker/try_candidates.go | 72 +++++++ .../internal/check/examiner_test.go | 10 + .../internal/check/graph_cycle.go | 2 +- .../internal/check/judge.go | 68 ++++++- .../internal/check/manifests_judge.go | 2 +- .../internal/check/read.go | 2 + .../internal/check/try.go | 189 +++++++++++++++--- .../internal/check/try_test.go | 142 +++++++++++-- 13 files changed, 528 insertions(+), 78 deletions(-) create mode 100644 modules/mesh-delivery-checker/cmd/mesh-delivery-checker/try_candidates.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index fd39fe86..d570a958 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -103,15 +103,25 @@ func TryTools(tr Trying) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } return []stdio.Tool{ {Name: check.TryTool, - Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " + - "its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " + - "where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " + - "the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " + - "bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).", - Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{ - "repository": map[string]any{"type": "string", "description": "/"}, - "head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"}, - "trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}, + Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " + + "name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " + + "or one repository and its head for a try of one member. The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " + + "each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " + + "the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " + + "the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " + + "check of the same heads on the same trunk runs only what the try did not pass. A try narrowed to a level or to some tests judges only those, no manifest and no composition, and keeps nothing of tests. " + + "Answers the try's id at once; " + check.TryReadTool + " reads where it stands, " + check.TryEndTool + " ends it by hand. A try of the same commits that waits or runs is the answer to a second ask " + + "(whatever the order of the members); a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide. A repository the checker checks no level of (one whose trunk the gate does not name) " + + "is refused. A try whose ask no build node took within the step bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).", + Input: map[string]any{"type": "object", "properties": map[string]any{ + "members": map[string]any{"type": "array", "description": "the group: one repository at most once, each at the exact commit asked and the trunk commit it is merged onto", + "items": map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{ + "repository": map[string]any{"type": "string", "description": "/"}, + "head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"}, + "trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}}}}, + "repository": map[string]any{"type": "string", "description": "/, for a try of one member (instead of members)"}, + "head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits (with repository)"}, + "trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent (with repository)"}, "tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"}, "level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}}, Run: func(args map[string]any) (any, error) { @@ -126,16 +136,32 @@ func TryTools(tr Trying) []stdio.Tool { if tr.Now != nil { now = tr.Now } + var members []check.TryAskMember + if raw, ok := args["members"]; ok { + list, isList := raw.([]any) + if !isList { + return nil, fmt.Errorf("members is a list of {repository, head, trunk}") + } + for _, item := range list { + m, isMap := item.(map[string]any) + if !isMap { + return nil, fmt.Errorf("a member is {repository, head, trunk}") + } + members = append(members, check.TryAskMember{Repository: str(m, "repository"), Head: str(m, "head"), Trunk: str(m, "trunk")}) + } + } id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"), - Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now()) + Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level"), Members: members}, now()) if err != nil { return nil, err } return map[string]any{"id": id}, nil }}, {Name: check.TryReadTool, - Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " + - "the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " + + Description: "Where a try (of a group or of one head) stands, read whole, as stored: queued, running, or its end (pass, fail, or error when it could not run, or " + + "when its manifests and composition could not be judged), with what was said of it, every member's commit and the trunk commit it was merged onto, the time it took, " + + "every failure one line each (members, levels, tests, manifests, the composition per node), the levels' tests run and kept, the composition on every placed node, " + + "the notes of the check (manifests judged, module graph, kept verdicts); while it runs, each ask it put as soon as it is put, " + "with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.", Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{ "id": map[string]any{"type": "string", "description": "the try's id, try-"}}}, @@ -146,7 +172,7 @@ func TryTools(tr Trying) []stdio.Tool { return check.ReadTry(tr.Tries, str(args, "id")) }}, {Name: check.TryEndTool, - Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " + + Description: "End a try of a group (or of one head) that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " + "the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " + "was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " + "takes it within the step bound. A try that ended already is refused.", diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 779922bc..8f460c3e 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -106,6 +106,34 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil { t.Error("the verb tried a repository nothing checks") } + // A group try (issue 1921): the members, each an exact commit; members and one repository together are refused; the + // description says what it does in the words a search for "try a group" finds. + later := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(2, 5) }, + Trunks: func() (map[string]string, error) { + return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil + }}) + g, err := later[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}}) + if err != nil || g.(map[string]any)["id"] == id { + t.Errorf("a group try = %v, %v, want a try of its own", g, err) + } + if _, err := tools[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}, + "repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40)}); err == nil { + t.Error("members and a repository were both taken") + } + if _, err := tools[0].Run(map[string]any{"members": "novox/mesh-delivery"}); err == nil { + t.Error("members that are no list were taken") + } + for _, tl := range tools { + for _, words := range []string{"group"} { + if !strings.Contains(strings.ToLower(tl.Description), words) { + t.Errorf("%s says nothing of a %s: %q", tl.Name, words, tl.Description) + } + } + } + if d := strings.ToLower(tools[0].Description); !strings.Contains(d, "try a group") || !strings.Contains(d, "exact commit") || !strings.Contains(d, "every failure") || + !strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") { + t.Errorf("the try's description does not say what a group try does: %q", d) + } // Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew. ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"}) if err != nil || ended.(map[string]any)["state"] != "error" { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index 823deb6c..9eeb0c8f 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -862,9 +862,11 @@ func (s triesStore) Create(key string, value []byte) error { // TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker, // judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no -// verdict and no status owed (ADR 0501 §5). +// verdict and no status owed (ADR 0501 §5). A whole try waits for the indexer's candidate of its group, which it reads on the +// key's watch, and writes the merge previews the indexer makes it from (issue 1921). func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) { b := startBed(t) + asked := b.engine(t, "n1") // a whole try is composed on every placed node, as a group's check is b.start(t, nil) store := triesStore{b.buckets[check.TriesBucket], b.ctx} id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now()) @@ -872,19 +874,36 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) { t.Fatal(err) } var last check.Try + // The indexer's candidate of the try's group (here written by the test), at the heads of its members. + raw0, _ := store.Get(id) + first, _ := check.DecodeTry(raw0) + cand := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: check.TryCase(id, first, nil).HeadsHash, + Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: repo, Head: head1}, Modules: []string{}}}} + cand.FPSet = delivery.FingerprintSet(cand.Pulls) + data, err := delivery.EncodeCandidate(cand) + if err != nil { + t.Fatal(err) + } + ckey, _ := delivery.CandidatesKey(id) + if _, err := b.buckets[delivery.CandidatesBucket].Create(b.ctx, ckey, data); err != nil { + t.Fatal(err) + } for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { raw, _ := store.Get(id) if last, err = check.DecodeTry(raw); err == nil && last.Over() { break } } - if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" { - t.Fatalf("try = %+v, want passed on the trunk", last) + if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" || last.Composition == nil { + t.Fatalf("try = %+v, want passed on the trunk with its composition made", last) + } + if got := asked(); len(got) != 1 || !strings.HasPrefix(got[0].Candidate, id+".") { + t.Errorf("the placed node's engine was asked %+v, want the try's candidate %s.", got, id) } if b.fake.count("mesh-build-go.build_go_fetch") == 0 { t.Errorf("the try fetched nothing") } - for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} { + for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket} { if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 { t.Errorf("a try wrote %s: %v", name, keys) } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index d826ebee..72cd4811 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -29,7 +29,10 @@ type judgeDeps struct { trunks *check.Trunks kept check.KV previews check.KV - repos *repoModules + // candidates answers the indexer's candidate of a try's group (try_candidates.go); nil leaves a whole try with no + // manifest or composition judged, said. + candidates check.TryCandidates + repos *repoModules // published is the indexer's packages state as it is followed (packages.go). published *publishedPackages forge check.Forge @@ -157,7 +160,7 @@ func newJudge(d judgeDeps) check.Judge { nodes = d.nodes } ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, - Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf} + Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf} if d.nodes != nil && d.builds != nil && d.repos != nil { ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf} } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 1cc2b977..fc2222e1 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -216,7 +216,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6) ch.Judge = newJudge(judgeDeps{ asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks, - kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, + kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]}, repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf, announce: announce, compositions: compositions, nodes: openNodeNames(ctx, js, logger.Printf), @@ -333,6 +333,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro }) }() each := func(e jetstream.KeyValueEntry) { + if check.IsTryGroup(e.Key()) { + return // a group try's candidate: its try judges it, and the forge is owed nothing (issue 1921) + } if opOf(e) { ch.Candidate(ctx, e.Key(), e.Value()) } else { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/try_candidates.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/try_candidates.go new file mode 100644 index 00000000..2eeb9a08 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/try_candidates.go @@ -0,0 +1,72 @@ +package main + +import ( + "context" + "fmt" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// candidateWaitsOnPreview is the None2 of a module the indexer wrote from a pull message alone, before it had the merge +// preview of the head (the indexer's NoneNoPreview): its fingerprint comes with the preview, so a candidate that still says +// it is not yet the candidate of the heads previewed. It moves to mesh-sdk's delivery package with the next change there. +const candidateWaitsOnPreview = "no merge preview is computed" + +// tryCandidates answers the indexer's candidate of a try's group (novox/hq ADR 0593 §7, issue 1921): the candidate the +// engines compose and the manifest validator judges, which the indexer makes of the try's previews. It waits for it on the +// key's own watch, with no timer between reads, and gives up at the bound. +type tryCandidates struct{ kv jetstream.KeyValue } + +func (t tryCandidates) TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) { + key, err := delivery.CandidatesKey(group) + if err != nil { + return delivery.Candidate{}, err + } + if bound <= 0 { + bound = time.Minute + } + waiting, cancel := context.WithTimeout(ctx, bound) + defer cancel() + w, err := t.kv.Watch(waiting, key) // the value now, then each change + if err != nil { + return delivery.Candidate{}, fmt.Errorf("the indexer's candidate %s cannot be watched: %w", key, err) + } + defer func() { _ = w.Stop() }() + last := "the indexer holds none" + for { + select { + case <-waiting.Done(): + return delivery.Candidate{}, fmt.Errorf("the indexer wrote no candidate of %s at heads %s within %s (%s)", group, heads, bound, last) + case e := <-w.Updates(): + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + c, err := delivery.DecodeCandidate(e.Value()) + switch { + case err != nil: + last = "what it holds is not a candidate: " + err.Error() + case c.HeadsHash != heads: + last = "it holds heads " + c.HeadsHash + case waitsOnPreview(c): + last = "it has not yet the merge previews' fingerprints" + default: + return c, nil + } + } + } +} + +// waitsOnPreview says a module of the candidate has no fingerprint yet because its preview was not read. +func waitsOnPreview(c delivery.Candidate) bool { + for _, p := range c.Pulls { + for _, m := range p.Fps { + if m.None2 == candidateWaitsOnPreview { + return true + } + } + } + return false +} diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 5cf259ff..07a468f7 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -1040,8 +1040,14 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { // trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23) indexer string ownJudge bool + // try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's + // group (issue 1921): the same judge, the same verdicts. + try bool }{ {name: "accepted"}, + {name: "accepted in a group try", try: true}, + {name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`}, + {name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"}, {name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"}, {name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"}, {name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"}, @@ -1120,6 +1126,10 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) { p.Modules = []string{"web"} p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}} cs := caseOf(t, p) + if row.try { + cs = TryCase("try-1", Try{Members: []TryMember{{Repository: repoA, Head: head1, Trunk: trunkA}}}, nil) + b.ex.Candidates = &fakeTryCandidates{cand: delivery.Candidate{FPSet: "fps", Pulls: []delivery.CandidatePull{p}}} + } v := b.judge(t, cs).Verdict if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool { return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator") diff --git a/modules/mesh-delivery-checker/internal/check/graph_cycle.go b/modules/mesh-delivery-checker/internal/check/graph_cycle.go index 56766476..b3209485 100644 --- a/modules/mesh-delivery-checker/internal/check/graph_cycle.go +++ b/modules/mesh-delivery-checker/internal/check/graph_cycle.go @@ -30,7 +30,7 @@ type ManifestReader interface { // refuseCycle fails the check for a cycle of the change's module graph, naming it; nil when the case touches no manifest, // when nothing reads manifests, or when there is none. A manifest that cannot be read fails the check, said, never passes it. func (st *state) refuseCycle(ctx context.Context) *delivery.Verdict { - if st.j.Manifests == nil || st.j.Read == nil || st.c.Try != nil { + if st.j.Manifests == nil || st.j.Read == nil || st.narrowTry() { return nil } trunk := st.j.Manifests.ManifestDigests() diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 3480dca7..267f40c0 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -139,8 +139,18 @@ type Examiner struct { Nodes interface { Names(ctx context.Context) ([]string, error) } - Now func() time.Time - Logf func(format string, args ...any) + // Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests + // and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no + // manifest or composition judged, said. + Candidates TryCandidates + Now func() time.Time + Logf func(format string, args ...any) +} + +// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the +// bound; an error says it did not come (and why). +type TryCandidates interface { + TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) } // CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all. @@ -246,6 +256,46 @@ type state struct { // unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not // answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited. unpreviewed []string + // noCandidate says why a whole try has no candidate of its group from the indexer, so its manifests were not judged and + // it was not composed; empty when it has one, and for a pull request's check. + noCandidate string +} + +// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition. +func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() } + +// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds +// (the group's pull-request check has them from its candidate, CaseOf), once the indexer wrote it from the try's previews. +// A try with no such candidate is said so, in a note and in its composition, and is no pass (Tryer.run). +func (st *state) takeTryCandidate(ctx context.Context) { + if st.c.Try == nil || !st.c.Try.Whole() { + return + } + no := func(why string) { + st.noCandidate = "the group's manifests and composition were not judged: " + why + st.c.note(st.noCandidate) + } + if st.j.Candidates == nil { + no("the checker reads no candidate of the indexer's") + return + } + if len(st.unpreviewed) > 0 { + no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)") + return + } + cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait()) + if err != nil { + no(err.Error()) + return + } + for i, p := range st.c.Pulls { + for _, cp := range cand.Pulls { + if cp.Repository == p.Repository && cp.Head.Head == p.Head { + st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps + } + } + } + st.c.FPSet = cand.FPSet } // Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot @@ -256,6 +306,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { // Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR // 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02). tests := st.judgeAll(ctx) + st.takeTryCandidate(ctx) if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil { return st.finish(ctx, *v), nil } @@ -282,7 +333,14 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) { st.c.note("the composition is kept: no fingerprint of a module it composed changed") return } - if st.j.Compositions == nil || st.c.Try != nil { + if st.narrowTry() { + return + } + if st.noCandidate != "" { + v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: st.noCandidate} + return + } + if st.j.Compositions == nil { return } if len(st.unpreviewed) > 0 { @@ -433,7 +491,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict { // Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests // failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0). switch { - case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head + case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest case len(merged) == len(modules): st.j.writePreview(ctx, p, merged) default: @@ -562,7 +620,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch)) } if len(t.fetched.Conflicts) > 0 { - if st.c.Try == nil { + if !st.narrowTry() { st.j.writePreview(ctx, p, append(*merged, t.fetched)) } return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", "))) diff --git a/modules/mesh-delivery-checker/internal/check/manifests_judge.go b/modules/mesh-delivery-checker/internal/check/manifests_judge.go index c34e8308..878fb438 100644 --- a/modules/mesh-delivery-checker/internal/check/manifests_judge.go +++ b/modules/mesh-delivery-checker/internal/check/manifests_judge.go @@ -97,7 +97,7 @@ func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request, // refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that // cannot be asked or does not answer fails the check, said, never passes it. func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict { - if st.j.Manifests == nil || st.c.Try != nil { + if st.j.Manifests == nil || st.narrowTry() { return nil } req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls) diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go index 1fc0efbe..af42b94c 100644 --- a/modules/mesh-delivery-checker/internal/check/read.go +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -164,6 +164,7 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) { if err != nil { return VerdictRead{}, err } + keys = slices.DeleteFunc(keys, IsTryGroup) sort.Strings(keys) keys, cut := capKeys(keys, "") // The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it. @@ -444,6 +445,7 @@ func (r Reader) Status() (Status, error) { if err != nil { return st, err } + keys = slices.DeleteFunc(keys, IsTryGroup) // a try's candidate is judged by its try, never as a check sort.Strings(keys) total := len(keys) keys, cut := capKeys(keys, "") diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index 990cf686..c34b8c7a 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -62,13 +62,17 @@ var ( // Try is the value of a tries key. type Try struct { - V int `json:"v"` - ID string `json:"id"` + V int `json:"v"` + ID string `json:"id"` + // Repository, Head, Trunk and Branch are the first member's (the try's headline, and all a try of one repository + // names); Members are every member of the try, by repository: each an exact commit merged onto a trunk commit (a + // group try, issue 1921). A try written before members read as one member. Repository string `json:"repository"` // Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name. - Head string `json:"head"` - Trunk string `json:"trunk"` - Branch string `json:"branch,omitempty"` + Head string `json:"head"` + Trunk string `json:"trunk"` + Branch string `json:"branch,omitempty"` + Members []TryMember `json:"members,omitempty"` // Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that // level runs. Empty is every test of both levels. Tests string `json:"tests,omitempty"` @@ -83,6 +87,11 @@ type Try struct { Failed []string `json:"failed,omitempty"` Levels map[string]delivery.LevelResult `json:"levels,omitempty"` Machines []string `json:"machines,omitempty"` + // Composition is what the composition on every placed node came to (a whole try, not one narrowed to a level or to + // some tests), and Notes the words the check noted on its way (the manifests judged, the module graph, the kept + // verdicts taken): the same as a group's pull-request check reports. + Composition *delivery.Composition `json:"composition,omitempty"` + Notes []string `json:"notes,omitempty"` // Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key . of its // build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage). Asks []string `json:"asks,omitempty"` @@ -94,6 +103,23 @@ type Try struct { Ended time.Time `json:"ended,omitzero"` } +// TryMember is one repository of a try: the exact commit asked, never the branch's head now, and the trunk commit it is +// merged onto. Branch is the trunk's branch when the trunk was asked by name. +type TryMember struct { + Repository string `json:"repository"` + Head string `json:"head"` + Trunk string `json:"trunk"` + Branch string `json:"branch,omitempty"` +} + +// MembersOf are the try's members: its Members, or the one its headline names. +func (t Try) MembersOf() []TryMember { + if len(t.Members) > 0 { + return t.Members + } + return []TryMember{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk, Branch: t.Branch}} +} + // TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream. const TryAsked = "asked" @@ -195,6 +221,7 @@ func DecodeTry(raw []byte) (Try, error) { if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" { return Try{}, errors.New("not a try: it names no id, repository, head or trunk") } + t.Members = t.MembersOf() return t, nil } @@ -206,6 +233,14 @@ func encodeTry(t Try) ([]byte, error) { // TryAsk is what checker_try is given. type TryAsk struct { Repository, Head, Trunk, Tests, Level string + // Members are the repositories of a group try, each an exact commit and, optionally, its trunk (a commit, or the + // trunk's branch for its head now); the single Repository, Head and Trunk above are one member. Both at once are refused. + Members []TryAskMember +} + +// TryAskMember is one repository of a group try as it is asked. +type TryAskMember struct { + Repository, Head, Trunk string } // TryStore is the tries state as the tools see it. @@ -221,21 +256,16 @@ func NotChecked(repository string) error { return fmt.Errorf("no level of %s is checked by the checker", repository) } -// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second -// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the -// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now. -func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) { +// tryMember checks one member as it is asked and resolves its trunk: the gate's trunk head of its repository, a commit +// asked, or the trunk's branch (its head now). A repository with no trunk head is no repository the checker checks. +func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) { repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git")) head := strings.ToLower(strings.TrimSpace(a.Head)) switch { case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."): - return "", fmt.Errorf("%q is not a repository, /", a.Repository) + return TryMember{}, fmt.Errorf("%q is not a repository, /", a.Repository) case !tryHeadOK.MatchString(head): - return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head) - case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration: - return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level) - case a.Tests != "" && !tryTestsOK.MatchString(a.Tests): - return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests) + return TryMember{}, fmt.Errorf("the head %q of %s is not a full commit, 40 or 64 hex digits", a.Head, repo) } trunkHead := "" for r, h := range trunks { @@ -244,20 +274,67 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) ( } } if trunkHead == "" { - return "", NotChecked(repo) + return TryMember{}, NotChecked(repo) } - t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()} + m := TryMember{Repository: repo, Head: head} switch trunk := strings.TrimSpace(a.Trunk); { case trunk == "": - t.Trunk = trunkHead + m.Trunk = trunkHead case tryHeadOK.MatchString(strings.ToLower(trunk)): - t.Trunk = strings.ToLower(trunk) + m.Trunk = strings.ToLower(trunk) case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."): // The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now. - t.Trunk, t.Branch = trunkHead, trunk + m.Trunk, m.Branch = trunkHead, trunk default: - return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk) + return TryMember{}, fmt.Errorf("the trunk %q of %s is neither a full commit nor a branch", a.Trunk, repo) } + return m, nil +} + +// sameMembers says whether two tries ask the same commits onto the same trunk commits. +func sameMembers(a, b []TryMember) bool { + return slices.EqualFunc(a, b, func(x, y TryMember) bool { + return x.Repository == y.Repository && x.Head == y.Head && x.Trunk == y.Trunk + }) +} + +// AskTry writes a new try, queued, and answers its id: a try of the same commits that waits or runs is the answer to a second +// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. A try is a group of +// one or more members (issue 1921), each an exact commit, never the branch's head now: a push after the ask changes nothing +// of what is tried. trunks are the gate's trunk heads of every repository (source.main): a repository with none is no +// repository the checker checks. A trunk asked by name is its trunk head now. +func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) { + asked := a.Members + if a.Repository != "" || a.Head != "" || a.Trunk != "" { + if len(asked) > 0 { + return "", errors.New("name the members, or one repository with its head, not both") + } + asked = []TryAskMember{{Repository: a.Repository, Head: a.Head, Trunk: a.Trunk}} + } + if len(asked) == 0 { + return "", errors.New("name what to try: members (repository and head each), or a repository and its head") + } + switch { + case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration: + return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level) + case a.Tests != "" && !tryTestsOK.MatchString(a.Tests): + return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests) + } + var members []TryMember + for _, m := range asked { + tm, err := tryMember(trunks, m) + if err != nil { + return "", err + } + if slices.ContainsFunc(members, func(o TryMember) bool { return o.Repository == tm.Repository }) { + return "", fmt.Errorf("%s is named twice: a try holds one commit of a repository", tm.Repository) + } + members = append(members, tm) + } + slices.SortFunc(members, func(x, y TryMember) int { return strings.Compare(x.Repository, y.Repository) }) + first := members[0] + t := Try{Repository: first.Repository, Head: first.Head, Trunk: first.Trunk, Branch: first.Branch, Members: members, + Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()} keys, err := store.Keys() if err != nil { return "", fmt.Errorf("the tries cannot be listed: %w", err) @@ -291,10 +368,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) ( if !live[i] { continue } - if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level { + if sameMembers(cur.MembersOf(), t.Members) && cur.Tests == t.Tests && cur.Level == t.Level { return cur.ID, nil } - said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State) + said := fmt.Sprintf("%s (%s, %s)", cur.ID, sayMembers(cur.MembersOf()), cur.State) if cur.HoldsANode() { holding = append(holding, said) } else { @@ -379,12 +456,50 @@ func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) { return t, nil } +// IsTryGroup says a candidates key is the group of a try (the indexer makes the candidate of a group try's previews, for the +// manifest validator and the composition): the try judges it, so the checker's check of candidates leaves it alone, writes +// no verdict and owes the forge no status for it. +func IsTryGroup(key string) bool { return tryKeyOK.MatchString(key) } + // TryOptions narrow what a try runs; a pull request's check has none. type TryOptions struct { Level string Run string } +// Whole says the try runs everything a group's pull-request check runs (both levels of every member, the manifests judged by +// the manifest validator, the module graph, the composition on every placed node): it is narrowed to neither a level nor +// some tests. Such a try judges by the same code as the check, so the two cannot drift (issue 1921). +func (o *TryOptions) Whole() bool { return o != nil && o.Level == "" && o.Run == "" } + +// TryCase is the case a try is judged as: one pull of the group per member at its exact commit, with no pull request number +// (no ref: the commit is fetched as it is), the group named by the try's key, and the heads hash its candidate is keyed by. +func TryCase(key string, t Try, watch func(string, *delivery.Step)) *Case { + cs := &Case{Key: key, Group: key, Watch: watch, Trunk: map[string]string{}, Queued: t.Asked, Progress: &Progress{}, + Try: &TryOptions{Level: t.Level, Run: t.Tests}} + var heads []delivery.Head + for _, m := range t.MembersOf() { + cs.Pulls = append(cs.Pulls, Pull{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk}) + cs.Trunk[m.Repository] = m.Trunk + heads = append(heads, delivery.Head{Repository: m.Repository, Head: m.Head}) + } + cs.HeadsHash, _ = delivery.HeadsHash(heads) + cs.Heads = cs.HeadsHash + return cs +} + +// compositionMade says whether a composition was made; notMadeWords what it says when it was not. +func compositionMade(c *delivery.Composition) bool { + return c != nil && c.State != delivery.CompositionNotRun +} + +func notMadeWords(c *delivery.Composition) string { + if c == nil || c.Words == "" { + return "no reason was given" + } + return c.Words +} + // Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR // 0525 §1), and writes its end. It holds no timer: a try starts when its key is written. type Tryer struct { @@ -502,12 +617,7 @@ func (tr *Tryer) run(ctx context.Context, key string) { return } progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev} - cs := &Case{Key: key, Group: key, Watch: progress.heard, - Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}}, - Trunk: map[string]string{t.Repository: t.Trunk}, - Queued: t.Asked, - Progress: &Progress{}, - Try: &TryOptions{Level: t.Level, Run: t.Tests}} + cs := TryCase(key, t, progress.heard) judging := ctx if tr.NotTaken > 0 { judging = buildrun.NotTakenBound(ctx, tr.NotTaken) @@ -521,8 +631,13 @@ func (tr *Tryer) run(ctx context.Context, key string) { t.Ended = tr.now().UTC() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks() + t.Composition = v.Composition t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged) - for _, n := range cs.Notes() { + if len(cs.Pulls) > 1 { + t.Said = sayMembers(t.MembersOf()) + ": " + t.Said + } + t.Notes = cs.Notes() + for _, n := range t.Notes { t.Said += " | " + n } switch { @@ -532,6 +647,11 @@ func (tr *Tryer) run(ctx context.Context, key string) { if IsLostAsk(v) || IsNotTaken(v) { t.Failed = []string{v.Failed.Words} } + case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition): + // A whole try that judged the tests and not the manifests and the composition has not tried what its group's check + // judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure. + t.State = TryError + t.Failed = []string{"the composition was not made: " + notMadeWords(v.Composition)} case v.State == delivery.VerdictPass: t.State = TryPass default: @@ -579,6 +699,15 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O return out, true } +// sayMembers names a try's members, each repository at its short commit. +func sayMembers(ms []TryMember) string { + var out []string + for _, m := range ms { + out = append(out, fmt.Sprintf("%s@%.12s", m.Repository, m.Head)) + } + return strings.Join(out, " + ") +} + // orNone joins the tries said, or says none. func orNone(said []string) string { if len(said) == 0 { diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index be2ea659..1f11b00e 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -3,6 +3,7 @@ package check import ( "context" "encoding/json" + "errors" "fmt" "strings" "testing" @@ -70,6 +71,22 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR } // Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or // their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts. + // A group try is the same try whatever the order of its members; the commits asked are the commits tried. + group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}} + gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second)) + if err != nil || gid == id { + t.Fatalf("the group try = %q, %v, want a try of its own", gid, err) + } + group.Members[0], group.Members[1] = group.Members[1], group.Members[0] + if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid { + t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid) + } + if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS { + t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members) + } + if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") { + t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid) + } onNode := func(i int) Try { return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning, Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}} @@ -112,59 +129,113 @@ func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agr } } for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"}, - {Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} { + {Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}, + // A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked. + {}, + {Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}}, + {Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}}, + {Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}}, + {Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} { if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil { t.Errorf("%+v was taken", bad) } } } -func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5 +// fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked. +type fakeTryCandidates struct { + cand delivery.Candidate + err error + asked []string +} + +func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) { + f.asked = append(f.asked, group+"."+heads) + return f.cand, f.err +} + +// wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the +// indexer's candidate of the try's group. +func wholeBench(b *bench, cand delivery.Candidate) (*fakeCompositions, *fakeTryCandidates) { + comps := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}} + cands := &fakeTryCandidates{cand: cand} + b.ex.Compositions, b.ex.Candidates = comps, cands + return comps, cands +} + +// A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request +// check; the two cannot drift, and a later check of the same heads keeps what the try passed. +func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921 ctx := context.Background() kv := newMemKV() - id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now()) + id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now()) if err != nil { t.Fatal(err) } b := newBench(t) + comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"}) written := &writtenTries{memKV: kv} tr := &Tryer{Tries: written, Judge: b.ex} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() got := readTry(t, kv, id) - if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 { + if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 || + got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests t.Fatalf("try = %+v", got) } - // Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end - // keeps each ask's node and state. - if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked || - written.running[2].Steps[0].Node != "shanks" { - t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running) + if got.Composition == nil || got.Composition.State != delivery.CompositionPass || len(got.Machines) == 0 { + t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition) } - if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed { - t.Errorf("the ended try's steps = %+v", got.Steps) + // The same key the indexer's candidate of the group has: the try's id and the heads of its members. + if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") { + t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id) } - // The try names the asks it put, so the delivery view counts their steps as a try's (build_usage). - if len(got.Asks) != 1 { - t.Errorf("asks = %v, want the one check ask the try put", got.Asks) - } else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil { - t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err) + // Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands. + if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked { + t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running) } + if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 { + t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks) + } + for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage) + if _, err := delivery.AskStepKey("shanks", a); err != nil { + t.Errorf("the try names %q, which is no ask id: %v", a, err) + } + } + // Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref. fetch := b.steps.tool("fetch") - if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" { - t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch) + if len(fetch) != 2 { + t.Fatalf("fetch = %+v, want one per member", fetch) } - if keys, _ := b.prev.Keys(ctx); len(keys) != 0 { - t.Errorf("a try wrote the previews %v", keys) + for _, f := range fetch { + want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)] + if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != "" { + t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1]) + } + } + // A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve. + if keys, _ := b.prev.Keys(ctx); len(keys) != 2 { + t.Errorf("a whole try wrote the previews %v, want one per member", keys) } // An ended try is not run again. raw, _, _, _ = kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() - if n := len(b.steps.tool("fetch")); n != 1 { + if n := len(b.steps.tool("fetch")); n != 2 { t.Errorf("an ended try was fetched again: %d fetches", n) } + // The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of + // the same head on the same trunk runs none of it again (and a push of other code runs only what changed). + before := len(b.steps.tool("test")) + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if out.Verdict.State != delivery.VerdictPass || out.Verdict.Levels[delivery.LevelUnit].Kept == 0 || out.Verdict.Levels[delivery.LevelUnit].Run != 0 || + out.Verdict.Levels[delivery.LevelIntegration].Run != 0 { + t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels) + } + if after := len(b.steps.tool("test")); after != before { + t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before) + } } func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { @@ -208,6 +279,34 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) !strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02) t.Errorf("try = %+v", got) } + // A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside + // the failed tests (the group's check does as much), and a candidate of other heads is not the try's. + for _, row := range []struct { + name string + cand *fakeTryCandidates + comp delivery.CompositionVerdict + state, said string + }{ + {name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"}, + {name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"}, + state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"}, + } { + kv := newMemKV() + b := newBench(t) + comps, _ := wholeBench(b, delivery.Candidate{}) + b.ex.Candidates, comps.v = row.cand, row.comp + if row.comp.State == "" { + comps.v.State = delivery.CompositionPass + } + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) + tr := &Tryer{Tries: kv, Judge: b.ex} + raw, _, _, _ := kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) { + t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said) + } + } // No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4). } @@ -224,6 +323,7 @@ func TestATryLeftRunningIsTakenOver(t *testing.T) { t.Fatal(err) } b := newBench(t) + wholeBench(b, delivery.Candidate{}) tr := &Tryer{Tries: kv, Judge: b.ex} tr.Reconsider(ctx) tr.Wait() From b97e277d2d46badead43c11e07b1e361015529b4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:13:29 +0100 Subject: [PATCH 03/13] The indexer writes a group try's candidate from the checker's try and its previews (issue 1921) A whole try has no pull request, yet compose-candidate and the manifest validator read the indexer's candidate of a group. The indexer reads the checker's tries, writes candidates. (number 0 per member) from the previews, judges its manifests with the seat definitions the members carry, leaves try-* alone in the open message, deletes it when the try ends, and has a tool that reads and one that drops it. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-checker-tools/main.go | 2 +- .../cmd/mesh-delivery-indexer-tools/main.go | 1 + .../cmd/mesh-delivery-indexer/previews.go | 31 +- .../cmd/mesh-delivery-indexer/takein.go | 6 +- .../internal/index/candidates.go | 6 +- .../internal/index/pending_tools.go | 62 ++++ .../internal/index/previews.go | 5 +- .../internal/index/previews_test.go | 62 ++++ .../internal/index/takein.go | 22 +- .../internal/index/trycandidates.go | 302 ++++++++++++++++++ .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 3 + 12 files changed, 488 insertions(+), 18 deletions(-) create mode 100644 modules/mesh-delivery-indexer/internal/index/trycandidates.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index d570a958..8fb3566a 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -105,7 +105,7 @@ func TryTools(tr Trying) []stdio.Tool { {Name: check.TryTool, Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " + "name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " + - "or one repository and its head for a try of one member. The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " + + "or one repository and its head for a try of one member (the seat verb try of mesh-delivery-checker takes members from the registry milestone on; until it is delivered, ask a group through this tool). The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " + "each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " + "the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " + "the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " + diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 37bf9725..754f3b94 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -217,6 +217,7 @@ func main() { ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, ask, time.Now)...) + tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, ask, time.Now)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index 21f0b645..8c15b996 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -39,7 +39,7 @@ type previewWatch struct { } func newPreviewWatch(ctx context.Context, js bucketOpener, ix *index.Indexer, logger *log.Logger) *previewWatch { - return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 1)} + return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 2)} } // ensure watches the previews bucket if it is not watched and exists, settling what it holds, then writes the candidates' @@ -73,7 +73,34 @@ func (w *previewWatch) ensure() error { } }) }() - return w.ix.RecomputeCandidates(w.ctx) + w.watchTries() + if err := w.ix.RecomputeCandidates(w.ctx); err != nil { + return err + } + return w.ix.WriteTryCandidates(w.ctx) +} + +// watchTries follows the checker's tries (index.TriesBucket) beside its previews: a whole try that waits or runs gets its +// candidate, once its previews are there, and loses it when it ends (index.Indexer.ApplyTry). A bucket not there is watched at +// the checker's next preview.written, as the previews are. +func (w *previewWatch) watchTries() { + kv, err := w.js.KeyValue(w.ctx, index.TriesBucket) + if err != nil { + w.logger.Printf("the bucket %s cannot be read (%v): a group try's candidate is written when the checker says it wrote a preview", index.TriesBucket, err) + return + } + tw, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutTry(e.Key(), e.Value(), !opOf(e)) }) + if err != nil { + process.NotWatched(w.logger.Printf, index.TriesBucket, err, "it is watched when the checker says it wrote a preview") + return + } + go func() { + w.errs <- follow(w.ctx, tw, func(e jetstream.KeyValueEntry) { + if err := w.ix.ApplyTry(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil { + w.logger.Printf("%s.%s could not be taken into the candidates: %v", index.TriesBucket, e.Key(), err) + } + }) + }() } // Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index cfa942d2..7a40aabb 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -19,6 +19,7 @@ type takerIn interface { TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn DropPending(ctx context.Context, key, by, why string) index.TakenIn TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn + DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn } // takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the @@ -37,7 +38,10 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream return } asked, r := "take-in", index.TakenIn{} - if a.Merge != "" { + if a.TryCandidate != "" && a.Drop { + asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) + a.Module = a.TryCandidate + } else if a.Merge != "" { asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why) a.Module = r.Module } else if a.Pending != "" && a.Drop { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 72acca96..c04f53bd 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -137,7 +137,8 @@ type Indexer struct { Pending Store // SayTakeIn publishes a take-in event on the indexer's own stream take-in (ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error - pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) + tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu + pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } func (ix *Indexer) logf(format string, args ...any) { @@ -283,6 +284,9 @@ func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error { return fmt.Errorf("listing the candidates: %w", err) } for _, key := range keys { + if IsTryGroup(key) { + continue // a try's candidate is no pull request's: the checker's try owns it (trycandidates.go) + } err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) { if !found { return nil, false, nil diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index db6cd499..b49a883f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -2,9 +2,13 @@ package index import ( "context" + "encoding/json" "fmt" + "strings" "time" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" stdio "git.novox.be/novox/mesh-sdk/go" ) @@ -92,3 +96,61 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, Run: merge}, } } + +// TryCandidateTools are the tools of a group try's candidate (issue 1921, trycandidates.go): indexer_try_candidates reads the +// candidates the indexer holds for tries (the try, its heads, each member's modules and whether each has its fingerprint), +// and indexer_drop_try_candidate drops one by an ask in takeins. candidates is the candidates state as the tools bundle reads it. +func TryCandidateTools(candidates Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { + return []stdio.Tool{ + {Name: TryCandidatesTool, + Description: "The candidates the indexer holds for group tries (a try of several repositories at exact commits, mesh-delivery-checker try): " + + "per try its heads hash, each member's repository and commit, the modules the try touches and, per module, its manifest digest, its " + + "fingerprint or why it has none yet (a merge preview not yet written). The checker's manifest validator and the node-engines' " + + "compose-candidate read this candidate. Read-only; an empty answer means no whole try waits or runs.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + ctx := context.Background() + keys, err := candidates.Keys(ctx) + if err != nil { + return nil, err + } + var out []delivery.Candidate + for _, k := range keys { + if !IsTryGroup(k) { + continue + } + v, _, found, err := candidates.Get(ctx, k) + if err != nil || !found { + continue + } + if c, err := delivery.DecodeCandidate(v); err == nil { + out = append(out, c) + } + } + return map[string]any{"tries": out}, nil + }}, + {Name: DropTryCandidateTool, + Description: "Drop a group try's candidate by hand (the indexer deletes it itself when the try ends): for a candidate left for a try that " + + "is gone, or to make the indexer write it again from the previews at the try's next change. A hand act: who and why are said in the journal.", + Input: map[string]any{"type": "object", "required": []string{"try", "why"}, "properties": map[string]any{ + "try": map[string]any{"type": "string", "description": "the try's id, try-"}, + "why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + id, _ := args["try"].(string) + why, _ := args["why"].(string) + id = strings.TrimSpace(id) + if !IsTryGroup(id) || strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("name the try (try-) and say why") + } + data, err := json.Marshal(TakeInAsk{V: 1, TryCandidate: id, Drop: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + a, err := ask("try-candidate.drop."+id, data) + if err != nil { + return nil, err + } + return a.Result, nil + }}, + } +} diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index bb367fb6..18bf869d 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -72,7 +72,10 @@ func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, d if !ix.PutPreview(key, value, deleted) { return nil } - return ix.RecomputeCandidates(ctx) + if err := ix.RecomputeCandidates(ctx); err != nil { + return err + } + return ix.WriteTryCandidates(ctx) // a try that waited for this preview gets its candidate } // RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 282ad860..e7ced5db 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -6,6 +6,7 @@ import ( "os" "strings" "testing" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -182,6 +183,67 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview { t.Errorf("a new head: %+v, want no preview", got) } + // Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try and the + // previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates, + // and deletes it when the try ends. A try narrowed to a level or to tests has none. + t.Run("a group try's candidate", func(t *testing.T) { + ix, mods, cands := previewBed(t, true) + ctx := context.Background() + changed := catalogueManifest(t, "avahi") // not the manifest the trunk holds, so the try touches avahi + pv := avahiPreview(t, trunkNow, h1) + pv.Modules["modules/avahi"] = delivery.PreviewModule{Tree: "tree-avahi", Manifest: delivery.ManifestDigest([]byte(changed)), Locks: map[string]string{"go.sum": avahiGoSum}} + ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) { + return RepoRead{Manifests: map[string]string{dir: changed}}, nil + } + tryValue := func(state, extra string) []byte { + return []byte(`{"id":"try-7","state":"` + state + `",` + extra + `"members":[{"repository":"` + catalogueRepo + `","head":"` + h1 + `","trunk":"` + trunkNow + `"}]}`) + } + held := func() []string { k, _ := cands.Keys(ctx); return k } + if err := ix.ApplyTry(ctx, "try-7", tryValue("running", `"level":"unit",`), false); err != nil || len(held()) != 0 { + t.Fatalf("a narrowed try: %v, candidates %v", err, held()) + } + if err := ix.ApplyTry(ctx, "try-7", tryValue("running", ""), false); err != nil || len(held()) != 0 { + t.Fatalf("a try before its preview: %v, candidates %v", err, held()) + } + applyPreview(t, ix, pv) + key, _ := delivery.CandidatesKey("try-7") + raw, _, found, _ := cands.Get(ctx, key) + if !found { + t.Fatalf("no candidate for the try after its preview: %v", held()) + } + c, err := delivery.DecodeCandidate(raw) + _, _ = moduleOf(t, mods, "avahi") + if err != nil || c.Group != "try-7" || len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head.Head != h1 || + c.Pulls[0].Fps["avahi"].FP2 == "" || c.Pulls[0].Fps["avahi"].Manifest == "" { + t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's fingerprint", c, err) + } + if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 { + t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err) + } + // The tools: one reads the candidates held for tries, one drops by an ask the indexer answers. + asks := func(key string, value []byte) (TakeInAsk, error) { + a, err := DecodeTakeIn(value) + r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) + a.Result = &r + return a, err + } + tools := TryCandidateTools(cands, asks, time.Now) + if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 { + t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err) + } + if _, err := tools[1].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil { + t.Error("a branch was dropped as a try") + } + if out, err := tools[1].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { + t.Errorf("drop = %v, %v, candidates %v", out, err, held()) + } + if err := ix.ApplyTry(ctx, "try-7", tryValue("running", ""), false); err != nil || len(held()) != 1 { + t.Fatalf("the try's next change: %v, candidates %v, want it written again", err, held()) + } + if err := ix.ApplyTry(ctx, "try-7", tryValue("pass", ""), false); err != nil || len(held()) != 0 { + t.Errorf("an ended try: %v, candidates %v, want none", err, held()) + } + }) } func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index e6cb17c6..55ec09a2 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -40,14 +40,16 @@ type TakeInAsk struct { Pending string `json:"pending,omitempty"` // Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message // is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence. - Merge string `json:"merge,omitempty"` - MergeRepository string `json:"merge-repository,omitempty"` - Sequence uint64 `json:"sequence,omitempty"` - Drop bool `json:"drop,omitempty"` - By string `json:"by,omitempty"` - At time.Time `json:"at"` - Result *TakenIn `json:"result,omitempty"` - Answered time.Time `json:"answered,omitzero"` + Merge string `json:"merge,omitempty"` + MergeRepository string `json:"merge-repository,omitempty"` + Sequence uint64 `json:"sequence,omitempty"` + // TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go. + TryCandidate string `json:"try-candidate,omitempty"` + Drop bool `json:"drop,omitempty"` + By string `json:"by,omitempty"` + At time.Time `json:"at"` + Result *TakenIn `json:"result,omitempty"` + Answered time.Time `json:"answered,omitzero"` } // Done says the ask is answered. @@ -62,8 +64,8 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" { - return a, errors.New("a takeins value naming no module, repository, pending wait or merge") + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" { + return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") } return a, nil } diff --git a/modules/mesh-delivery-indexer/internal/index/trycandidates.go b/modules/mesh-delivery-indexer/internal/index/trycandidates.go new file mode 100644 index 00000000..6f44914e --- /dev/null +++ b/modules/mesh-delivery-indexer/internal/index/trycandidates.go @@ -0,0 +1,302 @@ +package index + +// **A group try's candidate** (issue 1921; novox/hq ADR 0593 §7): a try of several repositories at exact commits has no pull +// request, so the forge bridge sends the indexer no message for it; yet the node-engine's compose-candidate and the checker's +// manifest validator read the indexer's candidate of a group. The indexer therefore watches the checker's state tries (the +// checker keeps it; this module reads it) and, for a try that is not narrowed to a level or to some tests, writes +// `candidates.`: one pull per member (number 0), its modules and manifests taken from the merge preview the try's +// fetch wrote, its manifests kept and judged with the seat definitions the members carry, as a take-in judges a revision's. +// The fingerprints come with the previews (RecomputeCandidates). The candidate is deleted when the try ends. `try-` +// is the reserved group name of a try: the bridge's open message never takes it out, and the checker never checks it. + +import ( + "context" + "encoding/json" + "fmt" + "path" + "regexp" + "slices" + "strings" + + "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-indexer/judge" +) + +// TriesBucket is the checker's state of tries, which the indexer reads. +const TriesBucket = delivery.CheckerModule + "_tries" + +// TryCandidateTools: the tools that read and drop a try's candidate. +const ( + TryCandidatesTool = "indexer_try_candidates" + DropTryCandidateTool = "indexer_drop_try_candidate" +) + +var tryGroupOK = regexp.MustCompile(`^try-[0-9]{1,20}$`) + +// IsTryGroup says a candidates key is the group of a try. +func IsTryGroup(key string) bool { return tryGroupOK.MatchString(key) } + +// TryMember is one repository of a try: its exact commit and the trunk commit it is merged onto. +type TryMember struct{ Repository, Head, Trunk string } + +// tryValue is what the indexer reads of a value of the checker's tries. +type tryValue struct { + ID string `json:"id"` + State string `json:"state"` + Level string `json:"level"` + Tests string `json:"tests"` + Members []struct { + Repository string `json:"repository"` + Head string `json:"head"` + Trunk string `json:"trunk"` + } `json:"members"` + Repository, Head, Trunk string +} + +// liveTry reads a tries value: its members, and whether it is a whole try that waits or runs. +func liveTry(value []byte) (id string, members []TryMember, live bool) { + var t tryValue + if json.Unmarshal(value, &t) != nil || !IsTryGroup(t.ID) { + return "", nil, false + } + for _, m := range t.Members { + members = append(members, TryMember{m.Repository, m.Head, m.Trunk}) + } + if len(members) == 0 && t.Repository != "" { + members = []TryMember{{t.Repository, t.Head, t.Trunk}} + } + live = (t.State == "queued" || t.State == "running") && t.Level == "" && t.Tests == "" && len(members) > 0 + return t.ID, members, live +} + +// PutTry takes a change of the tries state in without writing a candidate; it answers the try's id, and whether it is a +// whole try that waits or runs now, and whether it was one before. +func (ix *Indexer) PutTry(key string, value []byte, deleted bool) (id string, live, was bool) { + ix.mu.Lock() + defer ix.mu.Unlock() + _, was = ix.tries[key] + if deleted { + delete(ix.tries, key) + return key, false, was + } + _, members, live := liveTry(value) + if !live { + delete(ix.tries, key) + return key, false, was + } + if ix.tries == nil { + ix.tries = map[string][]TryMember{} + } + ix.tries[key] = members + return key, true, was +} + +// ApplyTry takes one change of the tries in: a whole try that waits or runs gets its candidate (once its previews are +// there), and a try that ended or went loses it. +func (ix *Indexer) ApplyTry(ctx context.Context, key string, value []byte, deleted bool) error { + id, live, was := ix.PutTry(key, value, deleted) + switch { + case live: + return ix.writeTryCandidate(ctx, id) + case was || deleted || IsTryGroup(key): + return ix.dropTryCandidate(ctx, key) + } + return nil +} + +// WriteTryCandidates writes the candidate of every live try that has none yet and whose previews are all there: after a +// preview was written, the try that waited for it. +func (ix *Indexer) WriteTryCandidates(ctx context.Context) error { + ix.mu.Lock() + ids := make([]string, 0, len(ix.tries)) + for id := range ix.tries { + ids = append(ids, id) + } + ix.mu.Unlock() + slices.Sort(ids) + for _, id := range ids { + if err := ix.writeTryCandidate(ctx, id); err != nil { + return err + } + } + return nil +} + +func (ix *Indexer) dropTryCandidate(ctx context.Context, id string) error { + if ix.Store == nil { + return nil + } + key, err := delivery.CandidatesKey(id) + if err != nil { + return err + } + return write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil }) +} + +// DropTryCandidate deletes a try's candidate by hand (indexer_drop_try_candidate); a live try gets it again at its next +// change. +func (ix *Indexer) DropTryCandidate(ctx context.Context, id, by, why string) TakenIn { + r := TakenIn{Module: id} + switch { + case !IsTryGroup(id): + r.Answer = id + " is no try's group, try-: nothing dropped" + default: + if err := ix.dropTryCandidate(ctx, id); err != nil { + r.Answer = "the candidate of " + id + " could not be dropped: " + err.Error() + } else { + r.Stored, r.Answer = true, "the candidate of "+id+" is dropped by "+by+" ("+why+")" + ix.logf("%s", r.Answer) + } + } + return r +} + +// writeTryCandidate writes the candidate of a live try when it has none at its members' heads and every member's merge +// preview is held. +func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error { + if ix.Store == nil || ix.Read == nil { + return nil + } + ix.mu.Lock() + members := slices.Clone(ix.tries[id]) + ix.mu.Unlock() + if len(members) == 0 { + return nil + } + key, err := delivery.CandidatesKey(id) + if err != nil { + return err + } + slices.SortFunc(members, func(a, b TryMember) int { return strings.Compare(a.Repository, b.Repository) }) + heads := make([]delivery.Head, len(members)) + for i, m := range members { + heads[i] = delivery.Head{Repository: m.Repository, Head: m.Head} + } + hash, err := delivery.HeadsHash(heads) + if err != nil { + return err + } + if value, _, found, err := ix.Store.Get(ctx, key); err == nil && found { + if c, err := delivery.DecodeCandidate(value); err == nil && c.HeadsHash == hash { + return nil // fingerprints follow the previews (RecomputeCandidates) + } + } + previews := make([]delivery.Preview, len(members)) + for i, m := range members { + pk, err := delivery.PreviewKey(m.Repository, m.Head, m.Trunk) + if err != nil { + return fmt.Errorf("the try %s: %w", id, err) + } + ix.mu.Lock() + pv, ok := ix.previews[pk] + ix.mu.Unlock() + if !ok { + return nil // its fetch has not written the preview yet: this is called again when it does + } + previews[i] = pv + } + held, err := ix.heldModules(ctx) + if err != nil { + return err + } + pulls := make([]delivery.CandidatePull, len(members)) + var entries []PendingEntry + over := map[string]string{} + for i, m := range members { + pull := delivery.CandidatePull{Head: heads[i], Modules: []string{}} + raws := map[string]string{} + for _, dir := range slices.Sorted(mapKeys(previews[i].Modules)) { + pm := previews[i].Modules[dir] + name := "" + for n, h := range held { + if delivery.SameRepository(h.Repository, m.Repository) && h.Path == forgeDir(dir) { + name = n + if h.Manifest == pm.Manifest { + name = "" // the manifest the trunk holds: the try does not touch the module + } + break + } + } + if name == "" && heldAt(held, m.Repository, dir) { + continue + } + rr, err := ix.Read(ctx, m.Repository, m.Head, forgeDir(dir)) + if err != nil { + return fmt.Errorf("the try %s: reading the manifest of %s at %.12s: %w", id, dir, m.Head, err) + } + raw := rr.Manifests[forgeDir(dir)] + if raw == "" { + ix.logf("the try %s: %s holds no manifest at %.12s, so no module of it is taken", id, dir, m.Head) + continue + } + module := manifestModule(raw) + if module == "" { + module = path.Base(strings.Trim(dir, "/.")) + } + digest, err := ix.keepManifest(ctx, module, raw) + if err != nil { + return err + } + if digest != pm.Manifest { + ix.logf("the try %s: the manifest of %s read at %.12s is not the one its preview names, left", id, dir, m.Head) + continue + } + if pull.Fps == nil { + pull.Fps = map[string]delivery.CandidateModule{} + } + pull.Modules = append(pull.Modules, module) + pull.Fps[module] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview} + raws[dir] = raw + over[module] = digest + } + pulls[i] = pull + entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: raws}}) + } + // The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4). + seats, err := ix.revisionSeats(ctx, entries) + if err != nil { + return fmt.Errorf("the try %s: reading the seat definitions of its members: %w", id, err) + } + judge.WithSeats(seats, func() { ix.judgeDigests(ctx, over, "the try "+id) }) + c := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: hash, Pulls: pulls} + c.FPSet = delivery.FingerprintSet(c.Pulls) + ix.withPreviews(ctx, &c) + if err := ix.refuseDisagreement(c); err != nil { + return err + } + data, err := delivery.EncodeCandidate(c) + if err != nil { + return err + } + return write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil }) +} + +func mapKeys[V any](m map[string]V) func(func(string) bool) { + return func(yield func(string) bool) { + for k := range m { + if !yield(k) { + return + } + } + } +} + +// forgeDir is a preview's directory as the forge reader spells it: the root as "". +func forgeDir(dir string) string { + if dir == "." || dir == "/" { + return "" + } + return strings.Trim(dir, "/") +} + +// heldAt says the trunk holds a module of the repository in that directory. +func heldAt(held map[string]delivery.Module, repository, dir string) bool { + for _, h := range held { + if delivery.SameRepository(h.Repository, repository) && h.Path == forgeDir(dir) { + return true + } + } + return false +} diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index f04f6d04..d372e294 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -112,7 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: graph.State, History: graph.History}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-checker.tries", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control @@ -137,7 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). // 1585/T12, T13 (ADR 0633 §5): indexer_pending reads the pending state; indexer_take_in_pending and indexer_drop_pending // ask through takeins; indexer_take_in_merge takes one merge in, its message asked of the forge bridge (issue 1585). - if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.TakeInMergeTool}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // ADR 0604 §2: it holds the seat mesh-catalog, serving its four verbs from its tools bundle (which stays without a diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 3ae6276e..a468f962 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -22,6 +22,8 @@ "indexer_pending", "indexer_take_in_pending", "indexer_drop_pending", + "indexer_try_candidates", + "indexer_drop_try_candidate", "indexer_take_in_merge" ], "consumes": [ @@ -57,6 +59,7 @@ "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", + "mesh-delivery-checker.tries", "mesh-delivery-gate.source" ], "invokes": [ From 45386212da04f821964738d358c9e82923842195 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:28:52 +0100 Subject: [PATCH 04/13] A group try's candidate follows the checker's try events, judged in one locked pass (issue 1921) The checker says try.started (members at their commits) and try.ended for a whole try; the indexer writes and deletes the candidate from them and finds the open tries again at its start through checker_tries, never reading the checker's state. The try's digests are judged inside the one pass that takes the seats; the definitions of every contracts module of each member are read; the config declarations leave a try's candidate alone. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-checker-tools/main.go | 20 +- .../mesh-delivery-checker-tools/main_test.go | 7 + .../cmd/mesh-delivery-checker/main.go | 10 +- .../internal/check/try.go | 75 +++++++ .../internal/check/try_test.go | 23 ++- .../mesh-delivery-checker/manifest_test.go | 6 +- modules/mesh-delivery-checker/module.json | 3 +- .../cmd/mesh-delivery-indexer/main.go | 9 +- .../cmd/mesh-delivery-indexer/named.go | 21 ++ .../cmd/mesh-delivery-indexer/previews.go | 68 +++--- .../internal/index/candidates.go | 3 + .../internal/index/control_records.go | 3 + .../internal/index/pending_test.go | 2 +- .../internal/index/previews_test.go | 194 ++++++++++++------ .../internal/index/trycandidates.go | 163 ++++++++------- .../internal/index/verdicts.go | 28 +++ .../mesh-delivery-indexer/manifest_test.go | 6 +- modules/mesh-delivery-indexer/module.json | 4 +- sdk/tryevent/tryevent.go | 57 +++++ 19 files changed, 532 insertions(+), 170 deletions(-) create mode 100644 sdk/tryevent/tryevent.go diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index 8fb3566a..a996d06c 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -196,6 +196,24 @@ func TryTools(tr Trying) []stdio.Tool { } } +// OpenTriesTools is the tool that lists the tries that wait or run, each with its members at their commits, and whether it is +// a whole try (the indexer writes the candidate of a whole try's group): the read a part uses at its start to find the tries +// whose start event it missed, instead of reading the checker's state. +func OpenTriesTools(tr Trying) []stdio.Tool { + return []stdio.Tool{{Name: check.TriesTool, + Description: "The tries that wait or run now (a try of a group or of one head; try.started and try.ended on the checker's stream say the same), each with its id, state, its members at their exact commits and " + + "trunk commits, and whether it is a whole try (not narrowed to a level or to tests, so it is composed and its manifests judged). For a part that " + + "missed the checker's try.started or try.ended event (the indexer at its start); checker_try_read reads one try whole. Read-only.", + Input: map[string]any{"type": "object", "properties": map[string]any{}}, + Run: func(map[string]any) (any, error) { + if tr.Tries == nil { + return nil, fmt.Errorf("the tries state cannot be read here") + } + open, err := check.OpenTries(tr.Tries) + return map[string]any{"tries": open}, err + }}} +} + // Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named .. const Seat = "mesh-delivery-checker" @@ -351,7 +369,7 @@ func main() { _, err := verdicts.Update(key, json.RawMessage(value), revision) return err }} - tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...) + tools := append(append(append(append(Tools(r), TryTools(trying)...), OpenTriesTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...) if err := stdio.Serve("", logged(tools)); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go index 8f460c3e..08aa1ca0 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go @@ -8,6 +8,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" + "mesh-delivery-checker/internal/check" ) @@ -134,6 +136,11 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i !strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") { t.Errorf("the try's description does not say what a group try does: %q", d) } + if open := OpenTriesTools(Trying{Tries: store}); len(open) != 1 || open[0].Name != check.TriesTool || !strings.Contains(open[0].Description, "try.started") { + t.Errorf("open tries tool = %+v", open) + } else if out, err := open[0].Run(nil); err != nil || len(out.(map[string]any)["tries"].([]tryevent.Open)) != 2 { + t.Errorf("open tries = %v, %v, want the two tries asked", out, err) + } // Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew. ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"}) if err != nil || ended.(map[string]any)["state"] != "error" { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index fc2222e1..3f898c41 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -21,6 +21,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/announce" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/process" @@ -189,7 +190,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Verdicts: verdicts.all, } // The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries. - tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf} + tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf, + // A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group. + Say: func(ctx context.Context, event string, body tryevent.Event) { + data, _ := json.Marshal(body) + if _, err := js.Publish(ctx, eventSubject(event), data); err != nil { + logger.Printf("%s of %s could not be said: %v (the indexer finds the open tries at its start)", event, body.ID, err) + } + }} announce := func(ctx context.Context, event, key string) { body, _ := json.Marshal(map[string]string{"key": key}) if _, err := js.Publish(ctx, eventSubject(event), body); err != nil { diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index c34b8c7a..51bfb026 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -14,6 +14,7 @@ import ( "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -27,6 +28,8 @@ const ( TryTool = "checker_try" TryReadTool = "checker_try_read" TryEndTool = "checker_try_end" + // TriesTool lists the tries that wait or run (OpenTries). + TriesTool = "checker_tries" // TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-. TriesState = "tries" TriesBucket = delivery.CheckerModule + "_" + TriesState @@ -511,7 +514,12 @@ type Tryer struct { Now func() time.Time Logf func(format string, args ...any) + // Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The + // indexer, which hears them, writes and deletes the candidate of a whole try's group. + Say func(ctx context.Context, event string, body tryevent.Event) + mu sync.Mutex + told map[string]bool // whole tries whose start was said and whose end was not wg sync.WaitGroup inflight map[string]context.CancelFunc seen map[string]bool // looked at since this process started: a try left running is taken over the first time @@ -550,7 +558,12 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) { if stop := tr.inflight[key]; stop != nil { stop() } + said := tr.told[key] + delete(tr.told, key) tr.mu.Unlock() + if said { // ended by hand while it ran: the indexer drops its candidate + tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) + } return } if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) { @@ -617,6 +630,19 @@ func (tr *Tryer) run(ctx context.Context, key string) { return } progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev} + if (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() { + started := tryevent.Event{ID: key} + for _, m := range t.MembersOf() { + started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk}) + } + tr.mu.Lock() + if tr.told == nil { + tr.told = map[string]bool{} + } + tr.told[key] = true + tr.mu.Unlock() + tr.say(ctx, tryevent.Started, started) + } cs := TryCase(key, t, progress.heard) judging := ctx if tr.NotTaken > 0 { @@ -627,6 +653,13 @@ func (tr *Tryer) run(ctx context.Context, key string) { return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end } t, rev = progress.end() + tr.mu.Lock() + said := tr.told[key] + delete(tr.told, key) + tr.mu.Unlock() + if said { + defer tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key}) + } v := out.Verdict t.Ended = tr.now().UTC() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() @@ -671,6 +704,48 @@ func (tr *Tryer) run(ctx context.Context, key string) { tr.logf("%s: %s", key, t.State) } +// say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start). +func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) { + if tr.Say != nil { + tr.Say(ctx, event, body) + } +} + +// Open are the tries that wait or run now, newest scan first (TryScan): what checker_tries answers, so a part that missed an +// event (the indexer at its start) finds them without reading the checker's state. +func OpenTries(store TryStore) ([]tryevent.Open, error) { + keys, err := store.Keys() + if err != nil { + return nil, fmt.Errorf("the tries cannot be listed: %w", err) + } + sort.Slice(keys, func(i, j int) bool { + if len(keys[i]) != len(keys[j]) { + return len(keys[i]) > len(keys[j]) + } + return keys[i] > keys[j] + }) + if len(keys) > TryScan { + keys = keys[:TryScan] + } + var out []tryevent.Open + for _, k := range keys { + raw, err := store.Get(k) + if err != nil || len(raw) == 0 { + continue + } + t, err := DecodeTry(raw) + if err != nil || t.Over() { + continue + } + o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()} + for _, m := range t.MembersOf() { + o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk}) + } + out = append(out, o) + } + return out, nil +} + // judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A // judge that fails to judge, and a check that ran past its bound, are verdicts of failure naming the machine and the step, // and judged says false for them. diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index 1f11b00e..1dc31e87 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -84,6 +85,9 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS { t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members) } + if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 { + t.Errorf("open tries = %+v, %v, want the two queued", open, err) + } if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") { t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid) } @@ -175,11 +179,21 @@ func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR b := newBench(t) comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"}) written := &writtenTries{memKV: kv} - tr := &Tryer{Tries: written, Judge: b.ex} + var said []string + tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { + said = append(said, fmt.Sprintf("%s %s %d", event, e.ID, len(e.Members))) + }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() + // The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate). + if len(said) != 2 || said[0] != "try.started "+id+" 2" || said[1] != "try.ended "+id+" 0" { + t.Errorf("events = %v, want the start with 2 members and the end", said) + } got := readTry(t, kv, id) + if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 { + t.Errorf("open tries after the end = %v, %v", open, err) + } if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 || got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests t.Fatalf("try = %+v", got) @@ -246,10 +260,15 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { t.Fatal(err) } b := newBench(t) - tr := &Tryer{Tries: kv, Judge: b.ex} + tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) { + t.Error("a try of one level and some tests said an event: it has no candidate") + }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() + if open, _ := OpenTries(memTries{kv}); len(open) != 0 { + t.Errorf("open = %v", open) + } tests := b.steps.tool("test") if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" { t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests) diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 6a379ebe..276a1a89 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -130,14 +130,14 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if m.SecretsOwner != "mesh-delivery-checker" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - // The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). - if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) { + // The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). A whole try says try.started with its members and try.ended, so the indexer writes and deletes the candidate of its group (issue 1921). + if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check // is asked again by hand (checker_ask_again). - if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { + if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code. diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 537551f1..23dcc24a 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -18,7 +18,7 @@ "mesh-build-node.toolchain.reported" ], "streams": [ - { "name": "checks", "events": ["preview.**", "owed.**"] } + { "name": "checks", "events": ["preview.**", "owed.**", "try.**"] } ], "state": [ { "name": "asks", "history": 1 }, @@ -56,6 +56,7 @@ "checker_previews", "checker_pull", "checker_status", + "checker_tries", "checker_try", "checker_try_end", "checker_try_read", diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b925b5b2..b9677157 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -348,7 +348,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call), - OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}, Message: messageVia(calls.Call), + OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), Graph: awaitedStore{graph}, Message: messageVia(calls.Call), // The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear; // until then every merge is taken in at once, as before. Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, @@ -498,7 +498,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro case err != nil: logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err) default: - pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf) + pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf) if err != nil { return err } @@ -513,6 +513,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } else if words != "" { logger.Printf("take-in at start: %s", words) } + // The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a + // try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates. + if err := ix.RecoverTries(ctx); err != nil { + logger.Printf("%v", err) + } cons, err := cred.EventConsumer(ctx, js, forgeStream) if err != nil { // Essential: the forge's merges are what the indexer indexes. diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 8822cb8e..791ca492 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -10,6 +10,7 @@ import ( "strings" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-indexer/internal/fingerprint" @@ -294,3 +295,23 @@ func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, er return out, nil } } + +// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries). +const openTriesTool = "mesh-delivery-checker.checker_tries" + +// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596). +func openTriesVia(call toolCaller) func(ctx context.Context) ([]tryevent.Open, error) { + return func(context.Context) ([]tryevent.Open, error) { + raw, err := call(openTriesTool, map[string]any{}) + if err != nil { + return nil, err + } + var o struct { + Tries *[]tryevent.Open `json:"tries"` + } + if err := json.Unmarshal(raw, &o); err != nil || o.Tries == nil { + return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err) + } + return *o.Tries, nil + } +} diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go index 8c15b996..db85de8f 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go @@ -11,6 +11,7 @@ import ( "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-indexer/internal/index" @@ -39,7 +40,7 @@ type previewWatch struct { } func newPreviewWatch(ctx context.Context, js bucketOpener, ix *index.Indexer, logger *log.Logger) *previewWatch { - return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 2)} + return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 1)} } // ensure watches the previews bucket if it is not watched and exists, settling what it holds, then writes the candidates' @@ -73,36 +74,12 @@ func (w *previewWatch) ensure() error { } }) }() - w.watchTries() if err := w.ix.RecomputeCandidates(w.ctx); err != nil { return err } return w.ix.WriteTryCandidates(w.ctx) } -// watchTries follows the checker's tries (index.TriesBucket) beside its previews: a whole try that waits or runs gets its -// candidate, once its previews are there, and loses it when it ends (index.Indexer.ApplyTry). A bucket not there is watched at -// the checker's next preview.written, as the previews are. -func (w *previewWatch) watchTries() { - kv, err := w.js.KeyValue(w.ctx, index.TriesBucket) - if err != nil { - w.logger.Printf("the bucket %s cannot be read (%v): a group try's candidate is written when the checker says it wrote a preview", index.TriesBucket, err) - return - } - tw, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutTry(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - process.NotWatched(w.logger.Printf, index.TriesBucket, err, "it is watched when the checker says it wrote a preview") - return - } - go func() { - w.errs <- follow(w.ctx, tw, func(e jetstream.KeyValueEntry) { - if err := w.ix.ApplyTry(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil { - w.logger.Printf("%s.%s could not be taken into the candidates: %v", index.TriesBucket, e.Key(), err) - } - }) - }() -} - // Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the // checker's is terminated; a failure stays unacknowledged, and the server delivers it again. func (w *previewWatch) Written(m delivered) { @@ -117,3 +94,44 @@ func (w *previewWatch) Written(m delivered) { } _ = m.Ack() } + +// tryEventSubject is the subject of an event of the checker's stream checks about a try. +func tryEventSubject(event string) string { + return "mesh.stream." + delivery.CheckerModule + ".event." + event +} + +// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from +// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and +// the server delivers it again; a body that is no try event is terminated. +func (w *previewWatch) heardChecks(m delivered) { + var event string + switch m.Subject() { + case tryEventSubject(tryevent.Started): + event = tryevent.Started + case tryEventSubject(tryevent.Ended): + event = tryevent.Ended + default: + w.Written(m) + return + } + e, err := tryevent.Decode(event, m.Data()) + if err != nil { + w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err) + _ = m.Term() + return + } + if event == tryevent.Started { + var ms []index.TryMember + for _, x := range e.Members { + ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk}) + } + err = w.ix.StartTry(w.ctx, e.ID, ms) + } else { + err = w.ix.EndTry(w.ctx, e.ID) + } + if err != nil { + w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err) + return + } + _ = m.Ack() +} diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index c04f53bd..b9ccd9e7 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -13,6 +13,7 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -137,6 +138,8 @@ type Indexer struct { Pending Store // SayTakeIn publishes a take-in event on the indexer's own stream take-in (ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error + // OpenTries lists the tries that wait or run, from the checker's tool (checker_tries); nil recovers none (trycandidates.go). + OpenTries func(ctx context.Context) ([]tryevent.Open, error) tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } diff --git a/modules/mesh-delivery-indexer/internal/index/control_records.go b/modules/mesh-delivery-indexer/internal/index/control_records.go index 19941487..9a6dbe21 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records.go @@ -244,6 +244,9 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Fo } var errs []error for _, g := range groups { + if IsTryGroup(g) { + continue // a try's candidate holds no pull request's config declarations (trycandidates.go) + } raw, _, found, err := ix.Store.Get(ctx, g) if err != nil || !found { continue diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 3d2ea4d9..f87f13fd 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -578,7 +578,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) named := map[string]bool{} for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", - "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile"} { + "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index e7ced5db..5a81a83d 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -4,11 +4,13 @@ import ( "context" "encoding/json" "os" + "slices" "strings" "testing" "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" + "git.novox.be/novox/mesh-delivery/sdk/tryevent" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-indexer/internal/fingerprint" @@ -183,67 +185,137 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview { t.Errorf("a new head: %+v, want no preview", got) } - // Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try and the - // previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates, - // and deletes it when the try ends. A try narrowed to a level or to tests has none. - t.Run("a group try's candidate", func(t *testing.T) { - ix, mods, cands := previewBed(t, true) - ctx := context.Background() - changed := catalogueManifest(t, "avahi") // not the manifest the trunk holds, so the try touches avahi - pv := avahiPreview(t, trunkNow, h1) - pv.Modules["modules/avahi"] = delivery.PreviewModule{Tree: "tree-avahi", Manifest: delivery.ManifestDigest([]byte(changed)), Locks: map[string]string{"go.sum": avahiGoSum}} - ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) { - return RepoRead{Manifests: map[string]string{dir: changed}}, nil - } - tryValue := func(state, extra string) []byte { - return []byte(`{"id":"try-7","state":"` + state + `",` + extra + `"members":[{"repository":"` + catalogueRepo + `","head":"` + h1 + `","trunk":"` + trunkNow + `"}]}`) - } - held := func() []string { k, _ := cands.Keys(ctx); return k } - if err := ix.ApplyTry(ctx, "try-7", tryValue("running", `"level":"unit",`), false); err != nil || len(held()) != 0 { - t.Fatalf("a narrowed try: %v, candidates %v", err, held()) - } - if err := ix.ApplyTry(ctx, "try-7", tryValue("running", ""), false); err != nil || len(held()) != 0 { - t.Fatalf("a try before its preview: %v, candidates %v", err, held()) - } - applyPreview(t, ix, pv) - key, _ := delivery.CandidatesKey("try-7") - raw, _, found, _ := cands.Get(ctx, key) - if !found { - t.Fatalf("no candidate for the try after its preview: %v", held()) - } - c, err := delivery.DecodeCandidate(raw) - _, _ = moduleOf(t, mods, "avahi") - if err != nil || c.Group != "try-7" || len(c.Pulls) != 1 || c.Pulls[0].Number != 0 || c.Pulls[0].Head.Head != h1 || - c.Pulls[0].Fps["avahi"].FP2 == "" || c.Pulls[0].Fps["avahi"].Manifest == "" { - t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's fingerprint", c, err) - } - if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 { - t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err) - } - // The tools: one reads the candidates held for tries, one drops by an ask the indexer answers. - asks := func(key string, value []byte) (TakeInAsk, error) { - a, err := DecodeTakeIn(value) - r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) - a.Result = &r - return a, err - } - tools := TryCandidateTools(cands, asks, time.Now) - if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 { - t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err) - } - if _, err := tools[1].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil { - t.Error("a branch was dropped as a try") - } - if out, err := tools[1].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { - t.Errorf("drop = %v, %v, candidates %v", out, err, held()) - } - if err := ix.ApplyTry(ctx, "try-7", tryValue("running", ""), false); err != nil || len(held()) != 1 { - t.Fatalf("the try's next change: %v, candidates %v, want it written again", err, held()) - } - if err := ix.ApplyTry(ctx, "try-7", tryValue("pass", ""), false); err != nil || len(held()) != 0 { - t.Errorf("an ended try: %v, candidates %v, want none", err, held()) - } - }) + // Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try events and + // the previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates, + // and deletes it when the try ends. Its manifests are judged with the seat definitions its members carry, in one locked + // pass, whether the contracts module's manifest changed or only its seat.json did (the wait is bounded: a lock held twice + // would show as a failure here, not as a stuck run). + for _, row := range []struct { + name string + unchanged bool // the contracts module's module.json is the trunk's: only its definitions changed + }{ + {name: "a changed manifest that carries definitions"}, + {name: "an unchanged manifest that carries definitions", unchanged: true}, + } { + t.Run("a group try's candidate: "+row.name, func(t *testing.T) { + ix, mods, cands := previewBed(t, true) + ctx := context.Background() + ix.Verdicts = newMem() + var changed map[string]any + if err := json.Unmarshal([]byte(catalogueManifest(t, "avahi")), &changed); err != nil { + t.Fatal(err) + } + changed["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go-seats", "kind": "package", "language": "go", "paths": []any{"go"}, "generated-from": "defs"}}} + raw, _ := json.Marshal(changed) + pv := avahiPreview(t, trunkNow, h1) + pm := delivery.PreviewModule{Tree: "tree-avahi", Manifest: delivery.ManifestDigest(raw), Locks: map[string]string{"go.sum": avahiGoSum}} + if row.unchanged { + m, found := moduleOf(t, mods, "avahi") + _ = found + pm.Manifest = m.Manifest + key, _ := delivery.ManifestsKey(m.Manifest) + data, _ := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(raw)}) + if _, _, _, err := ix.Manifests.Get(ctx, key); err != nil { + t.Fatal(err) + } + _ = ix.Manifests.(*memStore).Update(ctx, key, data, ix.Manifests.(*memStore).revs[key]) + } + pv.Modules["modules/avahi"] = pm + ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) { + return RepoRead{Manifests: map[string]string{dir: string(raw)}}, nil + } + listed := 0 + ix.Files = func(_ context.Context, _, commit string, dirs []string) (map[string]map[string]string, error) { + listed++ + if commit != h1 || !slices.Equal(dirs, []string{"modules/avahi/defs/seats"}) { + t.Errorf("listed %v at %s, want the member's head", dirs, commit) + } + return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil + } + ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) { + switch p { + case "modules/avahi/defs/seats/node-engine/seat.json": + return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil + case "modules/avahi/defs/parts/reads.json": + return []byte(`{"format":1,"part":"reads","verbs":[{"name":"try-verb"}]}`), nil + } + return nil, nil + } + started := `{"id":"try-7"}` + _ = started + members := []TryMember{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}} + held := func() []string { k, _ := cands.Keys(ctx); return k } + if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 0 { + t.Fatalf("a try before its preview: %v, candidates %v", err, held()) + } + done := make(chan struct{}) + go func() { defer close(done); applyPreview(t, ix, pv) }() + select { + case <-done: + case <-time.After(10 * time.Second): + panic("writing the try's candidate hangs: the seat lock is held twice (a locked pass cannot be left by t.Fatal: the lock stays held)") + } + if listed == 0 { + t.Error("the member's definitions were not read") + } + key, _ := delivery.CandidatesKey("try-7") + rawc, _, found, _ := cands.Get(ctx, key) + if !found { + t.Fatalf("no candidate for the try after its preview: %v", held()) + } + c, err := delivery.DecodeCandidate(rawc) + pull := delivery.CandidatePull{} + if len(c.Pulls) == 1 { + pull = c.Pulls[0] + } + if row.unchanged { + if err != nil || c.Group != "try-7" || len(pull.Modules) != 0 { + t.Fatalf("candidate = %+v, %v, want the try's group with no module touched", c, err) + } + } else if err != nil || c.Group != "try-7" || pull.Number != 0 || pull.Head.Head != h1 || pull.Fps["avahi"].Manifest == "" { // no fingerprint asserted: this mesh-sdk pin does not read generated-from yet + t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's manifest", c, err) + } + if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 { + t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err) + } + // The tools: one reads the candidates held for tries, one drops by an ask the indexer answers. + asks := func(key string, value []byte) (TakeInAsk, error) { + a, err := DecodeTakeIn(value) + r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) + a.Result = &r + return a, err + } + tools := TryCandidateTools(cands, asks, time.Now) + if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 { + t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err) + } + if _, err := tools[1].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil { + t.Error("a branch was dropped as a try") + } + if out, err := tools[1].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { + t.Errorf("drop = %v, %v, candidates %v", out, err, held()) + } + // The start recovered from the checker's tool: an open whole try is written again, one the checker no longer + // lists is deleted, and a try narrowed to a level is none. + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { + return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}, + {ID: "try-8", Whole: false, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h2, Trunk: trunkNow}}}}, nil + } + if err := ix.RecoverTries(ctx); err != nil || len(held()) != 1 { + t.Fatalf("recovered: %v, candidates %v, want try-7's alone", err, held()) + } + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil } + if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 { + t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held()) + } + if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 { + t.Fatalf("started again: %v, candidates %v", err, held()) + } + if err := ix.EndTry(ctx, "try-7"); err != nil || len(held()) != 0 { + t.Errorf("an ended try: %v, candidates %v, want none", err, held()) + } + }) + } } func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/trycandidates.go b/modules/mesh-delivery-indexer/internal/index/trycandidates.go index 6f44914e..6044ebe3 100644 --- a/modules/mesh-delivery-indexer/internal/index/trycandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/trycandidates.go @@ -11,7 +11,6 @@ package index import ( "context" - "encoding/json" "fmt" "path" "regexp" @@ -20,8 +19,6 @@ import ( "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" - - "mesh-delivery-indexer/judge" ) // TriesBucket is the checker's state of tries, which the indexer reads. @@ -41,67 +38,67 @@ func IsTryGroup(key string) bool { return tryGroupOK.MatchString(key) } // TryMember is one repository of a try: its exact commit and the trunk commit it is merged onto. type TryMember struct{ Repository, Head, Trunk string } -// tryValue is what the indexer reads of a value of the checker's tries. -type tryValue struct { - ID string `json:"id"` - State string `json:"state"` - Level string `json:"level"` - Tests string `json:"tests"` - Members []struct { - Repository string `json:"repository"` - Head string `json:"head"` - Trunk string `json:"trunk"` - } `json:"members"` - Repository, Head, Trunk string -} - -// liveTry reads a tries value: its members, and whether it is a whole try that waits or runs. -func liveTry(value []byte) (id string, members []TryMember, live bool) { - var t tryValue - if json.Unmarshal(value, &t) != nil || !IsTryGroup(t.ID) { - return "", nil, false +// StartTry takes a whole try in, as the checker said it started (tryevent.Started): its members at their commits are kept, and +// its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596). +func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error { + if !IsTryGroup(id) || len(members) == 0 { + return fmt.Errorf("%q with %d members is no try to start", id, len(members)) } - for _, m := range t.Members { - members = append(members, TryMember{m.Repository, m.Head, m.Trunk}) - } - if len(members) == 0 && t.Repository != "" { - members = []TryMember{{t.Repository, t.Head, t.Trunk}} - } - live = (t.State == "queued" || t.State == "running") && t.Level == "" && t.Tests == "" && len(members) > 0 - return t.ID, members, live -} - -// PutTry takes a change of the tries state in without writing a candidate; it answers the try's id, and whether it is a -// whole try that waits or runs now, and whether it was one before. -func (ix *Indexer) PutTry(key string, value []byte, deleted bool) (id string, live, was bool) { ix.mu.Lock() - defer ix.mu.Unlock() - _, was = ix.tries[key] - if deleted { - delete(ix.tries, key) - return key, false, was - } - _, members, live := liveTry(value) - if !live { - delete(ix.tries, key) - return key, false, was - } if ix.tries == nil { ix.tries = map[string][]TryMember{} } - ix.tries[key] = members - return key, true, was + ix.tries[id] = slices.Clone(members) + ix.mu.Unlock() + return ix.writeTryCandidate(ctx, id) } -// ApplyTry takes one change of the tries in: a whole try that waits or runs gets its candidate (once its previews are -// there), and a try that ended or went loses it. -func (ix *Indexer) ApplyTry(ctx context.Context, key string, value []byte, deleted bool) error { - id, live, was := ix.PutTry(key, value, deleted) - switch { - case live: - return ix.writeTryCandidate(ctx, id) - case was || deleted || IsTryGroup(key): - return ix.dropTryCandidate(ctx, key) +// EndTry takes a try out, as the checker said it ended (tryevent.Ended), and deletes its candidate. +func (ix *Indexer) EndTry(ctx context.Context, id string) error { + ix.mu.Lock() + delete(ix.tries, id) + ix.mu.Unlock() + if !IsTryGroup(id) { + return nil + } + return ix.dropTryCandidate(ctx, id) +} + +// RecoverTries is, at the indexer's start, what the events it missed would have said: the candidates of the tries that wait +// or run now are written, and the candidate of a try the checker no longer lists is deleted. The checker's tool lists them +// (OpenTries); its bucket is not read. Nothing to recover from when no part is wired to ask. +func (ix *Indexer) RecoverTries(ctx context.Context) error { + if ix.OpenTries == nil || ix.Store == nil { + return nil + } + open, err := ix.OpenTries(ctx) + if err != nil { + return fmt.Errorf("the open tries cannot be listed, so a missed try event is not recovered: %w", err) + } + live := map[string]bool{} + for _, o := range open { + if !o.Whole || len(o.Members) == 0 { + continue + } + live[o.ID] = true + var ms []TryMember + for _, m := range o.Members { + ms = append(ms, TryMember{m.Repository, m.Head, m.Trunk}) + } + if err := ix.StartTry(ctx, o.ID, ms); err != nil { + return err + } + } + keys, err := ix.Store.Keys(ctx) + if err != nil { + return err + } + for _, k := range keys { + if IsTryGroup(k) && !live[k] { + if err := ix.EndTry(ctx, k); err != nil { + return err + } + } } return nil } @@ -206,21 +203,23 @@ func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error { over := map[string]string{} for i, m := range members { pull := delivery.CandidatePull{Head: heads[i], Modules: []string{}} - raws := map[string]string{} + // defs are every module directory's manifest of the member at its head, the unchanged ones too: a contracts module whose + // seat.json changed while its module.json did not carries definitions all the same (ADR 0636 §4). + defs := map[string]string{} for _, dir := range slices.Sorted(mapKeys(previews[i].Modules)) { pm := previews[i].Modules[dir] - name := "" + name, unchanged := "", false for n, h := range held { if delivery.SameRepository(h.Repository, m.Repository) && h.Path == forgeDir(dir) { - name = n - if h.Manifest == pm.Manifest { - name = "" // the manifest the trunk holds: the try does not touch the module - } + name, unchanged = n, h.Manifest == pm.Manifest break } } - if name == "" && heldAt(held, m.Repository, dir) { - continue + if unchanged { + if raw := ix.keptManifest(ctx, pm.Manifest); raw != "" { + defs[dir] = raw + } + continue // the manifest the trunk holds: the try does not touch the module } rr, err := ix.Read(ctx, m.Repository, m.Head, forgeDir(dir)) if err != nil { @@ -232,6 +231,9 @@ func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error { continue } module := manifestModule(raw) + if module == "" { + module = name + } if module == "" { module = path.Base(strings.Trim(dir, "/.")) } @@ -248,18 +250,21 @@ func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error { } pull.Modules = append(pull.Modules, module) pull.Fps[module] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview} - raws[dir] = raw + defs[dir] = raw over[module] = digest } pulls[i] = pull - entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: raws}}) + entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: defs}}) } - // The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4). + // The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4), in one pass that takes the + // seats. Their verdicts are the verdicts state's, keyed by the manifest's digest, as a pull request's candidate's are + // (judgeDigests in ApplyMessage): nothing reads them but the engines' composition of that digest and the module + // value of a trunk module with the same digest. seats, err := ix.revisionSeats(ctx, entries) if err != nil { return fmt.Errorf("the try %s: reading the seat definitions of its members: %w", id, err) } - judge.WithSeats(seats, func() { ix.judgeDigests(ctx, over, "the try "+id) }) + ix.judgeDigestsWith(ctx, seats, over, "the try "+id) c := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: hash, Pulls: pulls} c.FPSet = delivery.FingerprintSet(c.Pulls) ix.withPreviews(ctx, &c) @@ -273,6 +278,26 @@ func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error { return write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil }) } +// keptManifest is the text of the manifest the indexer keeps under a digest, or empty. +func (ix *Indexer) keptManifest(ctx context.Context, digest string) string { + if ix.Manifests == nil || digest == "" { + return "" + } + key, err := delivery.ManifestsKey(digest) + if err != nil { + return "" + } + value, _, found, err := ix.Manifests.Get(ctx, key) + if err != nil || !found { + return "" + } + m, err := delivery.DecodeManifest(value) + if err != nil { + return "" + } + return m.JSON +} + func mapKeys[V any](m map[string]V) func(func(string) bool) { return func(yield func(string) bool) { for k := range m { diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 5eabc0a2..94f65448 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -65,6 +65,13 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, over ma // The seat list stands still for this pass: a take-in's dry run may put its revision's seats over it (seatsof.go). defer judge.HoldSeats()() } + return ix.judgeOverHeld(ctx, fresh, over) +} + +// judgeOverHeld is judgeOver for a caller that holds the judge pass (judgeMu) and keeps the seat list as it wants it +// (judge.HoldSeats, or inside judge.WithSeats): it takes no lock, so a pass that put its seats over the list judges inside +// it without asking the shared lock the exclusive one forbids. +func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, over map[string]string) (Judged, error) { var out Judged if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil { return out, nil @@ -252,6 +259,27 @@ func (ix *Indexer) judgeDigests(ctx context.Context, over map[string]string, why } } +// judgeDigestsWith is judgeDigests with the seats put over the catalogue's list for the whole pass (judge.WithSeats): one +// locked pass that takes the seats, and never a second lock under it (the exclusive one of WithSeats and the shared one of +// judgeOver would wait for each other for good). +func (ix *Indexer) judgeDigestsWith(ctx context.Context, seats []catalogue.Seat, over map[string]string, why string) { + if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil { + return + } + ix.judgeMu.Lock() + defer ix.judgeMu.Unlock() + judge.WithSeats(seats, func() { + for _, module := range slices.Sorted(maps.Keys(over)) { + if !ix.staleVerdict(ctx, over[module]) { + continue + } + if _, err := ix.judgeOverHeld(ctx, nil, map[string]string{module: over[module]}); err != nil { + ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err) + } + } + }) +} + // judgeStale judges again every digest of the verdicts state another judge refused or held waiting, each as a manifest // of the module its verdict names (judgeDigests). Never on a timer: at the start pass. func (ix *Indexer) judgeStale(ctx context.Context) { diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index d372e294..db8524e0 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -112,12 +112,12 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build {Name: graph.State, History: graph.History}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-checker.tries", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { + if want := []string{"mesh-build-go.toolchains", "mesh-build-node.toolchains", "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", "mesh-delivery-gate.source"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } // "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control // internal/broker/state.go:62-93). - if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-checker.preview.written", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) { + if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) { t.Errorf("consumes = %v, want %v", m.Consumes, want) } if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" { @@ -129,7 +129,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths: // ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a // commit is in another (issue 1900). - if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition"}; !reflect.DeepEqual(m.Invokes, want) { + if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition"}; !reflect.DeepEqual(m.Invokes, want) { t.Errorf("invokes = %v, want %v", m.Invokes, want) } // 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index a468f962..4b39a52c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -30,6 +30,8 @@ "mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-checker.preview.written", + "mesh-delivery-checker.try.started", + "mesh-delivery-checker.try.ended", "mesh-delivery-forge-bridge.forge.**" ], "state": [ @@ -59,7 +61,6 @@ "mesh-delivery-assigner.placement", "mesh-delivery-builder.build-log", "mesh-delivery-checker.previews", - "mesh-delivery-checker.tries", "mesh-delivery-gate.source" ], "invokes": [ @@ -67,6 +68,7 @@ "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", + "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition" ], diff --git a/sdk/tryevent/tryevent.go b/sdk/tryevent/tryevent.go new file mode 100644 index 00000000..8d1e83c5 --- /dev/null +++ b/sdk/tryevent/tryevent.go @@ -0,0 +1,57 @@ +// Package tryevent is what the checker says on its stream checks when a whole try (a try of a delivery group, not narrowed +// to a level or to some tests) starts and ends (issue 1921), and what the indexer, which hears it, reads: a try has no pull +// request, so the indexer writes the candidate of its group from the members at their commits, and deletes it at the end. +// The events carry the members, so no part reads the checker's state. +package tryevent + +import ( + "encoding/json" + "fmt" + "regexp" +) + +// The events of the checker's stream checks. +const ( + Started = "try.started" + Ended = "try.ended" +) + +// IDOK is the shape of a try's id, which is also the name of its group (the key of its candidate). +var IDOK = regexp.MustCompile(`^try-[0-9]{1,20}$`) + +// Member is one repository of the try: the exact commit and the trunk commit it is merged onto. +type Member struct { + Repository string `json:"repository"` + Head string `json:"head"` + Trunk string `json:"trunk"` +} + +// Event is the body of both events: the try, and for Started its members; for Ended the state it ended in. +type Event struct { + ID string `json:"id"` + Members []Member `json:"members,omitempty"` + State string `json:"state,omitempty"` +} + +// Decode reads an event body; a started event names its members. +func Decode(event string, body []byte) (Event, error) { + var e Event + if err := json.Unmarshal(body, &e); err != nil { + return e, err + } + if !IDOK.MatchString(e.ID) { + return e, fmt.Errorf("%q is not a try's id, try-", e.ID) + } + if event == Started && len(e.Members) == 0 { + return e, fmt.Errorf("the try %s starts with no member", e.ID) + } + return e, nil +} + +// Open is one try that waits or runs, as the checker's tool checker_tries answers it. +type Open struct { + ID string `json:"id"` + State string `json:"state"` + Whole bool `json:"whole"` + Members []Member `json:"members"` +} From 92c38981ea36c0485b1bcb0dfb78906268ceb62c Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:30:35 +0100 Subject: [PATCH 05/13] mesh-sdk's tests read mesh-contracts and mesh-delivery beside it (issue 1921) mesh-sdk compares its bootstrapped seat definitions with their contracts files, read beside it; the checker places both, at the group's head for a member, else at the trunk, for a check and for a group try alike. Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- modules/mesh-delivery-checker/internal/check/beside.go | 9 +++++++-- .../mesh-delivery-checker/internal/check/pins_test.go | 8 +++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/modules/mesh-delivery-checker/internal/check/beside.go b/modules/mesh-delivery-checker/internal/check/beside.go index d64b4c37..d0fa103c 100644 --- a/modules/mesh-delivery-checker/internal/check/beside.go +++ b/modules/mesh-delivery-checker/internal/check/beside.go @@ -29,18 +29,23 @@ const ( nodeEngineModule = "mesh-host" // networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read. networkingRepository = "novox/mesh-networking" + // contractsRepository and deliveryRepository hold the seat definitions mesh-sdk's bootstrapped seats are compared with: its + // tests read both beside it (beside.Dir "mesh-contracts" and "mesh-delivery"), at the group's head for a member, else at the + // trunk (a group try lays them out the same way: besideFor serves the check and the try). + contractsRepository = "novox/mesh-contracts" + deliveryRepository = DeliveryRepository ) // besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside // beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog // checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its // unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no -// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593 +// siblings placed either. mesh-sdk reads mesh-contracts and mesh-delivery (its bootstrapped seats against their contracts files). The catalogue's tests read the node-engine since its composer command moved there (ADR 0593 // §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a // composition moved into them (ADR 0593). var besideRules = map[string][]string{ nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository}, - SDKRepository: {catalogueRepository, nodeEngineRepo}, + SDKRepository: {catalogueRepository, nodeEngineRepo, contractsRepository, deliveryRepository}, catalogueRepository: {SDKRepository, nodeEngineRepo}, "novox/mesh-delivery": {SDKRepository, catalogueRepository}, } diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go index 81a7616f..5c61971e 100644 --- a/modules/mesh-delivery-checker/internal/check/pins_test.go +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -111,10 +111,16 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR } // Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it. b = newBench(t) - b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2}) + const contracts = "novox/mesh-contracts" + b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, contracts: trunkA, DeliveryRepository: trunkA2}) if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA { t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA) } + // mesh-sdk's tests compare its bootstrapped seats with their contracts files, read beside it: mesh-contracts and + // mesh-delivery, at their trunks alone (a group try is placed by the same besideFor, so it lays them out the same way). + if got := besideOf(t, b, SDKRepository); got[contracts].Commit != trunkA || got[DeliveryRepository].Commit != trunkA2 { + t.Errorf("beside mesh-sdk alone = %+v, want mesh-contracts at %s and mesh-delivery at %s", got, trunkA, trunkA2) + } if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" { t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA) } From c2b8ac3370278ba040d5313881a2f302e3f08b3d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:35:22 +0100 Subject: [PATCH 06/13] The indexer hears the checker's events after it lists the open tries; rows for the start order and the config skip (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-indexer/main.go | 21 ++++++++------ .../internal/index/control_records_test.go | 17 +++++++++++ .../internal/index/previews_test.go | 28 +++++++++++++++++++ .../internal/index/trycandidates.go | 3 -- 4 files changed, 57 insertions(+), 12 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index b9677157..86ba8676 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -494,15 +494,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer tc.Stop() } - switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); { - case err != nil: - logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err) - default: - pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf) - if err != nil { - return err - } - defer pc.Stop() + // The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event + // said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way. + pcons, perr := cred.EventConsumer(ctx, js, PreviewsStream) + if perr != nil { + logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews and tries are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, perr) } // The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's // stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer @@ -518,6 +514,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := ix.RecoverTries(ctx); err != nil { logger.Printf("%v", err) } + if perr == nil { + pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf) + if err != nil { + return err + } + defer pc.Stop() + } cons, err := cred.EventConsumer(ctx, js, forgeStream) if err != nil { // Essential: the forge's merges are what the indexer indexes. diff --git a/modules/mesh-delivery-indexer/internal/index/control_records_test.go b/modules/mesh-delivery-indexer/internal/index/control_records_test.go index 358d802b..0b2ddaba 100644 --- a/modules/mesh-delivery-indexer/internal/index/control_records_test.go +++ b/modules/mesh-delivery-indexer/internal/index/control_records_test.go @@ -85,5 +85,22 @@ func TestTheControlRecords(t *testing.T) { if _, ok := cands.vals[key]; ok { t.Error("a closed pull request's declarations stayed") } + // A group try's candidate (number 0) holds no pull request's declarations: a merge that touches its module writes none + // for it (fails when configCandidatesAfterMerge no longer skips a try's group). + tryPull := delivery.CandidatePull{Head: delivery.Head{Repository: catalogueRepo, Head: h1}, Modules: []string{"avahi"}, + Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(trunk)), None2: NoneNoPreview}}} + tryCand := delivery.Candidate{V: delivery.Version, Group: "try-5", Pulls: []delivery.CandidatePull{tryPull}} + tryCand.HeadsHash, _ = delivery.HeadsHash([]delivery.Head{tryPull.Head}) + tryCand.FPSet = delivery.FingerprintSet(tryCand.Pulls) + data, err := delivery.EncodeCandidate(tryCand) + if err != nil { + t.Fatal(err) + } + if err := ix.Store.Create(ctx, "try-5", data); err != nil { + t.Fatal(err) + } + if err := ix.configCandidatesAfterMerge(ctx, mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t2"}), []string{"avahi"}); err != nil || len(cands.vals) != 0 { + t.Errorf("a merge touching a try's module wrote %v for it (%v), want none", cands.vals, err) + } }) } diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 5a81a83d..701fdc8f 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -308,6 +308,34 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 { t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held()) } + // ADR 0638 §3, §4: the consumer exists before the call, so what happens between the call and the hearing is heard after + // it. A try the call listed and whose start is then heard gets one candidate; a try started after the list gets one; a + // try that ended during the call, listed as open, leaves none once its end is heard. + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { + return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}, + {ID: "try-9", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}}, nil + } + if err := ix.RecoverTries(ctx); err != nil { + t.Fatal(err) + } + for _, step := range []struct { + what string + do func() error + want int + }{ + {"the start heard after the call, of a try the call listed", func() error { return ix.StartTry(ctx, "try-7", members) }, 2}, + {"a start after the list", func() error { return ix.StartTry(ctx, "try-10", members) }, 3}, + {"the end heard of a try ended during the call", func() error { return ix.EndTry(ctx, "try-9") }, 2}, + } { + if err := step.do(); err != nil || len(held()) != step.want { + t.Fatalf("%s: %v, candidates %v, want %d", step.what, err, held(), step.want) + } + } + for _, id := range []string{"try-10"} { + if err := ix.EndTry(ctx, id); err != nil { + t.Fatal(err) + } + } if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 { t.Fatalf("started again: %v, candidates %v", err, held()) } diff --git a/modules/mesh-delivery-indexer/internal/index/trycandidates.go b/modules/mesh-delivery-indexer/internal/index/trycandidates.go index 6044ebe3..ae2b0fb7 100644 --- a/modules/mesh-delivery-indexer/internal/index/trycandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/trycandidates.go @@ -21,9 +21,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// TriesBucket is the checker's state of tries, which the indexer reads. -const TriesBucket = delivery.CheckerModule + "_tries" - // TryCandidateTools: the tools that read and drop a try's candidate. const ( TryCandidatesTool = "indexer_try_candidates" From bd5eb33e9ed99604fbd214227c0ccb0af4a49c44 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:38:11 +0100 Subject: [PATCH 07/13] A failed recovery of the open tries is said and retried; every open try is listed; a silent node ends a try as error (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-checker/main.go | 2 +- .../internal/check/try.go | 34 ++++-- .../internal/check/try_test.go | 22 ++++ .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 8 +- .../cmd/mesh-delivery-indexer/takein.go | 6 +- .../internal/index/candidates.go | 9 +- .../internal/index/pending_test.go | 2 +- .../internal/index/pending_tools.go | 37 +++++- .../internal/index/previews_test.go | 49 +++++++- .../internal/index/takein.go | 6 +- .../internal/index/trycandidates.go | 108 +++++++++++++++++- .../mesh-delivery-indexer/manifest_test.go | 4 +- modules/mesh-delivery-indexer/module.json | 2 + 14 files changed, 257 insertions(+), 34 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 3f898c41..63bcfcae 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -194,7 +194,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group. Say: func(ctx context.Context, event string, body tryevent.Event) { data, _ := json.Marshal(body) - if _, err := js.Publish(ctx, eventSubject(event), data); err != nil { + if _, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second)); err != nil { logger.Printf("%s of %s could not be said: %v (the indexer finds the open tries at its start)", event, body.ID, err) } }} diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index 51bfb026..e794ebc8 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -680,6 +680,11 @@ func (tr *Tryer) run(ctx context.Context, key string) { if IsLostAsk(v) || IsNotTaken(v) { t.Failed = []string{v.Failed.Words} } + case v.State == delivery.VerdictPass && cs.Try.Whole() && compositionMade(v.Composition) && strings.Contains(v.Composition.Words, delivery.ComposeNoAnswer): + // A node that gave no answer passes a group's check with a warning; a try that is to say the group composes says it + // did not, naming the node (ADR 0637 §3). + t.State = TryError + t.Failed = []string{"the composition was not made on every placed node: " + v.Composition.Words} case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition): // A whole try that judged the tests and not the manifests and the composition has not tried what its group's check // judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure. @@ -711,8 +716,9 @@ func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) { } } -// Open are the tries that wait or run now, newest scan first (TryScan): what checker_tries answers, so a part that missed an -// event (the indexer at its start) finds them without reading the checker's state. +// OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at +// its start) finds them without reading the checker's state. All the keys are read (an old open try is as open as a new one), in +// parallel. func OpenTries(store TryStore) ([]tryevent.Open, error) { keys, err := store.Keys() if err != nil { @@ -724,24 +730,30 @@ func OpenTries(store TryStore) ([]tryevent.Open, error) { } return keys[i] > keys[j] }) - if len(keys) > TryScan { - keys = keys[:TryScan] - } - var out []tryevent.Open - for _, k := range keys { - raw, err := store.Get(k) + found := make([]*tryevent.Open, len(keys)) + if err := parallel(len(keys), func(i int) error { + raw, err := store.Get(keys[i]) if err != nil || len(raw) == 0 { - continue + return nil } t, err := DecodeTry(raw) if err != nil || t.Over() { - continue + return nil } o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()} for _, m := range t.MembersOf() { o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk}) } - out = append(out, o) + found[i] = &o + return nil + }); err != nil { + return nil, err + } + var out []tryevent.Open + for _, o := range found { + if o != nil { + out = append(out, *o) + } } return out, nil } diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index 1dc31e87..46133c57 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -284,6 +284,26 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { } } +func TestATryOfOneLevelKeepsOnlyThatLevel(t *testing.T) { + ctx := context.Background() + kv := newMemKV() + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now()) + b := newBench(t) + tr := &Tryer{Tries: kv, Judge: b.ex} + raw, _, _, _ := kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + keys, _ := b.kept.Keys(ctx) + if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 { + t.Fatalf("try = %+v, kept %v", got, keys) + } + for _, k := range keys { + if !strings.HasPrefix(k, delivery.LevelUnit+".") { + t.Errorf("a try of the unit level kept %s", k) + } + } +} + func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) { ctx := context.Background() kv := newMemKV() @@ -307,6 +327,8 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) state, said string }{ {name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"}, + {name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x", Warnings: []string{"g14: " + delivery.ComposeNoAnswer}}, + state: TryError, said: "g14: " + delivery.ComposeNoAnswer}, {name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"}, state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"}, } { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 754f3b94..2c98c41a 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -217,7 +217,7 @@ func main() { ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...) tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, ask, time.Now)...) - tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, ask, time.Now)...) + tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) os.Exit(1) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 86ba8676..9bf29c33 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -343,7 +343,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf) // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf) - ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf) + ix := &index.Indexer{TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), @@ -511,9 +512,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } // The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a // try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates. - if err := ix.RecoverTries(ctx); err != nil { - logger.Printf("%v", err) - } + // A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same. + _ = ix.RecoverTries(ctx) if perr == nil { pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf) if err != nil { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go index 7a40aabb..fc272890 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/takein.go @@ -20,6 +20,7 @@ type takerIn interface { DropPending(ctx context.Context, key, by, why string) index.TakenIn TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn + RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn } // takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the @@ -38,7 +39,10 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream return } asked, r := "take-in", index.TakenIn{} - if a.TryCandidate != "" && a.Drop { + if a.RecoverTries { + asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why) + a.Module = "tries" + } else if a.TryCandidate != "" && a.Drop { asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why) a.Module = a.TryCandidate } else if a.Merge != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index b9ccd9e7..bbae2758 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -136,12 +136,15 @@ type Indexer struct { // Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil // takes every merge in at once, as before the take-in step. Pending Store - // SayTakeIn publishes a take-in event on the indexer's own stream take-in (ADR 0633 §5); nil says none. + // SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none. SayTakeIn func(ctx context.Context, t takein.TakeIn) error // OpenTries lists the tries that wait or run, from the checker's tool (checker_tries); nil recovers none (trycandidates.go). OpenTries func(ctx context.Context) ([]tryevent.Open, error) - tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu - pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) + // TryRecovery keeps how the last recovery of the open tries went (trycandidates.go); nil keeps none. + TryRecovery Store + recoveryFailed atomic.Bool // the last recovery of the open tries failed (trycandidates.go) + tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu + pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) } func (ix *Indexer) logf(format string, args ...any) { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index f87f13fd..b7c1ba71 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -578,7 +578,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) named := map[string]bool{} for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", - "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries"} { + "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries", "TryRecovery"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index b49a883f..e8811e78 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -100,7 +100,7 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, // TryCandidateTools are the tools of a group try's candidate (issue 1921, trycandidates.go): indexer_try_candidates reads the // candidates the indexer holds for tries (the try, its heads, each member's modules and whether each has its fingerprint), // and indexer_drop_try_candidate drops one by an ask in takeins. candidates is the candidates state as the tools bundle reads it. -func TryCandidateTools(candidates Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { +func TryCandidateTools(candidates, recovery Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { return []stdio.Tool{ {Name: TryCandidatesTool, Description: "The candidates the indexer holds for group tries (a try of several repositories at exact commits, mesh-delivery-checker try): " + @@ -127,7 +127,40 @@ func TryCandidateTools(candidates Store, ask func(key string, value []byte) (Tak out = append(out, c) } } - return map[string]any{"tries": out}, nil + // How the recovery of the open tries went: loud when it is not done (since when, why). + said := map[string]any{"done": false, "why": "no recovery has been written yet"} + if v, _, found, err := recovery.Get(ctx, TryRecoveryKey); err == nil && found { + var r TryRecovery + if json.Unmarshal(v, &r) == nil { + said = map[string]any{"done": r.Done} + if !r.Done { + said["why"], said["since"] = r.Why, r.Since + said["says"] = "recovery not done, since " + r.Since.Format(time.RFC3339) + ": " + r.Why + } + } + } + return map[string]any{"tries": out, "recovery": said}, nil + }}, + {Name: RecoverTriesTool, + Description: "Run the recovery of the open tries now: the indexer lists them through the checker's checker_tries, writes the candidate of each " + + "whole try that waits or runs and deletes the candidate of a try no longer listed. It runs by itself at the indexer's start; when it failed " + + "(indexer_try_candidates says recovery not done, since when and why) this runs it again. A hand act: who and why are said in the journal.", + Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{ + "why": map[string]any{"type": "string", "description": "why it is run now, in plain words"}}}, + Run: func(args map[string]any) (any, error) { + why, _ := args["why"].(string) + if strings.TrimSpace(why) == "" { + return nil, fmt.Errorf("say why: a hand act is said in the journal") + } + data, err := json.Marshal(TakeInAsk{V: 1, RecoverTries: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()}) + if err != nil { + return nil, err + } + a, err := ask("tries.recover."+now().UTC().Format("20060102T150405.000000000"), data) + if err != nil { + return nil, err + } + return a.Result, nil }}, {Name: DropTryCandidateTool, Description: "Drop a group try's candidate by hand (the indexer deletes it itself when the try ends): for a candidate left for a try that " + diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 701fdc8f..6dfe4310 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -3,6 +3,7 @@ package index import ( "context" "encoding/json" + "errors" "os" "slices" "strings" @@ -285,14 +286,16 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW a.Result = &r return a, err } - tools := TryCandidateTools(cands, asks, time.Now) + recovery := newMem() + ix.TryRecovery = recovery + tools := TryCandidateTools(cands, recovery, asks, time.Now) if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 { t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err) } - if _, err := tools[1].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil { + if _, err := tools[2].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil { t.Error("a branch was dropped as a try") } - if out, err := tools[1].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { + if out, err := tools[2].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { t.Errorf("drop = %v, %v, candidates %v", out, err, held()) } // The start recovered from the checker's tool: an open whole try is written again, one the checker no longer @@ -308,6 +311,46 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 { t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held()) } + // A recovery whose call fails is loud and deletes nothing: the candidate stays, indexer_try_candidates says recovery not + // done since when and why, and the next try event runs the recovery again, which clears it; the tool runs it by hand. + if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 { + t.Fatalf("started: %v, candidates %v", err, held()) + } + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { + return nil, errors.New("checker_tries does not answer") + } + if err := ix.RecoverTries(ctx); err == nil || len(held()) != 1 { + t.Fatalf("a failed call: %v, candidates %v, want the error and the candidate kept", err, held()) + } + if out, err := tools[0].Run(map[string]any{}); err != nil || !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "recovery not done, since") || + !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "checker_tries does not answer") { + t.Errorf("%s = %v, %v, want recovery not done with since and why", tools[0].Name, out, err) + } + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil } + if err := ix.StartTry(ctx, "try-11", members); err != nil || len(held()) != 1 || held()[0] != "try-11" { + t.Fatalf("a try event after a failed recovery: %v, candidates %v, want the recovery run (try-7 not listed, gone) and try-11 written", err, held()) + } + if out, _ := tools[0].Run(map[string]any{}); out.(map[string]any)["recovery"].(map[string]any)["done"] != true { + t.Errorf("recovery still said not done after a success: %v", out) + } + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, errors.New("down") } + recoverAsk := func(key string, value []byte) (TakeInAsk, error) { + a, err := DecodeTakeIn(value) + r := ix.RecoverTriesByHand(ctx, a.By, a.Why) + a.Result = &r + return a, err + } + byHand := TryCandidateTools(cands, recovery, recoverAsk, time.Now) + if out, err := byHand[1].Run(map[string]any{"why": "the checker was replaced"}); err != nil || out.(*TakenIn).Stored || !strings.Contains(out.(*TakenIn).Answer, "recovery not done") { + t.Errorf("%s with the checker down = %v, %v", byHand[1].Name, out, err) + } + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil } + if out, err := byHand[1].Run(map[string]any{"why": "the checker is back"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 { + t.Errorf("%s = %v, %v, candidates %v, want it recovered (try-11 not listed)", byHand[1].Name, out, err, held()) + } + if err := ix.StartTry(ctx, "try-7", members); err != nil { + t.Fatal(err) + } // ADR 0638 §3, §4: the consumer exists before the call, so what happens between the call and the hearing is heard after // it. A try the call listed and whose start is then heard gets one candidate; a try started after the list gets one; a // try that ended during the call, listed as open, leaves none once its end is heard. diff --git a/modules/mesh-delivery-indexer/internal/index/takein.go b/modules/mesh-delivery-indexer/internal/index/takein.go index 55ec09a2..e6318147 100644 --- a/modules/mesh-delivery-indexer/internal/index/takein.go +++ b/modules/mesh-delivery-indexer/internal/index/takein.go @@ -44,7 +44,9 @@ type TakeInAsk struct { MergeRepository string `json:"merge-repository,omitempty"` Sequence uint64 `json:"sequence,omitempty"` // TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go. - TryCandidate string `json:"try-candidate,omitempty"` + TryCandidate string `json:"try-candidate,omitempty"` + // RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries). + RecoverTries bool `json:"recover-tries,omitempty"` Drop bool `json:"drop,omitempty"` By string `json:"by,omitempty"` At time.Time `json:"at"` @@ -64,7 +66,7 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) { if a.V != 1 { return a, fmt.Errorf("a takeins value of version %d, not 1", a.V) } - if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" { + if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries { return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try") } return a, nil diff --git a/modules/mesh-delivery-indexer/internal/index/trycandidates.go b/modules/mesh-delivery-indexer/internal/index/trycandidates.go index ae2b0fb7..75a71b51 100644 --- a/modules/mesh-delivery-indexer/internal/index/trycandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/trycandidates.go @@ -11,22 +11,41 @@ package index import ( "context" + "encoding/json" + "errors" "fmt" "path" "regexp" "slices" "strings" + "time" "git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg" "git.novox.be/novox/mesh-sdk/go/delivery" ) -// TryCandidateTools: the tools that read and drop a try's candidate. +// TryCandidateTools: the tools that read, drop and recover a try's candidate. const ( TryCandidatesTool = "indexer_try_candidates" DropTryCandidateTool = "indexer_drop_try_candidate" + RecoverTriesTool = "indexer_recover_tries" + // TryRecoveryState is the indexer's state of how the last recovery of the open tries went: the bucket + // mesh-delivery-indexer_try-recovery, one key (TryRecoveryKey). + TryRecoveryState = "try-recovery" + TryRecoveryBucket = delivery.IndexerModule + "_" + TryRecoveryState + TryRecoveryKey = "state" ) +// TryRecovery is the value of the try-recovery state: whether the candidates of the open tries were recovered at the +// indexer's start or by hand, and when it began to fail, why. +type TryRecovery struct { + V int `json:"v"` + Done bool `json:"done"` + Since time.Time `json:"since,omitzero"` + Why string `json:"why,omitempty"` + At time.Time `json:"at"` +} + var tryGroupOK = regexp.MustCompile(`^try-[0-9]{1,20}$`) // IsTryGroup says a candidates key is the group of a try. @@ -38,6 +57,18 @@ type TryMember struct{ Repository, Head, Trunk string } // StartTry takes a whole try in, as the checker said it started (tryevent.Started): its members at their commits are kept, and // its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596). func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error { + ix.retryRecovery(ctx) + return ix.startTry(ctx, id, members) +} + +// retryRecovery runs the recovery again on a try event when the last one failed: the failure stays said until one succeeds. +func (ix *Indexer) retryRecovery(ctx context.Context) { + if ix.recoveryFailed.Load() && ix.OpenTries != nil { + _ = ix.RecoverTries(ctx) + } +} + +func (ix *Indexer) startTry(ctx context.Context, id string, members []TryMember) error { if !IsTryGroup(id) || len(members) == 0 { return fmt.Errorf("%q with %d members is no try to start", id, len(members)) } @@ -52,6 +83,11 @@ func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) // EndTry takes a try out, as the checker said it ended (tryevent.Ended), and deletes its candidate. func (ix *Indexer) EndTry(ctx context.Context, id string) error { + ix.retryRecovery(ctx) + return ix.endTry(ctx, id) +} + +func (ix *Indexer) endTry(ctx context.Context, id string) error { ix.mu.Lock() delete(ix.tries, id) ix.mu.Unlock() @@ -64,10 +100,76 @@ func (ix *Indexer) EndTry(ctx context.Context, id string) error { // RecoverTries is, at the indexer's start, what the events it missed would have said: the candidates of the tries that wait // or run now are written, and the candidate of a try the checker no longer lists is deleted. The checker's tool lists them // (OpenTries); its bucket is not read. Nothing to recover from when no part is wired to ask. +// +// A recovery that fails is said loudly and kept (the try-recovery state, read by indexer_try_candidates) until one succeeds: +// the events are still heard after it, and indexer_recover_tries runs it again by hand. func (ix *Indexer) RecoverTries(ctx context.Context) error { if ix.OpenTries == nil || ix.Store == nil { return nil } + err := ix.recoverTries(ctx) + ix.recoveryFailed.Store(err != nil) + ix.noteRecovery(ctx, err) + return err +} + +// noteRecovery writes how a recovery went: failing keeps the time it began to fail, and the journal says it each time. +func (ix *Indexer) noteRecovery(ctx context.Context, err error) { + now := ix.now() + r := TryRecovery{V: 1, Done: err == nil, At: now.UTC()} + if err != nil { + r.Why, r.Since = err.Error(), now.UTC() + ix.logf("RECOVERY OF THE TRIES NOT DONE: %v (the try candidates may be stale or missing until indexer_recover_tries or the next start succeeds)", err) + } + if ix.TryRecovery == nil { + return + } + for range writeTries { + cur, rev, found, gerr := ix.TryRecovery.Get(ctx, TryRecoveryKey) + if gerr != nil { + ix.logf("the try-recovery state cannot be read: %v", gerr) + return + } + if err != nil && found { + var was TryRecovery + if json.Unmarshal(cur, &was) == nil && !was.Done && !was.Since.IsZero() { + r.Since = was.Since + } + } + data, _ := json.Marshal(r) + var werr error + if found { + werr = ix.TryRecovery.Update(ctx, TryRecoveryKey, data, rev) + } else { + werr = ix.TryRecovery.Create(ctx, TryRecoveryKey, data) + } + if werr == nil { + return + } + if !errors.Is(werr, ErrConflict) { + ix.logf("the try-recovery state cannot be written: %v", werr) + return + } + } +} + +// RecoverTriesByHand runs the recovery now (indexer_recover_tries), a hand act said in the journal. +func (ix *Indexer) RecoverTriesByHand(ctx context.Context, by, why string) TakenIn { + r := TakenIn{Module: "tries"} + ix.logf("recovery of the tries asked by %s (%s)", by, why) + if ix.OpenTries == nil { + r.Answer = "the indexer is wired to list no open try: nothing to recover from" + return r + } + if err := ix.RecoverTries(ctx); err != nil { + r.Answer = "recovery not done: " + err.Error() + return r + } + r.Stored, r.Answer = true, "the candidates of the open tries are recovered" + return r +} + +func (ix *Indexer) recoverTries(ctx context.Context) error { open, err := ix.OpenTries(ctx) if err != nil { return fmt.Errorf("the open tries cannot be listed, so a missed try event is not recovered: %w", err) @@ -82,7 +184,7 @@ func (ix *Indexer) RecoverTries(ctx context.Context) error { for _, m := range o.Members { ms = append(ms, TryMember{m.Repository, m.Head, m.Trunk}) } - if err := ix.StartTry(ctx, o.ID, ms); err != nil { + if err := ix.startTry(ctx, o.ID, ms); err != nil { return err } } @@ -92,7 +194,7 @@ func (ix *Indexer) RecoverTries(ctx context.Context) error { } for _, k := range keys { if IsTryGroup(k) && !live[k] { - if err := ix.EndTry(ctx, k); err != nil { + if err := ix.endTry(ctx, k); err != nil { return err } } diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index db8524e0..731bfa52 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -108,7 +108,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want manifest-refused, manifest-accepted and the two take-in events", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, {Name: graph.State, History: graph.History}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } @@ -137,7 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build // (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections). // 1585/T12, T13 (ADR 0633 §5): indexer_pending reads the pending state; indexer_take_in_pending and indexer_drop_pending // ask through takeins; indexer_take_in_merge takes one merge in, its message asked of the forge bridge (issue 1585). - if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.TakeInMergeTool}; !reflect.DeepEqual(m.Tools, want) { + if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool}; !reflect.DeepEqual(m.Tools, want) { t.Errorf("tools = %v, want %v", m.Tools, want) } // ADR 0604 §2: it holds the seat mesh-catalog, serving its four verbs from its tools bundle (which stays without a diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index 4b39a52c..eabac22c 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -24,6 +24,7 @@ "indexer_drop_pending", "indexer_try_candidates", "indexer_drop_try_candidate", + "indexer_recover_tries", "indexer_take_in_merge" ], "consumes": [ @@ -45,6 +46,7 @@ { "name": "takeins", "history": 1 }, { "name": "verdicts", "history": 5 }, { "name": "oldsections", "history": 1 }, + { "name": "try-recovery", "history": 1 }, { "name": "packages", "history": 5 }, { "name": "graph", "history": 5 }, { "name": "pending", "history": 1 } From 88a47be251238650dea9db423f9e248929f759fa Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:47:31 +0100 Subject: [PATCH 08/13] A try says its end with the state it ended in, kept in the try's own state, so an end after a checker restart is still said; only the checker publishes try events (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../internal/check/try.go | 62 ++++++--- .../internal/check/try_test.go | 122 ++++++++++++------ .../mesh-delivery-checker/manifest_test.go | 26 ++++ 3 files changed, 151 insertions(+), 59 deletions(-) diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index e794ebc8..b2afc02c 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -81,6 +81,9 @@ type Try struct { Tests string `json:"tests,omitempty"` Level string `json:"level,omitempty"` State string `json:"state"` + // Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the + // try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts. + Told bool `json:"told,omitempty"` // Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part. Said string `json:"said,omitempty"` // Took is the time from its start to its end, as Go writes a duration. @@ -519,7 +522,7 @@ type Tryer struct { Say func(ctx context.Context, event string, body tryevent.Event) mu sync.Mutex - told map[string]bool // whole tries whose start was said and whose end was not + ending map[string]bool // tries whose end is being said now wg sync.WaitGroup inflight map[string]context.CancelFunc seen map[string]bool // looked at since this process started: a try left running is taken over the first time @@ -558,12 +561,8 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) { if stop := tr.inflight[key]; stop != nil { stop() } - said := tr.told[key] - delete(tr.told, key) tr.mu.Unlock() - if said { // ended by hand while it ran: the indexer drops its candidate - tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) - } + tr.sendEnd(ctx, key) // ended by hand while it ran, or its end was written and not said: the indexer drops its candidate return } if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) { @@ -619,6 +618,7 @@ func (tr *Tryer) run(ctx context.Context, key string) { return } t.State, t.Started = TryRunning, tr.now().UTC() + t.Told = (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() // its start is said below; its end is owed from now on data, err := encodeTry(t) if err != nil { return @@ -630,17 +630,11 @@ func (tr *Tryer) run(ctx context.Context, key string) { return } progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev} - if (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() { + if t.Told { started := tryevent.Event{ID: key} for _, m := range t.MembersOf() { started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk}) } - tr.mu.Lock() - if tr.told == nil { - tr.told = map[string]bool{} - } - tr.told[key] = true - tr.mu.Unlock() tr.say(ctx, tryevent.Started, started) } cs := TryCase(key, t, progress.heard) @@ -653,13 +647,6 @@ func (tr *Tryer) run(ctx context.Context, key string) { return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end } t, rev = progress.end() - tr.mu.Lock() - said := tr.told[key] - delete(tr.told, key) - tr.mu.Unlock() - if said { - defer tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key}) - } v := out.Verdict t.Ended = tr.now().UTC() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() @@ -707,6 +694,41 @@ func (tr *Tryer) run(ctx context.Context, key string) { return } tr.logf("%s: %s", key, t.State) + tr.sendEnd(ctx, key) +} + +// sendEnd says the end of an ended try whose start was said (Told) with the state it ended in, then clears Told in the try's +// own state; a checker that stopped between the two says it again at its start. Every end is said so: pass, fail, error and +// by hand. +func (tr *Tryer) sendEnd(ctx context.Context, key string) { + tr.mu.Lock() + if tr.ending[key] { + tr.mu.Unlock() + return + } + if tr.ending == nil { + tr.ending = map[string]bool{} + } + tr.ending[key] = true + tr.mu.Unlock() + defer func() { tr.mu.Lock(); delete(tr.ending, key); tr.mu.Unlock() }() + raw, rev, found, err := tr.Tries.Get(ctx, key) + if err != nil || !found { + return + } + t, err := DecodeTry(raw) + if err != nil || !t.Over() || !t.Told { + return + } + tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) + t.Told = false + data, err := encodeTry(t) + if err == nil { + _, err = tr.Tries.Update(ctx, key, data, rev) + } + if err != nil && !errors.Is(err, ErrConflict) { + tr.logf("%s: its end was said and Told could not be cleared: %v", key, err) + } } // say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start). diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index 46133c57..9aabfd73 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "slices" "strings" "testing" "time" @@ -181,16 +182,19 @@ func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR written := &writtenTries{memKV: kv} var said []string tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { - said = append(said, fmt.Sprintf("%s %s %d", event, e.ID, len(e.Members))) + said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State)) }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() // The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate). - if len(said) != 2 || said[0] != "try.started "+id+" 2" || said[1] != "try.ended "+id+" 0" { + if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass { t.Errorf("events = %v, want the start with 2 members and the end", said) } got := readTry(t, kv, id) + if got.Told { // the end was said: nothing is owed + t.Errorf("the ended try still owes its end: %+v", got) + } if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 { t.Errorf("open tries after the end = %v, %v", open, err) } @@ -282,26 +286,24 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") { t.Errorf("try = %+v", got) } -} - -func TestATryOfOneLevelKeepsOnlyThatLevel(t *testing.T) { - ctx := context.Background() - kv := newMemKV() - id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now()) - b := newBench(t) - tr := &Tryer{Tries: kv, Judge: b.ex} - raw, _, _, _ := kv.Get(ctx, id) - tr.Put(ctx, id, raw) - tr.Wait() - keys, _ := b.kept.Keys(ctx) - if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 { - t.Fatalf("try = %+v, kept %v", got, keys) - } - for _, k := range keys { - if !strings.HasPrefix(k, delivery.LevelUnit+".") { - t.Errorf("a try of the unit level kept %s", k) + t.Run("a try of one level keeps only that level", func(t *testing.T) { + kv := newMemKV() + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now()) + b := newBench(t) + tr := &Tryer{Tries: kv, Judge: b.ex} + raw, _, _, _ := kv.Get(ctx, id) + tr.Put(ctx, id, raw) + tr.Wait() + keys, _ := b.kept.Keys(ctx) + if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 { + t.Fatalf("try = %+v, kept %v", got, keys) } - } + for _, k := range keys { + if !strings.HasPrefix(k, delivery.LevelUnit+".") { + t.Errorf("a try of the unit level kept %s", k) + } + } + }) } func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) { @@ -340,10 +342,18 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) comps.v.State = delivery.CompositionPass } id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) - tr := &Tryer{Tries: kv, Judge: b.ex} + var ends []string + tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { + if event == tryevent.Ended { + ends = append(ends, e.State) + } + }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) tr.Wait() + if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail + t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state) + } if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) { t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said) } @@ -353,22 +363,56 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) func TestATryLeftRunningIsTakenOver(t *testing.T) { ctx := context.Background() - kv := newMemKV() - id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) - // Left running by a checker that stopped. - raw, rev, _, _ := kv.Get(ctx, id) - cur, _ := DecodeTry(raw) - cur.State = TryRunning - data, _ := encodeTry(cur) - if _, err := kv.Update(ctx, id, data, rev); err != nil { - t.Fatal(err) - } - b := newBench(t) - wholeBench(b, delivery.Candidate{}) - tr := &Tryer{Tries: kv, Judge: b.ex} - tr.Reconsider(ctx) - tr.Wait() - if got := readTry(t, kv, id); got.State != TryPass { - t.Errorf("the try left running = %+v, want taken over and passed", got) + // Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did + // not start, whatever way the try ended. + for _, row := range []struct { + name string + told bool + after func(t *testing.T, kv *memKV, id string) + ends []string + }{ + {name: "left running after its start was said", told: true, ends: []string{TryPass}}, + {name: "left running with no start said", told: false, ends: []string{TryPass}}, + {name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) { + if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil { + t.Fatal(err) + } + }}, + } { + t.Run(row.name, func(t *testing.T) { + kv := newMemKV() + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) + // Left running by a checker that stopped. + raw, rev, _, _ := kv.Get(ctx, id) + cur, _ := DecodeTry(raw) + cur.State, cur.Told = TryRunning, row.told + data, _ := encodeTry(cur) + if _, err := kv.Update(ctx, id, data, rev); err != nil { + t.Fatal(err) + } + if row.after != nil { + row.after(t, kv, id) + } + b := newBench(t) + wholeBench(b, delivery.Candidate{}) + var ends []string + tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { + if event == tryevent.Ended { + ends = append(ends, e.State) + } + }} + tr.Reconsider(ctx) + tr.Wait() + tr.Reconsider(ctx) // said once: the second look finds nothing owed + tr.Wait() + if row.after == nil { + if got := readTry(t, kv, id); got.State != TryPass { + t.Errorf("the try left running = %+v, want taken over and passed", got) + } + } + if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told { + t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends) + } + }) } } diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index 276a1a89..1f71c257 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -134,6 +134,32 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) { t.Errorf("streams = %+v, want %+v", m.Streams, want) } + // Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other + // module of the repository declares a stream carrying a try event (the indexer and the others only consume them). + siblings, err := filepath.Glob("../*/module.json") + if err != nil || len(siblings) < 2 { + t.Fatalf("the sibling manifests = %v, %v", siblings, err) + } + for _, f := range siblings { + raw, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + var other struct { + Module string `json:"module"` + Streams []stream `json:"streams"` + } + if err := json.Unmarshal(raw, &other); err != nil { + t.Fatalf("%s: %v", f, err) + } + for _, st := range other.Streams { + for _, e := range st.Events { + if strings.HasPrefix(e, "try.") && other.Module != m.Module { + t.Errorf("%s declares the stream %s with %s: only the checker's account publishes try events", other.Module, st.Name, e) + } + } + } + } // The read tools (issue 1686) are served by the tools bundle, not by the checker's process. // The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check // is asked again by hand (checker_ask_again). From 69a2790099515b39ea73585c1d89111c59487d92 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 18:47:31 +0100 Subject: [PATCH 09/13] =?UTF-8?q?The=20indexer=20fixes=20its=20own=20build?= =?UTF-8?q?=20once=20at=20its=20start=20and=20never=20learns=20it=20later;?= =?UTF-8?q?=20a=20merge=20held=20for=20an=20unknown=20own=20build=20is=20s?= =?UTF-8?q?aid=20pending=20with=20why=20and=20since=20(issue=201921,=20ADR?= =?UTF-8?q?=200633=20=C2=A71)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-indexer-tools/main.go | 2 +- .../mesh-delivery-indexer/integration_test.go | 2 +- .../cmd/mesh-delivery-indexer/main.go | 7 +- .../internal/index/candidates.go | 17 +++-- .../internal/index/pending.go | 28 +++++--- .../internal/index/pending_test.go | 61 +++++++++++------ .../internal/index/pending_tools.go | 17 ++++- .../internal/index/verdicts.go | 66 +++++++++++++++---- .../internal/index/verdicts_test.go | 14 ++-- .../mesh-delivery-indexer/manifest_test.go | 2 +- modules/mesh-delivery-indexer/module.json | 1 + .../mesh-delivery-indexer/takein/takein.go | 11 +++- .../takein/takein_test.go | 1 + .../internal/watch/findings_test.go | 18 +++++ .../internal/watch/takein.go | 3 + 15 files changed, 189 insertions(+), 61 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go index 2c98c41a..6bce7560 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer-tools/main.go @@ -216,7 +216,7 @@ var errReadOnly = errors.New("the tools bundle only reads this state") func main() { ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second) tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...) - tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, ask, time.Now)...) + tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, ask, time.Now)...) tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...) if err := stdio.Serve("", tools); err != nil { fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 023478a4..e7feb4a1 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -283,7 +283,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15 } waits := func() []takein.PendingWait { t.Helper() - answer, err := index.PendingTools(kvStore{pendingKV}, nil, time.Now)[0].Run(map[string]any{}) + answer, err := index.PendingTools(kvStore{pendingKV}, nil, nil, time.Now)[0].Run(map[string]any{}) if err != nil { t.Fatal(err) } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index 9bf29c33..5445dc48 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -343,8 +343,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf) // The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written. graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf) + ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf) tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf) - ix := &index.Indexer{TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, + ix := &index.Indexer{OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known, Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf), Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf, Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call), @@ -354,6 +355,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // until then every merge is taken in at once, as before. Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)}, SayTakeIn: sayTakeIn(js, cred)} + // Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the + // node-engine names the successor while it replaces this process, and an old process that learnt it later would take + // revisions in with its old judge. + ix.FixOwnBuild(ctx) // The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each // new one indexes the module it names when that module is registered and the bucket holds none (registered.go). var started atomic.Bool diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index bbae2758..137609cf 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -122,12 +122,17 @@ type Indexer struct { fillAgain atomic.Bool // a kick arrived while it ran atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished) // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. - OwnBuild func(ctx context.Context) (string, error) - underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) - judgeMu sync.Mutex // one judge pass at a time (verdicts.go) - ownMu sync.Mutex // guards ownFP - ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild) - judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) + OwnBuild func(ctx context.Context) (string, error) + underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) + judgeMu sync.Mutex // one judge pass at a time (verdicts.go) + ownMu sync.Mutex // guards ownFP + ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild) + ownFixed bool // FixOwnBuild asked already: it asks once + ownSince time.Time // when the build was asked and not known, with ownWhy + ownWhy string + // OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none. + OwnBuildState Store + judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go) // Message asks the forge bridge for the merged message of a merge, repository and merge commit in, the message whole out (a // pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none. Message func(ctx context.Context, repository, merge string) ([]byte, error) diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index 39189ff2..cd34f376 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -339,20 +339,27 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool { return !errors.Is(err, process.ErrNotYet) } -// ownBuildNow is the build this program runs: the one learnt, else asked of the node-engine now. -func (ix *Indexer) ownBuildNow(ctx context.Context) string { - ix.ownMu.Lock() - fp := ix.ownFP - ix.ownMu.Unlock() - if fp != "" || ix.OwnBuild == nil { - return fp - } - ix.learnOwnBuild(ctx) +// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again. +func (ix *Indexer) ownBuildNow(context.Context) string { ix.ownMu.Lock() defer ix.ownMu.Unlock() return ix.ownFP } +// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a +// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to +// pending; the stream delivers it again to the process that knows its build. +func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) { + ix.ownMu.Lock() + since, why := ix.ownSince, ix.ownWhy + ix.ownMu.Unlock() + if since.IsZero() { // never fixed: the start did not run (no engine wired) + since, why = ix.now(), "the build this program runs was never asked" + } + ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision, + Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why}) +} + // sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of // each wait from the state. A publish that fails is said in the log: the next evaluation says it again. func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool { @@ -401,6 +408,7 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap } own := ix.ownBuildNow(ctx) if own == "" { + ix.sayOwnBuildUnknown(ctx, seq, f) return Applied{}, "", ErrOwnBuildUnknown } ix.pendingMu.Lock() @@ -691,7 +699,7 @@ func (ix *Indexer) dry() *dryRun { d := &dryRun{} x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum, GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains, - ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, ReadFile: ix.ReadFile, underSeats: true} + ReadOpen: nil, OwnBuild: nil, OwnBuildState: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, ReadFile: ix.ReadFile, underSeats: true} x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) } x.Store, d.candidates = over(ix.Store) x.Modules, d.modules = over(ix.Modules) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index b7c1ba71..7f787eb7 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -61,6 +61,8 @@ func newTakeInBed(t *testing.T) *takeInBed { t.Fatalf("the bed's modules have no fingerprint") } ix.OwnBuild = func(context.Context) (string, error) { return own, nil } + ix.OwnBuildState = newMem() + ix.FixOwnBuild(context.Background()) // fixed once, at the start return b } @@ -131,11 +133,26 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) ctx := context.Background() t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) { b := newTakeInBed(t) + b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") } + b.ix.FixOwnBuild(ctx) + b.said = nil _, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", "")) if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') { t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi")) } + // The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the + // build is unknown, from the own-build state. + if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") || + b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" { + t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said) + } else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil { + t.Errorf("the unknown-build event is refused by its own contract: %v", err) + } + answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}) + if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() { + t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err) + } // Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once, // as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever). b.ix.Pending = &memStore{unbound: process.ErrNotYet} @@ -254,31 +271,36 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) g.Members = []delivery.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}} b.arrive(t, 5, g) named := b.fp(t, delivery.IndexerModule) - // The new indexer starts; its build is not known at first: a hand ask is answered meanwhile (R5), and the start waits. - next := &Indexer{} - *next = Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, - Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum} - next.toolchains = b.ix.toolchains - asked := 0 - next.OwnBuild = func(context.Context) (string, error) { - if asked++; asked < 3 { // the hand ask's and the first start's - return "", errors.New("not yet") - } - return named, nil + // The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered + // meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again. + mk := func(ask func(context.Context) (string, error)) *Indexer { + n := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts, + Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum} + n.toolchains = b.ix.toolchains + n.OwnBuild = ask + n.FixOwnBuild(ctx) + return n } + asked := 0 + next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") }) if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { t.Errorf("a hand ask while the own build is unknown: %+v", r) } - // The first start, its build not known, waits for nothing (no sleep, no timer): it answers at once, takes nothing - // in, and the forge consumer starts after it. - if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 2 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { - t.Fatalf("a start with its build unknown: %q %v, asked %d, pending %v: want it to answer at once and take nothing in", words, err, asked, b.keys()) + if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") { + t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys()) } + // The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start. + next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil } + if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") { + t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys()) + } + // The module started again fixes the build at its start and takes the revision in. + next = mk(func(context.Context) (string, error) { return named, nil }) if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) } - if asked < 3 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { - t.Errorf("start: asked %d, pending %v, avahi at %s: want the revision taken in, the group left", asked, b.keys(), b.commit(t, "avahi")) + if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') { + t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi")) } }) t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) { @@ -452,7 +474,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")) b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2")) var answer takein.PendingAnswer - for _, tool := range PendingTools(b.pending, nil, nil) { + for _, tool := range PendingTools(b.pending, nil, nil, nil) { if tool.Name == PendingTool { out, err := tool.Run(map[string]any{}) if err != nil { @@ -522,6 +544,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: bridgeDown, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }} next.toolchains = b.ix.toolchains + next.FixOwnBuild(ctx) b.said = nil if _, _, err := next.TakeInAtStart(ctx); err != nil { t.Fatal(err) @@ -578,7 +601,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) named := map[string]bool{} for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections", "JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk", - "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries", "TryRecovery"} { + "Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries", "TryRecovery", "OwnBuildState"} { named[f] = true } ty := reflect.TypeOf(Indexer{}) diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index e8811e78..733cb70a 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -16,7 +16,7 @@ import ( // PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the // pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in // does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer. -func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { +func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool { str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s } hand := func(drop bool) func(map[string]any) (any, error) { return func(args map[string]any) (any, error) { @@ -53,7 +53,8 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + - "indexer_take_in_pending and indexer_drop_pending steer a wait.", + "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build-unknown, when set, says the indexer cannot learn the build it " + + "runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { var unread []string @@ -63,7 +64,17 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, if err != nil { return nil, err } - return takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}, nil + answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread} + // Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why. + if ownBuild != nil { + if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { + var o OwnBuild + if json.Unmarshal(v, &o) == nil && !o.Known { + answer.OwnBuildUnknown, answer.OwnBuildSince = o.Why, o.Since + } + } + } + return answer, nil }}, {Name: TakeInPendingTool, Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " + diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 94f65448..1758f740 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -211,7 +211,6 @@ func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, ove // digest of the verdicts state that another judge refused or held waiting (one of a candidate, or one back on the trunk // later) is judged again by this judge (judgeStale), so a refusal an older judge made never stands for good. func (ix *Indexer) JudgeAtStart(ctx context.Context) (Judged, error) { - ix.learnOwnBuild(ctx) j, err := ix.JudgeAll(ctx, nil) ix.judgePending.Store(err != nil) if err == nil { @@ -355,21 +354,66 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v delivery.M return wrote, err } -// learnOwnBuild asks where the indexer's program is told which build it runs (OwnBuild: the node-engine's seat, never the -// trunk's value of the indexer, which names a build that may be merged and not delivered). Asked at the start pass; where -// it is not answered the build stays unknown and every unknown field waits, which is the safe side. -func (ix *Indexer) learnOwnBuild(ctx context.Context) { - if ix.OwnBuild == nil { +// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey): +// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it +// does not, since when and why. +const ( + OwnBuildStateName = "own-build" + OwnBuildBucket = delivery.IndexerModule + "_" + OwnBuildStateName + OwnBuildKey = "state" +) + +// OwnBuild is the value of the own-build state. +type OwnBuild struct { + V int `json:"v"` + Known bool `json:"known"` + FP string `json:"fp,omitempty"` + Since time.Time `json:"since"` + Why string `json:"why,omitempty"` +} + +// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's +// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before +// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the +// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not +// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the +// own-build state, indexer_pending), and the module is started again to learn it. +func (ix *Indexer) FixOwnBuild(ctx context.Context) { + ix.ownMu.Lock() + if ix.ownFixed || ix.OwnBuild == nil { + ix.ownMu.Unlock() return } + ix.ownFixed = true + ix.ownMu.Unlock() fp, err := ix.OwnBuild(ctx) - if err != nil || fp == "" { - ix.logf("judge: the build this program runs is not known (%v), so a manifest with a field it does not know waits and is not refused", err) - return + if err == nil && fp == "" { + err = errors.New("the node-engine names no build of the indexer") + } + st := OwnBuild{V: 1, Since: ix.now().UTC()} + if err != nil { + st.Why = err.Error() + ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err) + } else { + st.Known, st.FP = true, fp } ix.ownMu.Lock() - ix.ownFP = fp + ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why ix.ownMu.Unlock() + if ix.OwnBuildState == nil { + return + } + data, _ := json.Marshal(st) + if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found { + err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev) + } else if gerr == nil { + err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data) + } else { + err = gerr + } + if err != nil { + ix.logf("the own-build state cannot be written: %v", err) + } } // ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the @@ -510,8 +554,6 @@ func (ix *Indexer) JudgeAgain(ctx context.Context, module, digest string) TakenI out.Answer = "the modules state cannot be read: " + err.Error() return out } - // The build this program runs is asked again, so a start pass that missed the engine's answer is recovered without a restart. - ix.learnOwnBuild(ctx) if digest != "" { // One digest (a candidate's, or one the trunk no longer names), judged by this judge in the catalogue the trunk holds. value, _, found, err := ix.Manifests.Get(ctx, digest) diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go index 6dd718a7..781bc576 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts_test.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts_test.go @@ -273,12 +273,16 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) { if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module { t.Fatalf("indexer_verdicts, all: %d rows of %d verdicts", len(rows), len(keys)) } - // indexer_judge asks the engine for the running build again: a start pass that missed it is recovered (the trunk's value of - // the indexer is the newer one, so the field waits; asked and answered with that same build it is refused). - ix.ownFP = "" + // The build is fixed once, at the start, and never learnt later: this program started when the engine could not answer, and + // the engine now names the trunk's newest build (as it does while it replaces the indexer by a successor). indexer_judge + // does not take it for this program's own, so the field still waits and the build stays unknown. + ix.ownFP, ix.ownFixed = "", true ix.OwnBuild = func(context.Context) (string, error) { return "src2:" + strings.Repeat("a", 64), nil } - if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "refused") || ix.ownFP == "" { - t.Fatalf("indexer_judge did not learn the running build: %+v", got) + if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || ix.ownFP != "" { + t.Fatalf("indexer_judge learnt the build the engine names now: %+v (own %q)", got, ix.ownFP) + } + if _, err := ix.JudgeAtStart(ctx); err != nil || ix.ownFP != "" || !verdict(later).Waits() { + t.Fatalf("a judge pass learnt the build: %v %q %+v", err, ix.ownFP, verdict(later)) } ix.ownFP = ownFP ix.OwnBuild = nil diff --git a/modules/mesh-delivery-indexer/manifest_test.go b/modules/mesh-delivery-indexer/manifest_test.go index 731bfa52..3e536b66 100644 --- a/modules/mesh-delivery-indexer/manifest_test.go +++ b/modules/mesh-delivery-indexer/manifest_test.go @@ -108,7 +108,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build t.Errorf("emits = %v, want manifest-refused, manifest-accepted and the two take-in events", m.Emits) } // 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard. - if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, + if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory}, {Name: graph.State, History: graph.History}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } diff --git a/modules/mesh-delivery-indexer/module.json b/modules/mesh-delivery-indexer/module.json index eabac22c..1791056e 100644 --- a/modules/mesh-delivery-indexer/module.json +++ b/modules/mesh-delivery-indexer/module.json @@ -47,6 +47,7 @@ { "name": "verdicts", "history": 5 }, { "name": "oldsections", "history": 1 }, { "name": "try-recovery", "history": 1 }, + { "name": "own-build", "history": 1 }, { "name": "packages", "history": 5 }, { "name": "graph", "history": 5 }, { "name": "pending", "history": 1 } diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index 3e76e4ba..af9ba44f 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -140,6 +140,9 @@ type TakeIn struct { // the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for // an indexer whose manifest is accepted, and says why here, as it says what it lost. Refused string `json:"refused,omitempty"` + // Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build + // it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown. + Unknown string `json:"own-build-unknown,omitempty"` } func (t TakeIn) check() error { @@ -168,8 +171,8 @@ func (t TakeIn) check() error { if !strings.HasPrefix(t.Key, "m.") { return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m., not %q", t.Key) } - if t.Kind == TakeInPending && t.WaitsFor == "" { - return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for", t.Key) + if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" { + return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, or why the indexer does not know its own", t.Key) } case TakeInGroup: if t.Group == "" { @@ -247,6 +250,10 @@ type PendingWait struct { type PendingAnswer struct { Waits []PendingWait `json:"waits"` Unread []string `json:"unread,omitempty"` + // OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits + // (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known. + OwnBuildUnknown string `json:"own-build-unknown,omitempty"` + OwnBuildSince time.Time `json:"own-build-since,omitzero"` } // DecodePendingAnswer reads indexer_pending's answer, refusing a wait that is not a revision or a group, or names no diff --git a/modules/mesh-delivery-indexer/takein/takein_test.go b/modules/mesh-delivery-indexer/takein/takein_test.go index f9a5d6cd..a4d8b53b 100644 --- a/modules/mesh-delivery-indexer/takein/takein_test.go +++ b/modules/mesh-delivery-indexer/takein/takein_test.go @@ -20,6 +20,7 @@ func sampleTakeIns(since time.Time) []TakeIn { Members: []delivery.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since}, {Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since, By: "lead-47@shanks", Why: "the named indexer can never run"}, + {Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build } } diff --git a/modules/mesh-delivery-watchman/internal/watch/findings_test.go b/modules/mesh-delivery-watchman/internal/watch/findings_test.go index e03c00ec..4ee4fc29 100644 --- a/modules/mesh-delivery-watchman/internal/watch/findings_test.go +++ b/modules/mesh-delivery-watchman/internal/watch/findings_test.go @@ -106,6 +106,24 @@ func TestTheWatchmansFindings(t *testing.T) { } expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute) }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, + {"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { + w.TakeInCaughtUp() + body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t, + Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"}) + if err != nil { + t.Fatal(err) + } + w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body) + if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 { + t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings) + } + expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound) + for _, f := range o.findings { + if !strings.Contains(f.Summary, "cannot learn the build it runs") { + t.Errorf("finding %q does not say why", f.Summary) + } + } + }, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}}, {"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) { w.TakeInCaughtUp() // a live message, heard after the replay w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0)) diff --git a/modules/mesh-delivery-watchman/internal/watch/takein.go b/modules/mesh-delivery-watchman/internal/watch/takein.go index 3c3465ff..171aa403 100644 --- a/modules/mesh-delivery-watchman/internal/watch/takein.go +++ b/modules/mesh-delivery-watchman/internal/watch/takein.go @@ -72,6 +72,9 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) { return } bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor + if t.Unknown != "" { + what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")" + } if t.What == takein.TakeInGroup { bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come" } From f9578cb7b41893e1c3b1e69738722a917c1d8425 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 19:24:45 +0100 Subject: [PATCH 10/13] A try's end is said again until the stream acknowledges it; a waiting try says why; a hand recovery never overlaps a try event (issue 1921, ADR 0639) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-checker/main.go | 7 +- .../internal/check/case.go | 2 + .../internal/check/judge.go | 4 + .../internal/check/try.go | 34 ++++++-- .../internal/check/try_test.go | 79 +++++++++++++++++-- .../internal/index/candidates.go | 1 + .../internal/index/previews_test.go | 17 ++++ .../internal/index/trycandidates.go | 15 +++- 8 files changed, 141 insertions(+), 18 deletions(-) diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 63bcfcae..d464578d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -192,11 +192,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro // The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries. tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf, // A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group. - Say: func(ctx context.Context, event string, body tryevent.Event) { + Say: func(ctx context.Context, event string, body tryevent.Event) error { data, _ := json.Marshal(body) - if _, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second)); err != nil { - logger.Printf("%s of %s could not be said: %v (the indexer finds the open tries at its start)", event, body.ID, err) - } + _, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second)) + return err }} announce := func(ctx context.Context, event, key string) { body, _ := json.Marshal(map[string]string{"key": key}) diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 3f80136f..050bd601 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -55,6 +55,8 @@ type Case struct { Watch func(askID string, step *delivery.Step) // Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at. Progress *Progress + // Wait says what a whole try waits for now ("" when nothing); the try keeps it for try-read. Nil for a pull request's check. + Wait func(why string) // Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the // build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5). Kind string diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index 267f40c0..391d9f27 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -283,6 +283,10 @@ func (st *state) takeTryCandidate(ctx context.Context) { no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)") return } + if st.c.Wait != nil { + st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")") + defer st.c.Wait("") + } cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait()) if err != nil { no(err.Error()) diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index b2afc02c..16b64403 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -84,6 +84,9 @@ type Try struct { // Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the // try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts. Told bool `json:"told,omitempty"` + // Waiting is what the running try waits for now (its group's candidate from the indexer, ADR 0639 §5), empty when it waits for nothing; + // try-read gives it, so a try that sits still says why. + Waiting string `json:"waiting,omitempty"` // Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part. Said string `json:"said,omitempty"` // Took is the time from its start to its end, as Go writes a duration. @@ -180,6 +183,17 @@ func (w *tryWriter) heard(askID string, step *delivery.Step) { } } +// waiting writes what the try waits for now ("" when it waits for nothing). +func (w *tryWriter) waiting(why string) { + w.mu.Lock() + defer w.mu.Unlock() + if w.stopped || w.t.Waiting == why { + return + } + w.t.Waiting = why + w.write() +} + // write writes the record at the revision written last; a refusal stops the writes. func (w *tryWriter) write() { data, err := encodeTry(w.t) @@ -519,7 +533,8 @@ type Tryer struct { // Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The // indexer, which hears them, writes and deletes the candidate of a whole try's group. - Say func(ctx context.Context, event string, body tryevent.Event) + // It answers the publish's error, nil only when the stream acknowledged it: an end not acknowledged is said again. + Say func(ctx context.Context, event string, body tryevent.Event) error mu sync.Mutex ending map[string]bool // tries whose end is being said now @@ -638,6 +653,7 @@ func (tr *Tryer) run(ctx context.Context, key string) { tr.say(ctx, tryevent.Started, started) } cs := TryCase(key, t, progress.heard) + cs.Wait = progress.waiting judging := ctx if tr.NotTaken > 0 { judging = buildrun.NotTakenBound(ctx, tr.NotTaken) @@ -647,6 +663,7 @@ func (tr *Tryer) run(ctx context.Context, key string) { return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end } t, rev = progress.end() + t.Waiting = "" v := out.Verdict t.Ended = tr.now().UTC() t.Took = t.Ended.Sub(t.Started).Round(time.Second).String() @@ -720,7 +737,9 @@ func (tr *Tryer) sendEnd(ctx context.Context, key string) { if err != nil || !t.Over() || !t.Told { return } - tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) + if tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) != nil { + return // not acknowledged: Told stays, and the next Put or Reconsider says the end again + } t.Told = false data, err := encodeTry(t) if err == nil { @@ -732,10 +751,15 @@ func (tr *Tryer) sendEnd(ctx context.Context, key string) { } // say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start). -func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) { - if tr.Say != nil { - tr.Say(ctx, event, body) +func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) error { + if tr.Say == nil { + return nil } + err := tr.Say(ctx, event, body) + if err != nil { + tr.logf("%s of %s could not be said: %v", event, body.ID, err) + } + return err } // OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index 9aabfd73..74a4681f 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -152,9 +152,14 @@ type fakeTryCandidates struct { cand delivery.Candidate err error asked []string + // during is called while the try waits for the candidate. + during func() } func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) { + if f.during != nil { + f.during() + } f.asked = append(f.asked, group+"."+heads) return f.cand, f.err } @@ -180,9 +185,12 @@ func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR b := newBench(t) comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"}) written := &writtenTries{memKV: kv} + var waiting string + cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for var said []string - tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { + tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error { said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State)) + return nil }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) @@ -191,7 +199,25 @@ func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass { t.Errorf("events = %v, want the start with 2 members and the end", said) } + // What the indexer reads: tryevent.Decode accepts the events as said and refuses a body that is no try's. + for _, row := range []struct { + name, event, body string + ok bool + }{ + {"a start with its members", tryevent.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true}, + {"an end with its state", tryevent.Ended, `{"id":"` + id + `","state":"pass"}`, true}, + {"an id that is no try's", tryevent.Ended, `{"id":"pull-7","state":"pass"}`, false}, + {"a start with no member", tryevent.Started, `{"id":"` + id + `"}`, false}, + {"a body that is no JSON", tryevent.Ended, `try-7`, false}, + } { + if e, err := tryevent.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) { + t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok) + } + } got := readTry(t, kv, id) + if !strings.Contains(waiting, "candidate") || got.Waiting != "" { + t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting) + } if got.Told { // the end was said: nothing is owed t.Errorf("the ended try still owes its end: %+v", got) } @@ -264,8 +290,9 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) { t.Fatal(err) } b := newBench(t) - tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) { + tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) error { t.Error("a try of one level and some tests said an event: it has no candidate") + return nil }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) @@ -343,10 +370,11 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) } id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) var ends []string - tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { + tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error { if event == tryevent.Ended { ends = append(ends, e.State) } + return nil }} raw, _, _, _ := kv.Get(ctx, id) tr.Put(ctx, id, raw) @@ -366,11 +394,13 @@ func TestATryLeftRunningIsTakenOver(t *testing.T) { // Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did // not start, whatever way the try ended. for _, row := range []struct { - name string - told bool - after func(t *testing.T, kv *memKV, id string) - ends []string + name string + told bool + failFirst bool // the stream does not acknowledge the first end said + after func(t *testing.T, kv *memKV, id string) + ends []string }{ + {name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}}, {name: "left running after its start was said", told: true, ends: []string{TryPass}}, {name: "left running with no start said", told: false, ends: []string{TryPass}}, {name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) { @@ -396,13 +426,22 @@ func TestATryLeftRunningIsTakenOver(t *testing.T) { b := newBench(t) wholeBench(b, delivery.Candidate{}) var ends []string - tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) { + failed := false + tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error { if event == tryevent.Ended { + if row.failFirst && !failed { + failed = true + return errors.New("the stream did not acknowledge") + } ends = append(ends, e.State) } + return nil }} tr.Reconsider(ctx) tr.Wait() + if row.failFirst && !readTry(t, kv, id).Told { + t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again") + } tr.Reconsider(ctx) // said once: the second look finds nothing owed tr.Wait() if row.after == nil { @@ -415,4 +454,28 @@ func TestATryLeftRunningIsTakenOver(t *testing.T) { } }) } + // An ended try is kept TryKeep and removed by the next Reconsider after. + t.Run("an ended try is kept seven days", func(t *testing.T) { + kv := newMemKV() + id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) + raw, rev, _, _ := kv.Get(ctx, id) + cur, _ := DecodeTry(raw) + ended := time.Now() + cur.State, cur.Ended = TryPass, ended + data, _ := encodeTry(cur) + if _, err := kv.Update(ctx, id, data, rev); err != nil { + t.Fatal(err) + } + now := ended.Add(TryKeep - time.Hour) + tr := &Tryer{Tries: kv, Now: func() time.Time { return now }} + tr.Reconsider(ctx) + if _, _, found, _ := kv.Get(ctx, id); !found { + t.Fatal("the try was removed before TryKeep") + } + now = ended.Add(TryKeep + time.Hour) + tr.Reconsider(ctx) + if _, _, found, _ := kv.Get(ctx, id); found { + t.Error("the try was kept past TryKeep") + } + }) } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 137609cf..b8bfc4df 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -147,6 +147,7 @@ type Indexer struct { OpenTries func(ctx context.Context) ([]tryevent.Open, error) // TryRecovery keeps how the last recovery of the open tries went (trycandidates.go); nil keeps none. TryRecovery Store + tryMu sync.Mutex // one try event or recovery at a time (trycandidates.go) recoveryFailed atomic.Bool // the last recovery of the open tries failed (trycandidates.go) tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go) diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 6dfe4310..3edcb9ef 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "os" + "runtime" "slices" "strings" "testing" @@ -385,6 +386,22 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW if err := ix.EndTry(ctx, "try-7"); err != nil || len(held()) != 0 { t.Errorf("an ended try: %v, candidates %v, want none", err, held()) } + // The hand recovery never overlaps a try event: a start heard while the call to the checker is out writes its candidate + // before or after the sweep, never between the call and the sweep (which would delete it, the try not being listed). + startDone := make(chan error, 1) + ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { + go func() { startDone <- ix.StartTry(ctx, "try-12", members) }() + for i := 0; i < 2000 && len(held()) == 0; i++ { + runtime.Gosched() // lets the start write if nothing holds it back + } + return nil, nil + } + if r := ix.RecoverTriesByHand(ctx, "lead", "a test"); !r.Stored { + t.Fatalf("the hand recovery = %+v", r) + } + if err := <-startDone; err != nil || len(held()) != 1 || held()[0] != "try-12" { + t.Errorf("a start heard during the hand recovery: %v, candidates %v, want try-12 kept", err, held()) + } }) } } diff --git a/modules/mesh-delivery-indexer/internal/index/trycandidates.go b/modules/mesh-delivery-indexer/internal/index/trycandidates.go index 75a71b51..0cec912b 100644 --- a/modules/mesh-delivery-indexer/internal/index/trycandidates.go +++ b/modules/mesh-delivery-indexer/internal/index/trycandidates.go @@ -57,6 +57,8 @@ type TryMember struct{ Repository, Head, Trunk string } // StartTry takes a whole try in, as the checker said it started (tryevent.Started): its members at their commits are kept, and // its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596). func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error { + ix.tryMu.Lock() + defer ix.tryMu.Unlock() ix.retryRecovery(ctx) return ix.startTry(ctx, id, members) } @@ -64,7 +66,7 @@ func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) // retryRecovery runs the recovery again on a try event when the last one failed: the failure stays said until one succeeds. func (ix *Indexer) retryRecovery(ctx context.Context) { if ix.recoveryFailed.Load() && ix.OpenTries != nil { - _ = ix.RecoverTries(ctx) + _ = ix.recoverNow(ctx) } } @@ -83,6 +85,8 @@ func (ix *Indexer) startTry(ctx context.Context, id string, members []TryMember) // EndTry takes a try out, as the checker said it ended (tryevent.Ended), and deletes its candidate. func (ix *Indexer) EndTry(ctx context.Context, id string) error { + ix.tryMu.Lock() + defer ix.tryMu.Unlock() ix.retryRecovery(ctx) return ix.endTry(ctx, id) } @@ -103,7 +107,16 @@ func (ix *Indexer) endTry(ctx context.Context, id string) error { // // A recovery that fails is said loudly and kept (the try-recovery state, read by indexer_try_candidates) until one succeeds: // the events are still heard after it, and indexer_recover_tries runs it again by hand. +// +// A recovery and a try event never overlap (tryMu): a sweep that deleted the candidate a start event wrote after the call to the +// checker would leave a running try with none. func (ix *Indexer) RecoverTries(ctx context.Context) error { + ix.tryMu.Lock() + defer ix.tryMu.Unlock() + return ix.recoverNow(ctx) +} + +func (ix *Indexer) recoverNow(ctx context.Context) error { if ix.OpenTries == nil || ix.Store == nil { return nil } From 9f9e743b2f7ac5a7a8c03046654cebb0bc7e3aa4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 19:47:42 +0100 Subject: [PATCH 11/13] The indexer started during its replacement fixes the successor's build; a start that decides nothing says so; a failing head is said once (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-indexer/named_test.go | 13 +++++++++++ .../cmd/mesh-delivery-indexer/ownbuild.go | 11 +++++++++ .../internal/index/candidates.go | 1 + .../internal/index/packages.go | 15 +++++++++++- .../internal/index/pending.go | 8 +++++++ .../internal/index/pending_test.go | 12 ++++++++++ .../internal/index/pending_tools.go | 11 ++++++--- .../internal/index/previews_test.go | 23 +++++++++++++++++++ .../internal/index/verdicts.go | 1 + .../mesh-delivery-indexer/takein/takein.go | 2 ++ 10 files changed, 93 insertions(+), 4 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index 8db30456..ed7c21af 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -59,6 +59,19 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 if fp, err := ownBuildVia(own, "n1")(context.Background()); err != nil || fp != "src2:abc" || asked != "seat:node-engine.composition@n1" { t.Errorf("own build = %q %v asked %q", fp, err, asked) } + // Issue 1921: the start during a replacement. What runs still names the old build, the want (composed by the node itself) + // the successor, which this process is: it fixes the successor. A want that is not self, or is removed, leaves what runs. + for _, row := range []struct{ name, answer, want string }{ + {"replacing: runs old, want new and self", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true}}`, "src2:new"}, + {"a want the node does not compose", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new"}}`, "src2:old"}, + {"a removed want", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true,"removed":true}}`, "src2:old"}, + {"no build runs yet, want self", `{"want":{"fp":"src2:new","self":true}}`, "src2:new"}, + } { + ask := func(string, any) (json.RawMessage, error) { return json.RawMessage(row.answer), nil } + if fp, err := ownBuildVia(ask, "n1")(context.Background()); err != nil || fp != row.want { + t.Errorf("%s: own build = %q %v, want %q", row.name, fp, err, row.want) + } + } none := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil } if _, err := ownBuildVia(none, "n1")(context.Background()); err == nil { t.Error("an engine that says nothing runs gave a build") diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go index 79a31448..cf7bb311 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go @@ -25,10 +25,21 @@ func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string Runs *struct { FP string `json:"fp"` } `json:"runs"` + Want *struct { + FP string `json:"fp"` + Self bool `json:"self"` + Removed bool `json:"removed"` + } `json:"want"` } if err := json.Unmarshal(raw, &a); err != nil { return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err) } + // While the engine replaces the indexer, this process is the successor: what runs still names the old build, and the want, + // composed by the node itself (self), names this one. The start fixes that build (quick fix, ADR 0639 §7: nothing is learnt + // later and nothing waits on a timer); a process of the old build that is started again in that window would take it too. + if a.Want != nil && a.Want.Self && !a.Want.Removed && a.Want.FP != "" { + return a.Want.FP, nil + } if a.Runs == nil || a.Runs.FP == "" { return "", errors.New("the engine says no fingerprint of the indexer runs") } diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index b8bfc4df..46f7d37d 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -121,6 +121,7 @@ type Indexer struct { filling atomic.Bool // a FillTaken runs in the background (KickFill) fillAgain atomic.Bool // a kick arrived while it ran atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished) + failedAt map[string]string // the head of a repository whose packages the forge could not list, by repository: said once, not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. OwnBuild func(ctx context.Context) (string, error) underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go index de1f0e73..b7cb7a6d 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages.go +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -297,6 +297,12 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st ix.mu.Lock() got, ok := ix.atHeads[at] ix.mu.Unlock() + ix.mu.Lock() + failed := ix.failedAt[repo] == head + ix.mu.Unlock() + if failed { + continue // said once for this head; the candidate takes the published version until the head changes + } if !ok { var ms []string ms = append(ms, modules[repo]...) @@ -307,10 +313,17 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st } var err error if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil { - ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err) + ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version (said once for this head): %v", repo, head, err) + ix.mu.Lock() + if ix.failedAt == nil { + ix.failedAt = map[string]string{} + } + ix.failedAt[repo] = head + ix.mu.Unlock() continue } ix.mu.Lock() + delete(ix.failedAt, repo) if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept { ix.atHeads = map[string]map[string]delivery.PublishedPackages{} } diff --git a/modules/mesh-delivery-indexer/internal/index/pending.go b/modules/mesh-delivery-indexer/internal/index/pending.go index cd34f376..ccc5673f 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending.go +++ b/modules/mesh-delivery-indexer/internal/index/pending.go @@ -559,6 +559,10 @@ func (ix *Indexer) TakeInAtStart(ctx context.Context) (Applied, string, error) { if errors.Is(err, process.ErrNotYet) { return a, "the pending state is not on the bus yet: nothing waits", nil } + if err == nil && len(a.Written) == 0 { + // A start that takes nothing in says so, with the build it learnt (the wait's own build is in the line holdFor says). + ix.logf("take-in: at the start, this program runs build %s and took nothing in: %s", own, words) + } return a, words, err } @@ -978,6 +982,10 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own for _, old := range saidWaits(ended) { ix.sayTakeIn(ctx, old) } + if said && !changed { + // Nothing to say on the bus, but the start decided nothing and a reader of the journal sees why: the build learnt, the build the wait names. + ix.logf("take-in: %d merges, oldest %s, still name the indexer build %s, this program runs %s: they wait, as already said", len(ready), t.Key, names, own) + } if !said || changed { ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own) if refused != "" { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_test.go b/modules/mesh-delivery-indexer/internal/index/pending_test.go index 7f787eb7..f750a58c 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_test.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_test.go @@ -281,6 +281,18 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633) n.FixOwnBuild(ctx) return n } + // A start that decides nothing says so (issue 1921): the build it learnt, the build the held wait names, also when the + // wait was said before; and indexer_pending names the known build and since when. + oldBuild, _ := b.ix.OwnBuild(ctx) + oldStart := mk(func(context.Context) (string, error) { return oldBuild, nil }) + b.logs.Reset() + if _, words, err := oldStart.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(b.logs.String(), "still name the indexer build") || + !strings.Contains(b.logs.String(), "at the start, this program runs build") { + t.Fatalf("a start that holds: %q %v, journal %q: want the build learnt and the build the wait names said", words, err, b.logs.String()) + } + if answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{}); err != nil || answer.(takein.PendingAnswer).OwnBuild != oldBuild || answer.(takein.PendingAnswer).OwnBuildSince.IsZero() { + t.Errorf("indexer_pending = %+v %v, want the known build and since", answer, err) + } asked := 0 next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") }) if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") { diff --git a/modules/mesh-delivery-indexer/internal/index/pending_tools.go b/modules/mesh-delivery-indexer/internal/index/pending_tools.go index 733cb70a..f0d24c1a 100644 --- a/modules/mesh-delivery-indexer/internal/index/pending_tools.go +++ b/modules/mesh-delivery-indexer/internal/index/pending_tools.go @@ -53,7 +53,7 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T "merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " + "longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " + "every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " + - "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build-unknown, when set, says the indexer cannot learn the build it " + + "indexer_take_in_pending and indexer_drop_pending steer a wait. own-build says which build the indexer knows it runs and own-build-since since when; own-build-unknown, when set, says the indexer cannot learn the build it " + "runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { @@ -69,8 +69,13 @@ func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (T if ownBuild != nil { if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found { var o OwnBuild - if json.Unmarshal(v, &o) == nil && !o.Known { - answer.OwnBuildUnknown, answer.OwnBuildSince = o.Why, o.Since + if json.Unmarshal(v, &o) == nil { + answer.OwnBuildSince = o.Since + if o.Known { + answer.OwnBuild = o.FP + } else { + answer.OwnBuildUnknown = o.Why + } } } } diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 3edcb9ef..63060d55 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "os" "runtime" "slices" @@ -404,6 +405,28 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW } }) } + // Issue 1921: a head whose packages the forge cannot list is said once and asked once, however many passes read it; the + // entry is dropped when the repository's head changes, and the new head is asked. + t.Run("a failing head is said and asked once, until the head changes", func(t *testing.T) { + ix, _, _ := previewBed(t, false) + var logs strings.Builder + ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") } + asked := 0 + ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) { + asked++ + return nil, errors.New("the forge does not answer") + } + held := map[string]delivery.Module{"x": {Inputs2: &delivery.KeptInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}} + for _, head := range []string{h1, h1, h2} { + ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held) + if head == h1 && (asked != 1 || strings.Count(logs.String(), "its packages are not read") != 1) { + t.Fatalf("after head %s: asked %d, journal %q: want one ask and one line", head, asked, logs.String()) + } + } + if asked != 2 || strings.Count(logs.String(), "its packages are not read") != 2 { + t.Errorf("after a new head: asked %d, journal %q: want it asked and said again", asked, logs.String()) + } + }) } func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { diff --git a/modules/mesh-delivery-indexer/internal/index/verdicts.go b/modules/mesh-delivery-indexer/internal/index/verdicts.go index 1758f740..7f931a39 100644 --- a/modules/mesh-delivery-indexer/internal/index/verdicts.go +++ b/modules/mesh-delivery-indexer/internal/index/verdicts.go @@ -396,6 +396,7 @@ func (ix *Indexer) FixOwnBuild(ctx context.Context) { ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err) } else { st.Known, st.FP = true, fp + ix.logf("judge: this program runs the indexer build %s, fixed here at its start", fp) } ix.ownMu.Lock() ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why diff --git a/modules/mesh-delivery-indexer/takein/takein.go b/modules/mesh-delivery-indexer/takein/takein.go index af9ba44f..4ad0ca9e 100644 --- a/modules/mesh-delivery-indexer/takein/takein.go +++ b/modules/mesh-delivery-indexer/takein/takein.go @@ -254,6 +254,8 @@ type PendingAnswer struct { // (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known. OwnBuildUnknown string `json:"own-build-unknown,omitempty"` OwnBuildSince time.Time `json:"own-build-since,omitzero"` + // OwnBuild is the build the running indexer knows it runs, fixed at its start; OwnBuildSince is since when. Empty when unknown. + OwnBuild string `json:"own-build,omitempty"` } // DecodePendingAnswer reads indexer_pending's answer, refusing a wait that is not a revision or a group, or names no From 78d57e1cc6c839f83e40dc06961126dd78504bd6 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 19:49:27 +0100 Subject: [PATCH 12/13] A transient forge failure is asked again; only a directory missing from the tree is cached per head (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-indexer/named.go | 2 +- .../cmd/mesh-delivery-indexer/named_test.go | 10 ++++ .../internal/index/candidates.go | 3 +- .../internal/index/packages.go | 25 +++++++-- .../internal/index/previews_test.go | 53 +++++++++++-------- 5 files changed, 66 insertions(+), 27 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go index 791ca492..56b8ff07 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named.go @@ -102,7 +102,7 @@ func filesVia(call toolCaller) func(ctx context.Context, repository, sha string, } for _, d := range dirs { if _, ok := a.Files[d]; !ok { - return nil, fmt.Errorf("the answer of %s lists no files of %q", commitTreesTool, d) + return nil, fmt.Errorf("the answer of %s lists no files of %q: %w", commitTreesTool, d, index.ErrNotInTree) } } return a.Files, nil diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index ed7c21af..e8153b5c 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "mesh-delivery-indexer/internal/index" "slices" "strings" "testing" @@ -50,6 +51,15 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 if _, has := gotArgs.(map[string]any)["paths"]; has { t.Error("the trunk asked for paths") } + // A directory the answer lists no files of is a failure of the tree (index.ErrNotInTree); a call that fails is not. + noDir := func(string, any) (json.RawMessage, error) { return json.RawMessage(`{"files":{}}`), nil } + if _, err := filesVia(noDir)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); !errors.Is(err, index.ErrNotInTree) { + t.Errorf("a missing directory: %v, want ErrNotInTree", err) + } + down := func(string, any) (json.RawMessage, error) { return nil, errors.New("down") } + if _, err := filesVia(down)(context.Background(), "novox/mesh-sdk", strings.Repeat("a", 40), []string{"go"}); err == nil || errors.Is(err, index.ErrNotInTree) { + t.Errorf("a failed call: %v, want an error that is not ErrNotInTree", err) + } // The build the indexer runs is the engine's word (Runs.FP of its composition verb), asked of its own machine. var asked string own := func(key string, body any) (json.RawMessage, error) { diff --git a/modules/mesh-delivery-indexer/internal/index/candidates.go b/modules/mesh-delivery-indexer/internal/index/candidates.go index 46f7d37d..3e55a248 100644 --- a/modules/mesh-delivery-indexer/internal/index/candidates.go +++ b/modules/mesh-delivery-indexer/internal/index/candidates.go @@ -121,7 +121,8 @@ type Indexer struct { filling atomic.Bool // a FillTaken runs in the background (KickFill) fillAgain atomic.Bool // a kick arrived while it ran atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished) - failedAt map[string]string // the head of a repository whose packages the forge could not list, by repository: said once, not asked again until the head changes + saidAt map[string]string // the head of a repository whose failed read was said, by repository: said once per head + failedAt map[string]string // the head of a repository whose tree lacks a directory asked, by repository: not asked again until the head changes // OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none. OwnBuild func(ctx context.Context) (string, error) underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go) diff --git a/modules/mesh-delivery-indexer/internal/index/packages.go b/modules/mesh-delivery-indexer/internal/index/packages.go index b7cb7a6d..bda1e48e 100644 --- a/modules/mesh-delivery-indexer/internal/index/packages.go +++ b/modules/mesh-delivery-indexer/internal/index/packages.go @@ -2,6 +2,7 @@ package index import ( "context" + "errors" "fmt" "maps" "path" @@ -313,17 +314,29 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st } var err error if got, err = ix.packagesAt(ctx, repo, head, ms, imports[repo]); err != nil { - ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version (said once for this head): %v", repo, head, err) + // A failure of the tree itself (the directory is not in the tree at that head) no retry changes: it is asked once per + // head. Any other failure may be transient: it is asked again at the next pass, and said once per head. ix.mu.Lock() - if ix.failedAt == nil { - ix.failedAt = map[string]string{} + said := ix.saidAt[repo] == head + if ix.saidAt == nil { + ix.saidAt = map[string]string{} + } + ix.saidAt[repo] = head + if errors.Is(err, ErrNotInTree) { + if ix.failedAt == nil { + ix.failedAt = map[string]string{} + } + ix.failedAt[repo] = head } - ix.failedAt[repo] = head ix.mu.Unlock() + if !said { + ix.logf("%s at the head %s: its packages are not read, so the candidate takes their published version: %v", repo, head, err) + } continue } ix.mu.Lock() delete(ix.failedAt, repo) + delete(ix.saidAt, repo) if ix.atHeads == nil || len(ix.atHeads) >= maxHeadsKept { ix.atHeads = map[string]map[string]delivery.PublishedPackages{} } @@ -335,5 +348,9 @@ func (ix *Indexer) candidatePublished(ctx context.Context, members map[string]st return out } +// ErrNotInTree is a failure of the tree itself: a directory asked of the forge is not in the tree at that commit. No retry +// changes it, so a head that fails so is asked once (candidatePublished). +var ErrNotInTree = errors.New("the directory is not in the tree at that commit") + // maxHeadsKept bounds the member heads' packages kept between recomputes; past it they are read again. const maxHeadsKept = 64 diff --git a/modules/mesh-delivery-indexer/internal/index/previews_test.go b/modules/mesh-delivery-indexer/internal/index/previews_test.go index 63060d55..1d95b041 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews_test.go +++ b/modules/mesh-delivery-indexer/internal/index/previews_test.go @@ -405,28 +405,39 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW } }) } - // Issue 1921: a head whose packages the forge cannot list is said once and asked once, however many passes read it; the - // entry is dropped when the repository's head changes, and the new head is asked. - t.Run("a failing head is said and asked once, until the head changes", func(t *testing.T) { - ix, _, _ := previewBed(t, false) - var logs strings.Builder - ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") } - asked := 0 - ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) { - asked++ - return nil, errors.New("the forge does not answer") - } - held := map[string]delivery.Module{"x": {Inputs2: &delivery.KeptInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}} - for _, head := range []string{h1, h1, h2} { - ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held) - if head == h1 && (asked != 1 || strings.Count(logs.String(), "its packages are not read") != 1) { - t.Fatalf("after head %s: asked %d, journal %q: want one ask and one line", head, asked, logs.String()) + // Issue 1921: a head whose tree lacks the directory asked is asked once, however many passes read it, until the head changes; + // any other failure (the forge does not answer) is asked again at every pass and said once per head. + for _, row := range []struct { + name string + err error + askedAfterTwo int // the same head read twice + askedAfterNew int // then another head + linesAfterNew int + }{ + {"the directory is not in the tree", fmt.Errorf("listing: %w", ErrNotInTree), 1, 2, 2}, + {"the forge does not answer", errors.New("the forge does not answer"), 2, 3, 2}, + } { + t.Run("a failing head: "+row.name, func(t *testing.T) { + ix, _, _ := previewBed(t, false) + var logs strings.Builder + ix.Logf = func(format string, args ...any) { logs.WriteString(fmt.Sprintf(format, args...) + "\n") } + asked := 0 + ix.Files = func(context.Context, string, string, []string) (map[string]map[string]string, error) { + asked++ + return nil, row.err } - } - if asked != 2 || strings.Count(logs.String(), "its packages are not read") != 2 { - t.Errorf("after a new head: asked %d, journal %q: want it asked and said again", asked, logs.String()) - } - }) + held := map[string]delivery.Module{"x": {Inputs2: &delivery.KeptInputs2{Packages: []string{"git.novox.be/novox/mesh-sdk/go"}}}} + for pass, head := range []string{h1, h1, h2} { + ix.candidatePublished(context.Background(), map[string]string{"novox/mesh-sdk": head}, held) + if pass == 1 && (asked != row.askedAfterTwo || strings.Count(logs.String(), "its packages are not read") != 1) { + t.Fatalf("after two passes over %s: asked %d (want %d), journal %q: want one line", head, asked, row.askedAfterTwo, logs.String()) + } + } + if asked != row.askedAfterNew || strings.Count(logs.String(), "its packages are not read") != row.linesAfterNew { + t.Errorf("asked %d (want %d), journal %q (want %d lines)", asked, row.askedAfterNew, logs.String(), row.linesAfterNew) + } + }) + } } func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) { From e3735da928f4b405624478263eef7ccb32ca2f23 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 5 Nov 2026 19:53:05 +0100 Subject: [PATCH 13/13] The indexer takes the engine's want as its build only while the apply is in progress (issue 1921) Claude-Session: https://claude.ai/code/session_01NcbgfFSnn24M3GcqttoU7i --- .../cmd/mesh-delivery-indexer/named_test.go | 3 +++ .../cmd/mesh-delivery-indexer/ownbuild.go | 21 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go index e8153b5c..2c5cfb13 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/named_test.go @@ -74,6 +74,9 @@ func TestTheTrunkIsTheDirectoriesTheForgeListsTrees(t *testing.T) { // 1900/AC3 for _, row := range []struct{ name, answer, want string }{ {"replacing: runs old, want new and self", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true}}`, "src2:new"}, {"a want the node does not compose", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new"}}`, "src2:old"}, + {"replacing: the apply of the want failed or is held", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:new","held":true}}`, "src2:old"}, + {"replacing: an attempt of another build", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true},"attempt":{"fp":"src2:older"}}`, "src2:new"}, + {"the want already runs", `{"runs":{"fp":"src2:new"},"want":{"fp":"src2:new","self":true}}`, "src2:new"}, {"a removed want", `{"runs":{"fp":"src2:old"},"want":{"fp":"src2:new","self":true,"removed":true}}`, "src2:old"}, {"no build runs yet, want self", `{"want":{"fp":"src2:new","self":true}}`, "src2:new"}, } { diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go index cf7bb311..3491b443 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/ownbuild.go @@ -30,15 +30,26 @@ func ownBuildVia(call toolCaller, node string) func(ctx context.Context) (string Self bool `json:"self"` Removed bool `json:"removed"` } `json:"want"` + // Attempt is the engine's last apply that did not end running (failed, or held), with the build it was of. + Attempt *struct { + FP string `json:"fp"` + } `json:"attempt"` } if err := json.Unmarshal(raw, &a); err != nil { return "", fmt.Errorf("the answer of %s is not a composition: %w", compositionVerb, err) } - // While the engine replaces the indexer, this process is the successor: what runs still names the old build, and the want, - // composed by the node itself (self), names this one. The start fixes that build (quick fix, ADR 0639 §7: nothing is learnt - // later and nothing waits on a timer); a process of the old build that is started again in that window would take it too. - if a.Want != nil && a.Want.Self && !a.Want.Removed && a.Want.FP != "" { - return a.Want.FP, nil + // While the engine's apply of the want is in progress (what runs still names the old build, the want names another, and no + // attempt names that want as failed or held), this process is the successor and fixes the want's build. self is the node's + // standing way of composing the module, not a mark of this process, so it is not enough alone: an old process started again + // while the want names a successor that does not run (its apply failed or is held) takes what runs. The start fixes the + // build once (quick fix, ADR 0639 §7: nothing is learnt later, nothing waits on a timer); the thorough form is the + // engine passing each process the fingerprint it starts it with. + running := "" + if a.Runs != nil { + running = a.Runs.FP + } + if w := a.Want; w != nil && w.Self && !w.Removed && w.FP != "" && w.FP != running && (a.Attempt == nil || a.Attempt.FP != w.FP) { + return w.FP, nil } if a.Runs == nil || a.Runs.FP == "" { return "", errors.New("the engine says no fingerprint of the indexer runs")