Merge pull request 'The running indexer accepts its successor, and a delivery group can be tried whole (issue 1966, issue 1921)' (#155) from fix/1921-group-try into main

This commit was merged in pull request #155.
This commit is contained in:
2026-11-05 18:34:52 +00:00
48 changed files with 2640 additions and 208 deletions
@@ -103,15 +103,25 @@ func TryTools(tr Trying) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
return []stdio.Tool{
{Name: check.TryTool,
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
"where it stands. A try of the same head that waits or runs is the answer to a second ask; a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide; tries let in while waiting may later all run at once, and each node's build capacity bounds what runs on it. A repository " +
"the checker checks no level of (one whose trunk the gate does not name) is refused. A try whose ask no build node took within the step " +
"bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"},
Description: "Try a delivery group, or one pushed head, before its pull requests are opened (try a group, group try, try several repositories): " +
"name the members, each a repository and the exact commit asked (never the branch's head now, so a push after the ask changes nothing of what is tried), " +
"or one repository and its head for a try of one member (the seat verb try of mesh-delivery-checker takes members from the registry milestone on; until it is delivered, ask a group through this tool). The checker judges it by the code a group's pull-request check runs, so the two cannot drift: " +
"each member merged onto its trunk's head, the unit and the integration level of every member through the build modules, the manifest validator over " +
"the members' manifests, the module graph, and the composition on every placed node. Every failure is named (each member, level, test, manifest and node), not " +
"the first. A try owes the forge nothing (no status, no Approve asked) and writes no verdict of a pull request; what its passing levels prove is kept, so a pull request's " +
"check of the same heads on the same trunk runs only what the try did not pass. A try narrowed to a level or to some tests judges only those, no manifest and no composition, and keeps nothing of tests. " +
"Answers the try's id at once; " + check.TryReadTool + " reads where it stands, " + check.TryEndTool + " ends it by hand. A try of the same commits that waits or runs is the answer to a second ask " +
"(whatever the order of the members); a new try is refused while 4 tries run on a build node or 8 wait for one, mesh-wide. A repository the checker checks no level of (one whose trunk the gate does not name) " +
"is refused. A try whose ask no build node took within the step bound ends as error; its ask stays in the build seat's stream and may still run later, judged by nobody (issue 734).",
Input: map[string]any{"type": "object", "properties": map[string]any{
"members": map[string]any{"type": "array", "description": "the group: one repository at most once, each at the exact commit asked and the trunk commit it is merged onto",
"items": map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent"}}}},
"repository": map[string]any{"type": "string", "description": "<owner>/<name>, for a try of one member (instead of members)"},
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits (with repository)"},
"trunk": map[string]any{"type": "string", "description": "the trunk commit to merge it onto, or the trunk's branch for its head now; the trunk's head when absent (with repository)"},
"tests": map[string]any{"type": "string", "description": "a go test -run pattern: only those tests run, and nothing is kept"},
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "only this level; both when absent"}}},
Run: func(args map[string]any) (any, error) {
@@ -126,16 +136,32 @@ func TryTools(tr Trying) []stdio.Tool {
if tr.Now != nil {
now = tr.Now
}
var members []check.TryAskMember
if raw, ok := args["members"]; ok {
list, isList := raw.([]any)
if !isList {
return nil, fmt.Errorf("members is a list of {repository, head, trunk}")
}
for _, item := range list {
m, isMap := item.(map[string]any)
if !isMap {
return nil, fmt.Errorf("a member is {repository, head, trunk}")
}
members = append(members, check.TryAskMember{Repository: str(m, "repository"), Head: str(m, "head"), Trunk: str(m, "trunk")})
}
}
id, err := check.AskTry(tr.Tries, trunks, check.TryAsk{Repository: str(args, "repository"), Head: str(args, "head"),
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level")}, now())
Trunk: str(args, "trunk"), Tests: str(args, "tests"), Level: str(args, "level"), Members: members}, now())
if err != nil {
return nil, err
}
return map[string]any{"id": id}, nil
}},
{Name: check.TryReadTool,
Description: "Where a try stands, as stored: queued, running, or its end (pass, fail, or error when it could not run), with what was said of it, " +
"the head and the trunk commit it was merged onto, the time it took and what failed first; while it runs, each ask it put as soon as it is put, " +
Description: "Where a try (of a group or of one head) stands, read whole, as stored: queued, running, or its end (pass, fail, or error when it could not run, or " +
"when its manifests and composition could not be judged), with what was said of it, every member's commit and the trunk commit it was merged onto, the time it took, " +
"every failure one line each (members, levels, tests, manifests, the composition per node), the levels' tests run and kept, the composition on every placed node, " +
"the notes of the check (manifests judged, module graph, kept verdicts); while it runs, each ask it put as soon as it is put, " +
"with the node that took it, its state and the phase its step says (fetch and merge, a level's digest, lint, tests) and since when. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"id"}, "properties": map[string]any{
"id": map[string]any{"type": "string", "description": "the try's id, try-<digits>"}}},
@@ -146,7 +172,7 @@ func TryTools(tr Trying) []stdio.Tool {
return check.ReadTry(tr.Tries, str(args, "id"))
}},
{Name: check.TryEndTool,
Description: "End a try that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
Description: "End a try of a group (or of one head) that waits or runs, as error, by hand: say why (kept in what it says). Its place among the 4 is free at once, " +
"the checker stops waiting for its asks, and the next " + check.TryTool + " of its head asks anew with a new id. For a try whose ask " +
"was lost (no step runs for it in checker_status); the checker ends one by itself when the build seat gives its ask up, or no node " +
"takes it within the step bound. A try that ended already is refused.",
@@ -170,6 +196,24 @@ func TryTools(tr Trying) []stdio.Tool {
}
}
// OpenTriesTools is the tool that lists the tries that wait or run, each with its members at their commits, and whether it is
// a whole try (the indexer writes the candidate of a whole try's group): the read a part uses at its start to find the tries
// whose start event it missed, instead of reading the checker's state.
func OpenTriesTools(tr Trying) []stdio.Tool {
return []stdio.Tool{{Name: check.TriesTool,
Description: "The tries that wait or run now (a try of a group or of one head; try.started and try.ended on the checker's stream say the same), each with its id, state, its members at their exact commits and " +
"trunk commits, and whether it is a whole try (not narrowed to a level or to tests, so it is composed and its manifests judged). For a part that " +
"missed the checker's try.started or try.ended event (the indexer at its start); checker_try_read reads one try whole. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
if tr.Tries == nil {
return nil, fmt.Errorf("the tries state cannot be read here")
}
open, err := check.OpenTries(tr.Tries)
return map[string]any{"tries": open}, err
}}}
}
// Seat is the seat the checker holds (novox/hq ADR 0547 §10, issue 774); its verbs are served as tools named <seat>.<verb>.
const Seat = "mesh-delivery-checker"
@@ -325,7 +369,7 @@ func main() {
_, err := verdicts.Update(key, json.RawMessage(value), revision)
return err
}}
tools := append(append(append(Tools(r), TryTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
tools := append(append(append(append(Tools(r), TryTools(trying)...), OpenTriesTools(trying)...), SeatTryTools(trying)...), RecoveryTools(r, written)...)
if err := stdio.Serve("", logged(tools)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
@@ -8,6 +8,8 @@ import (
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"mesh-delivery-checker/internal/check"
)
@@ -106,6 +108,39 @@ func TestTheTryToolsAskAndReadATry(t *testing.T) { // ADR 0501 §5, the agreed i
if _, err := verbs[0].Run(map[string]any{"repository": "novox/hq", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("the verb tried a repository nothing checks")
}
// A group try (issue 1921): the members, each an exact commit; members and one repository together are refused; the
// description says what it does in the words a search for "try a group" finds.
later := TryTools(Trying{Tries: store, Ender: store, Now: func() time.Time { return time.Unix(2, 5) },
Trunks: func() (map[string]string, error) {
return map[string]string{"novox/mesh-delivery": strings.Repeat("a", 40)}, nil
}})
g, err := later[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}}})
if err != nil || g.(map[string]any)["id"] == id {
t.Errorf("a group try = %v, %v, want a try of its own", g, err)
}
if _, err := tools[0].Run(map[string]any{"members": []any{map[string]any{"repository": "novox/mesh-delivery", "head": strings.Repeat("2", 40)}},
"repository": "novox/mesh-delivery", "head": strings.Repeat("1", 40)}); err == nil {
t.Error("members and a repository were both taken")
}
if _, err := tools[0].Run(map[string]any{"members": "novox/mesh-delivery"}); err == nil {
t.Error("members that are no list were taken")
}
for _, tl := range tools {
for _, words := range []string{"group"} {
if !strings.Contains(strings.ToLower(tl.Description), words) {
t.Errorf("%s says nothing of a %s: %q", tl.Name, words, tl.Description)
}
}
}
if d := strings.ToLower(tools[0].Description); !strings.Contains(d, "try a group") || !strings.Contains(d, "exact commit") || !strings.Contains(d, "every failure") ||
!strings.Contains(d, "composition on every placed node") || !strings.Contains(d, "owes the forge nothing") || !strings.Contains(d, "kept") {
t.Errorf("the try's description does not say what a group try does: %q", d)
}
if open := OpenTriesTools(Trying{Tries: store}); len(open) != 1 || open[0].Name != check.TriesTool || !strings.Contains(open[0].Description, "try.started") {
t.Errorf("open tries tool = %+v", open)
} else if out, err := open[0].Run(nil); err != nil || len(out.(map[string]any)["tries"].([]tryevent.Open)) != 2 {
t.Errorf("open tries = %v, %v, want the two tries asked", out, err)
}
// Issue 734: a try whose ask was lost is ended by hand with why, and its head is asked anew.
ended, err := tools[2].Run(map[string]any{"id": id, "why": "its ask was lost"})
if err != nil || ended.(map[string]any)["state"] != "error" {
@@ -862,9 +862,11 @@ func (s triesStore) Create(key string, value []byte) error {
// TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed: a try written as checker_try writes it is taken by the running checker,
// judged through the build modules as a pull request's head merged onto its trunk, and ended pass in its key, with no
// verdict and no status owed (ADR 0501 §5).
// verdict and no status owed (ADR 0501 §5). A whole try waits for the indexer's candidate of its group, which it reads on the
// key's watch, and writes the merge previews the indexer makes it from (issue 1921).
func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
b := startBed(t)
asked := b.engine(t, "n1") // a whole try is composed on every placed node, as a group's check is
b.start(t, nil)
store := triesStore{b.buckets[check.TriesBucket], b.ctx}
id, err := check.AskTry(store, map[string]string{repo: trunk}, check.TryAsk{Repository: repo, Head: head1}, time.Now())
@@ -872,19 +874,36 @@ func TestATryIsJudgedAndItsEndWrittenWithNoStatusOwed(t *testing.T) {
t.Fatal(err)
}
var last check.Try
// The indexer's candidate of the try's group (here written by the test), at the heads of its members.
raw0, _ := store.Get(id)
first, _ := check.DecodeTry(raw0)
cand := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: check.TryCase(id, first, nil).HeadsHash,
Pulls: []delivery.CandidatePull{{Head: delivery.Head{Repository: repo, Head: head1}, Modules: []string{}}}}
cand.FPSet = delivery.FingerprintSet(cand.Pulls)
data, err := delivery.EncodeCandidate(cand)
if err != nil {
t.Fatal(err)
}
ckey, _ := delivery.CandidatesKey(id)
if _, err := b.buckets[delivery.CandidatesBucket].Create(b.ctx, ckey, data); err != nil {
t.Fatal(err)
}
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
raw, _ := store.Get(id)
if last, err = check.DecodeTry(raw); err == nil && last.Over() {
break
}
}
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" {
t.Fatalf("try = %+v, want passed on the trunk", last)
if last.State != check.TryPass || last.Trunk != trunk || last.Took == "" || last.Composition == nil {
t.Fatalf("try = %+v, want passed on the trunk with its composition made", last)
}
if got := asked(); len(got) != 1 || !strings.HasPrefix(got[0].Candidate, id+".") {
t.Errorf("the placed node's engine was asked %+v, want the try's candidate %s.<heads>", got, id)
}
if b.fake.count("mesh-build-go.build_go_fetch") == 0 {
t.Errorf("the try fetched nothing")
}
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket, delivery.PreviewsBucket} {
for _, name := range []string{delivery.CheckerOwedBucket, delivery.CheckerVerdictsBucket} {
if keys, _ := b.buckets[name].Keys(b.ctx); len(keys) != 0 {
t.Errorf("a try wrote %s: %v", name, keys)
}
@@ -29,7 +29,10 @@ type judgeDeps struct {
trunks *check.Trunks
kept check.KV
previews check.KV
repos *repoModules
// candidates answers the indexer's candidate of a try's group (try_candidates.go); nil leaves a whole try with no
// manifest or composition judged, said.
candidates check.TryCandidates
repos *repoModules
// published is the indexer's packages state as it is followed (packages.go).
published *publishedPackages
forge check.Forge
@@ -157,7 +160,7 @@ func newJudge(d judgeDeps) check.Judge {
nodes = d.nodes
}
ex := &check.Examiner{Nodes: nodes, Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Candidates: d.candidates, Modules: d.repos, Packages: d.repos, Published: d.published, Forge: d.forge, Runs: d.repos, Builds: d.builds, BuildEnds: d.ends, BuildWait: d.buildWait, Announce: d.announce, Trunks: d.trunks, Manifests: d.repos, Read: d.repos, Predefined: d.predefined, Logf: d.logf}
if d.nodes != nil && d.builds != nil && d.repos != nil {
ex.Serving = runningServes{have: d.nodes, builds: d.builds, read: d.repos, logf: d.logf}
}
@@ -21,6 +21,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/announce"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
@@ -189,7 +190,13 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
Verdicts: verdicts.all,
}
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf}
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, NotTaken: stepBound, Logf: logger.Printf,
// A whole try's start and end are said on the stream checks, for the indexer to write and delete the candidate of its group.
Say: func(ctx context.Context, event string, body tryevent.Event) error {
data, _ := json.Marshal(body)
_, err := js.Publish(ctx, eventSubject(event), data, jetstream.WithRetryAttempts(10), jetstream.WithRetryWait(time.Second))
return err
}}
announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
@@ -216,7 +223,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
ch.Runs = repos // the node-engine a check reads beside at its running commit (ADR 0548 §6)
ch.Judge = newJudge(judgeDeps{
asks: buildrun.Asker{Publisher: asks, Waiter: waiter, Bound: stepBound}, tools: tools, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, candidates: tryCandidates{buckets[delivery.CandidatesBucket]},
repos: repos, published: published, builds: builds, ends: ends, buildWait: set.buildWait, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce, compositions: compositions,
nodes: openNodeNames(ctx, js, logger.Printf),
@@ -333,6 +340,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
}()
each := func(e jetstream.KeyValueEntry) {
if check.IsTryGroup(e.Key()) {
return // a group try's candidate: its try judges it, and the forge is owed nothing (issue 1921)
}
if opOf(e) {
ch.Candidate(ctx, e.Key(), e.Value())
} else {
@@ -0,0 +1,72 @@
package main
import (
"context"
"fmt"
"time"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// candidateWaitsOnPreview is the None2 of a module the indexer wrote from a pull message alone, before it had the merge
// preview of the head (the indexer's NoneNoPreview): its fingerprint comes with the preview, so a candidate that still says
// it is not yet the candidate of the heads previewed. It moves to mesh-sdk's delivery package with the next change there.
const candidateWaitsOnPreview = "no merge preview is computed"
// tryCandidates answers the indexer's candidate of a try's group (novox/hq ADR 0593 §7, issue 1921): the candidate the
// engines compose and the manifest validator judges, which the indexer makes of the try's previews. It waits for it on the
// key's own watch, with no timer between reads, and gives up at the bound.
type tryCandidates struct{ kv jetstream.KeyValue }
func (t tryCandidates) TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error) {
key, err := delivery.CandidatesKey(group)
if err != nil {
return delivery.Candidate{}, err
}
if bound <= 0 {
bound = time.Minute
}
waiting, cancel := context.WithTimeout(ctx, bound)
defer cancel()
w, err := t.kv.Watch(waiting, key) // the value now, then each change
if err != nil {
return delivery.Candidate{}, fmt.Errorf("the indexer's candidate %s cannot be watched: %w", key, err)
}
defer func() { _ = w.Stop() }()
last := "the indexer holds none"
for {
select {
case <-waiting.Done():
return delivery.Candidate{}, fmt.Errorf("the indexer wrote no candidate of %s at heads %s within %s (%s)", group, heads, bound, last)
case e := <-w.Updates():
if e == nil || e.Operation() != jetstream.KeyValuePut {
continue
}
c, err := delivery.DecodeCandidate(e.Value())
switch {
case err != nil:
last = "what it holds is not a candidate: " + err.Error()
case c.HeadsHash != heads:
last = "it holds heads " + c.HeadsHash
case waitsOnPreview(c):
last = "it has not yet the merge previews' fingerprints"
default:
return c, nil
}
}
}
}
// waitsOnPreview says a module of the candidate has no fingerprint yet because its preview was not read.
func waitsOnPreview(c delivery.Candidate) bool {
for _, p := range c.Pulls {
for _, m := range p.Fps {
if m.None2 == candidateWaitsOnPreview {
return true
}
}
}
return false
}
@@ -29,18 +29,23 @@ const (
nodeEngineModule = "mesh-host"
// networkingRepository holds route-proxy, whose manifest the node-engine's proofs of a composition read.
networkingRepository = "novox/mesh-networking"
// contractsRepository and deliveryRepository hold the seat definitions mesh-sdk's bootstrapped seats are compared with: its
// tests read both beside it (beside.Dir "mesh-contracts" and "mesh-delivery"), at the group's head for a member, else at the
// trunk (a group try lays them out the same way: besideFor serves the check and the try).
contractsRepository = "novox/mesh-contracts"
deliveryRepository = DeliveryRepository
)
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
// checks/deps). Which commit of each is besideCommit's rule. No repository's tests read the controller, which ended by its
// unassignment (novox/hq ADR 0590, issue 1126), and its repository is emptied and archived (ADR 0593 §11), so it has no
// siblings placed either. The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// siblings placed either. mesh-sdk reads mesh-contracts and mesh-delivery (its bootstrapped seats against their contracts files). The catalogue's tests read the node-engine since its composer command moved there (ADR 0593
// §7): mesh-build-go builds it for a compose ask. The node-engine's read mesh-networking since route-proxy's proofs of a
// composition moved into them (ADR 0593).
var besideRules = map[string][]string{
nodeEngineRepo: {catalogueRepository, SDKRepository, networkingRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
SDKRepository: {catalogueRepository, nodeEngineRepo, contractsRepository, deliveryRepository},
catalogueRepository: {SDKRepository, nodeEngineRepo},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
}
@@ -55,6 +55,8 @@ type Case struct {
Watch func(askID string, step *delivery.Step)
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
// Wait says what a whole try waits for now ("" when nothing); the try keeps it for try-read. Nil for a pull request's check.
Wait func(why string)
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a carry after a trunk or sibling move is asked on the
// build seats' recheck stream, every other check on check (ADR 0536 §4, ADR 0548 §5).
Kind string
@@ -1040,8 +1040,14 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
// trunk's, in a second ask (ADR 0633 clause 6, 1585/T22, T23)
indexer string
ownJudge bool
// try judges the same change as a whole group try, whose manifests come from the indexer's candidate of the try's
// group (issue 1921): the same judge, the same verdicts.
try bool
}{
{name: "accepted"},
{name: "accepted in a group try", try: true},
{name: "refused in a group try", try: true, refused: "web has no id", fails: "the indexer's judge at mesh-delivery's trunk " + strings.Repeat("d", 40) + ` refuses the manifest of web: web has no id`},
{name: "a cycle the group try makes", try: true, cycle: "provisions", fails: "the change's module graph has the cycle mail -> web -> mail"},
{name: "the indexer accepted by the trunk's manifest validator", indexer: "ok"},
{name: "the indexer held by the trunk's manifest validator", indexer: "held", fails: "the indexer web, the holder of the seat mesh-catalog, would never run: the trunk's manifest validator at mesh-delivery's trunk " + strings.Repeat("d", 40) + " does not accept its manifest: the manifest validator holds it, with no verdict yet"},
{name: "the indexer with a field the trunk's manifest validator does not know", indexer: "aside", fails: "the manifest validator does not know its fields new_field, so it waits"},
@@ -1120,6 +1126,10 @@ func TestTheIndexersJudgeJudgesTheChangesManifests(t *testing.T) {
p.Modules = []string{"web"}
p.Fps = map[string]delivery.CandidateModule{"web": {FP2: "src2:" + strings.Repeat("f", 64), Manifest: web}}
cs := caseOf(t, p)
if row.try {
cs = TryCase("try-1", Try{Members: []TryMember{{Repository: repoA, Head: head1, Trunk: trunkA}}}, nil)
b.ex.Candidates = &fakeTryCandidates{cand: delivery.Candidate{FPSet: "fps", Pulls: []delivery.CandidatePull{p}}}
}
v := b.judge(t, cs).Verdict
if row.indexer == "ok" && !slices.ContainsFunc(cs.notes, func(n string) bool {
return strings.Contains(n, "the new indexer's manifest of web is accepted by the trunk's manifest validator")
@@ -30,7 +30,7 @@ type ManifestReader interface {
// refuseCycle fails the check for a cycle of the change's module graph, naming it; nil when the case touches no manifest,
// when nothing reads manifests, or when there is none. A manifest that cannot be read fails the check, said, never passes it.
func (st *state) refuseCycle(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.j.Read == nil || st.c.Try != nil {
if st.j.Manifests == nil || st.j.Read == nil || st.narrowTry() {
return nil
}
trunk := st.j.Manifests.ManifestDigests()
@@ -139,8 +139,18 @@ type Examiner struct {
Nodes interface {
Names(ctx context.Context) ([]string, error)
}
Now func() time.Time
Logf func(format string, args ...any)
// Candidates answers the indexer's candidate of a try's group once it wrote it from the try's previews, the manifests
// and fingerprints a group try's manifest validator and composition judge (issue 1921); nil leaves a whole try with no
// manifest or composition judged, said.
Candidates TryCandidates
Now func() time.Time
Logf func(format string, args ...any)
}
// TryCandidates answers the candidate the indexer holds for a try's group at its heads hash, waiting for it within the
// bound; an error says it did not come (and why).
type TryCandidates interface {
TryCandidate(ctx context.Context, group, heads string, bound time.Duration) (delivery.Candidate, error)
}
// CompositionReader makes a check's composition verdict; an error is only a case that cannot be composed at all.
@@ -246,6 +256,50 @@ type state struct {
// unpreviewed are the members (pullKey) with no preview of a merge: a head that does not merge, or a fetch that did not
// answer. The indexer holds no manifest of them, so the assigner's composition would only wait: it is not awaited.
unpreviewed []string
// noCandidate says why a whole try has no candidate of its group from the indexer, so its manifests were not judged and
// it was not composed; empty when it has one, and for a pull request's check.
noCandidate string
}
// narrowTry says the case is a try narrowed to a level or to some tests: it judges no manifest, graph or composition.
func (st *state) narrowTry() bool { return st.c.Try != nil && !st.c.Try.Whole() }
// takeTryCandidate gives a whole try's pulls the fingerprints and manifests the indexer's candidate of its group holds
// (the group's pull-request check has them from its candidate, CaseOf), once the indexer wrote it from the try's previews.
// A try with no such candidate is said so, in a note and in its composition, and is no pass (Tryer.run).
func (st *state) takeTryCandidate(ctx context.Context) {
if st.c.Try == nil || !st.c.Try.Whole() {
return
}
no := func(why string) {
st.noCandidate = "the group's manifests and composition were not judged: " + why
st.c.note(st.noCandidate)
}
if st.j.Candidates == nil {
no("the checker reads no candidate of the indexer's")
return
}
if len(st.unpreviewed) > 0 {
no("no preview of a merge of " + strings.Join(st.unpreviewed, ", ") + " (its head does not merge or its fetch did not answer)")
return
}
if st.c.Wait != nil {
st.c.Wait("the indexer's candidate of the group at its heads (up to " + st.j.buildWait().String() + ")")
defer st.c.Wait("")
}
cand, err := st.j.Candidates.TryCandidate(ctx, st.c.Group, st.c.HeadsHash, st.j.buildWait())
if err != nil {
no(err.Error())
return
}
for i, p := range st.c.Pulls {
for _, cp := range cand.Pulls {
if cp.Repository == p.Repository && cp.Head.Head == p.Head {
st.c.Pulls[i].Modules, st.c.Pulls[i].Fps = cp.Modules, cp.Fps
}
}
}
st.c.FPSet = cand.FPSet
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
@@ -256,6 +310,7 @@ func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
// Every member and level runs to its end, and every manifest the change touches is judged by the indexer's judge (ADR
// 0578 §2), also after a failure: the verdict names every failure, not only the first (the operator, 2026-11-02).
tests := st.judgeAll(ctx)
st.takeTryCandidate(ctx)
if v := settled(joinFailures(tests, st.judgeManifests(ctx), st.refuseCycle(ctx))); v != nil {
return st.finish(ctx, *v), nil
}
@@ -282,7 +337,14 @@ func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
st.c.note("the composition is kept: no fingerprint of a module it composed changed")
return
}
if st.j.Compositions == nil || st.c.Try != nil {
if st.narrowTry() {
return
}
if st.noCandidate != "" {
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: st.noCandidate}
return
}
if st.j.Compositions == nil {
return
}
if len(st.unpreviewed) > 0 {
@@ -433,7 +495,7 @@ func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
// Every build module's fetch merged (each is in merged once): the preview is written, also when the member's tests
// failed, so the indexer holds its manifests and the assigner can compose the change (review of 650a1d0).
switch {
case st.c.Try != nil: // a try owes the indexer no preview: no pull request holds its head
case st.narrowTry(): // a try of a level or some tests owes the indexer no preview: it judges no manifest
case len(merged) == len(modules):
st.j.writePreview(ctx, p, merged)
default:
@@ -562,7 +624,7 @@ func (st *state) askNeeding(ctx context.Context, t *target, merged *[]Fetched, l
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
}
if len(t.fetched.Conflicts) > 0 {
if st.c.Try == nil {
if !st.narrowTry() {
st.j.writePreview(ctx, p, append(*merged, t.fetched))
}
return fail(step.Node, fmt.Sprintf("%s#%d %s%s", p.Repository, p.Number, ConflictWords, strings.Join(t.fetched.Conflicts, ", ")))
@@ -97,7 +97,7 @@ func judgeRequest(trunk map[string]string, pulls []Pull) (req composer.Request,
// refuses. A case that touches no manifest, or an examiner that reads no trunk manifests, judges nothing; a judge that
// cannot be asked or does not answer fails the check, said, never passes it.
func (st *state) judgeManifests(ctx context.Context) *delivery.Verdict {
if st.j.Manifests == nil || st.c.Try != nil {
if st.j.Manifests == nil || st.narrowTry() {
return nil
}
req, touched := judgeRequest(st.j.Manifests.ManifestDigests(), st.c.Pulls)
@@ -111,10 +111,16 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
}
// Alone, mesh-sdk's tests read the catalogue's trunk, and the verdict names it.
b = newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2})
const contracts = "novox/mesh-contracts"
b.ex.Trunks = trunksOf(t, map[string]string{SDKRepository: trunkS, catalogue: trunkA, host: trunkA2, contracts: trunkA, DeliveryRepository: trunkA2})
if out := b.judge(t, caseOf(t, pullOf(SDKRepository, 7, head1))); out.Beside[catalogue] != trunkA {
t.Errorf("the siblings named for the verdict alone = %v, want the catalogue at its trunk %s", out.Beside, trunkA)
}
// mesh-sdk's tests compare its bootstrapped seats with their contracts files, read beside it: mesh-contracts and
// mesh-delivery, at their trunks alone (a group try is placed by the same besideFor, so it lays them out the same way).
if got := besideOf(t, b, SDKRepository); got[contracts].Commit != trunkA || got[DeliveryRepository].Commit != trunkA2 {
t.Errorf("beside mesh-sdk alone = %+v, want mesh-contracts at %s and mesh-delivery at %s", got, trunkA, trunkA2)
}
if c := besideOf(t, b, SDKRepository)[catalogue]; c.Commit != trunkA || c.Ref != "" {
t.Errorf("the catalogue beside mesh-sdk alone = %+v, want its trunk %s", c, trunkA)
}
@@ -164,6 +164,7 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
if err != nil {
return VerdictRead{}, err
}
keys = slices.DeleteFunc(keys, IsTryGroup)
sort.Strings(keys)
keys, cut := capKeys(keys, "")
// The candidates are read in batches, each in parallel and taken in key order, so the read stops at the batch that holds it.
@@ -444,6 +445,7 @@ func (r Reader) Status() (Status, error) {
if err != nil {
return st, err
}
keys = slices.DeleteFunc(keys, IsTryGroup) // a try's candidate is judged by its try, never as a check
sort.Strings(keys)
total := len(keys)
keys, cut := capKeys(keys, "")
@@ -14,6 +14,7 @@ import (
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-builder/buildrun"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -27,6 +28,8 @@ const (
TryTool = "checker_try"
TryReadTool = "checker_try_read"
TryEndTool = "checker_try_end"
// TriesTool lists the tries that wait or run (OpenTries).
TriesTool = "checker_tries"
// TriesState is the checker's state of tries: the bucket mesh-delivery-checker_tries, keys try-<unix nanoseconds>.
TriesState = "tries"
TriesBucket = delivery.CheckerModule + "_" + TriesState
@@ -62,18 +65,28 @@ var (
// Try is the value of a tries key.
type Try struct {
V int `json:"v"`
ID string `json:"id"`
V int `json:"v"`
ID string `json:"id"`
// Repository, Head, Trunk and Branch are the first member's (the try's headline, and all a try of one repository
// names); Members are every member of the try, by repository: each an exact commit merged onto a trunk commit (a
// group try, issue 1921). A try written before members read as one member.
Repository string `json:"repository"`
// Head is the commit tried; Trunk the trunk commit it is merged onto, and Branch the trunk's branch when it was asked by name.
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
Members []TryMember `json:"members,omitempty"`
// Tests is a go test -run pattern: only those tests run, and nothing is kept. Level is unit or integration: only that
// level runs. Empty is every test of both levels.
Tests string `json:"tests,omitempty"`
Level string `json:"level,omitempty"`
State string `json:"state"`
// Told is true from the moment a whole try's start is said on the checker's stream until its end is said there: kept in the
// try's own state, so an end that is written and not said (the checker stopped between) is said when the checker starts.
Told bool `json:"told,omitempty"`
// Waiting is what the running try waits for now (its group's candidate from the indexer, ADR 0639 §5), empty when it waits for nothing;
// try-read gives it, so a try that sits still says why.
Waiting string `json:"waiting,omitempty"`
// Said is the try's report in words: what failed first, the tests run, kept and skipped per level, the time of each part.
Said string `json:"said,omitempty"`
// Took is the time from its start to its end, as Go writes a duration.
@@ -83,6 +96,11 @@ type Try struct {
Failed []string `json:"failed,omitempty"`
Levels map[string]delivery.LevelResult `json:"levels,omitempty"`
Machines []string `json:"machines,omitempty"`
// Composition is what the composition on every placed node came to (a whole try, not one narrowed to a level or to
// some tests), and Notes the words the check noted on its way (the manifests judged, the module graph, the kept
// verdicts taken): the same as a group's pull-request check reports.
Composition *delivery.Composition `json:"composition,omitempty"`
Notes []string `json:"notes,omitempty"`
// Asks are the ids of the asks the try put on the build seats' streams, each ending on the step key <node>.<id> of its
// build module: the delivery view counts those steps as a try's, not a candidate's check (build_usage).
Asks []string `json:"asks,omitempty"`
@@ -94,6 +112,23 @@ type Try struct {
Ended time.Time `json:"ended,omitzero"`
}
// TryMember is one repository of a try: the exact commit asked, never the branch's head now, and the trunk commit it is
// merged onto. Branch is the trunk's branch when the trunk was asked by name.
type TryMember struct {
Repository string `json:"repository"`
Head string `json:"head"`
Trunk string `json:"trunk"`
Branch string `json:"branch,omitempty"`
}
// MembersOf are the try's members: its Members, or the one its headline names.
func (t Try) MembersOf() []TryMember {
if len(t.Members) > 0 {
return t.Members
}
return []TryMember{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk, Branch: t.Branch}}
}
// TryAsked is a TryStep's state before a build node took its ask: it waits in the seat's stream.
const TryAsked = "asked"
@@ -148,6 +183,17 @@ func (w *tryWriter) heard(askID string, step *delivery.Step) {
}
}
// waiting writes what the try waits for now ("" when it waits for nothing).
func (w *tryWriter) waiting(why string) {
w.mu.Lock()
defer w.mu.Unlock()
if w.stopped || w.t.Waiting == why {
return
}
w.t.Waiting = why
w.write()
}
// write writes the record at the revision written last; a refusal stops the writes.
func (w *tryWriter) write() {
data, err := encodeTry(w.t)
@@ -195,6 +241,7 @@ func DecodeTry(raw []byte) (Try, error) {
if !tryKeyOK.MatchString(t.ID) || t.Repository == "" || t.Head == "" || t.Trunk == "" {
return Try{}, errors.New("not a try: it names no id, repository, head or trunk")
}
t.Members = t.MembersOf()
return t, nil
}
@@ -206,6 +253,14 @@ func encodeTry(t Try) ([]byte, error) {
// TryAsk is what checker_try is given.
type TryAsk struct {
Repository, Head, Trunk, Tests, Level string
// Members are the repositories of a group try, each an exact commit and, optionally, its trunk (a commit, or the
// trunk's branch for its head now); the single Repository, Head and Trunk above are one member. Both at once are refused.
Members []TryAskMember
}
// TryAskMember is one repository of a group try as it is asked.
type TryAskMember struct {
Repository, Head, Trunk string
}
// TryStore is the tries state as the tools see it.
@@ -221,21 +276,16 @@ func NotChecked(repository string) error {
return fmt.Errorf("no level of %s is checked by the checker", repository)
}
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. trunks are the
// gate's trunk heads of every repository (source.main): a repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
// tryMember checks one member as it is asked and resolves its trunk: the gate's trunk head of its repository, a commit
// asked, or the trunk's branch (its head now). A repository with no trunk head is no repository the checker checks.
func tryMember(trunks map[string]string, a TryAskMember) (TryMember, error) {
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
head := strings.ToLower(strings.TrimSpace(a.Head))
switch {
case !tryRepoOK.MatchString(repo) || strings.Contains(repo, ".."):
return "", fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
return TryMember{}, fmt.Errorf("%q is not a repository, <owner>/<name>", a.Repository)
case !tryHeadOK.MatchString(head):
return "", fmt.Errorf("the head %q is not a full commit, 40 or 64 hex digits", a.Head)
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
return TryMember{}, fmt.Errorf("the head %q of %s is not a full commit, 40 or 64 hex digits", a.Head, repo)
}
trunkHead := ""
for r, h := range trunks {
@@ -244,20 +294,67 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
}
}
if trunkHead == "" {
return "", NotChecked(repo)
return TryMember{}, NotChecked(repo)
}
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
m := TryMember{Repository: repo, Head: head}
switch trunk := strings.TrimSpace(a.Trunk); {
case trunk == "":
t.Trunk = trunkHead
m.Trunk = trunkHead
case tryHeadOK.MatchString(strings.ToLower(trunk)):
t.Trunk = strings.ToLower(trunk)
m.Trunk = strings.ToLower(trunk)
case tryTrunkOK.MatchString(trunk) && !strings.Contains(trunk, ".."):
// The checker judges pull requests into the trunk alone (ADR 0484 clause 11): a branch named is the trunk's head now.
t.Trunk, t.Branch = trunkHead, trunk
m.Trunk, m.Branch = trunkHead, trunk
default:
return "", fmt.Errorf("the trunk %q is neither a full commit nor a branch", a.Trunk)
return TryMember{}, fmt.Errorf("the trunk %q of %s is neither a full commit nor a branch", a.Trunk, repo)
}
return m, nil
}
// sameMembers says whether two tries ask the same commits onto the same trunk commits.
func sameMembers(a, b []TryMember) bool {
return slices.EqualFunc(a, b, func(x, y TryMember) bool {
return x.Repository == y.Repository && x.Head == y.Head && x.Trunk == y.Trunk
})
}
// AskTry writes a new try, queued, and answers its id: a try of the same commits that waits or runs is the answer to a second
// ask, and a new try is refused while MaxTries tries run on a build node or MaxTriesWaiting wait for one. A try is a group of
// one or more members (issue 1921), each an exact commit, never the branch's head now: a push after the ask changes nothing
// of what is tried. trunks are the gate's trunk heads of every repository (source.main): a repository with none is no
// repository the checker checks. A trunk asked by name is its trunk head now.
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
asked := a.Members
if a.Repository != "" || a.Head != "" || a.Trunk != "" {
if len(asked) > 0 {
return "", errors.New("name the members, or one repository with its head, not both")
}
asked = []TryAskMember{{Repository: a.Repository, Head: a.Head, Trunk: a.Trunk}}
}
if len(asked) == 0 {
return "", errors.New("name what to try: members (repository and head each), or a repository and its head")
}
switch {
case a.Level != "" && a.Level != delivery.LevelUnit && a.Level != delivery.LevelIntegration:
return "", fmt.Errorf("the level %q is neither unit nor integration", a.Level)
case a.Tests != "" && !tryTestsOK.MatchString(a.Tests):
return "", fmt.Errorf("the tests %q are not a go test -run pattern", a.Tests)
}
var members []TryMember
for _, m := range asked {
tm, err := tryMember(trunks, m)
if err != nil {
return "", err
}
if slices.ContainsFunc(members, func(o TryMember) bool { return o.Repository == tm.Repository }) {
return "", fmt.Errorf("%s is named twice: a try holds one commit of a repository", tm.Repository)
}
members = append(members, tm)
}
slices.SortFunc(members, func(x, y TryMember) int { return strings.Compare(x.Repository, y.Repository) })
first := members[0]
t := Try{Repository: first.Repository, Head: first.Head, Trunk: first.Trunk, Branch: first.Branch, Members: members,
Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
keys, err := store.Keys()
if err != nil {
return "", fmt.Errorf("the tries cannot be listed: %w", err)
@@ -291,10 +388,10 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
if !live[i] {
continue
}
if cur.Repository == t.Repository && cur.Head == t.Head && cur.Trunk == t.Trunk && cur.Tests == t.Tests && cur.Level == t.Level {
if sameMembers(cur.MembersOf(), t.Members) && cur.Tests == t.Tests && cur.Level == t.Level {
return cur.ID, nil
}
said := fmt.Sprintf("%s (%s@%.12s, %s)", cur.ID, cur.Repository, cur.Head, cur.State)
said := fmt.Sprintf("%s (%s, %s)", cur.ID, sayMembers(cur.MembersOf()), cur.State)
if cur.HoldsANode() {
holding = append(holding, said)
} else {
@@ -379,12 +476,50 @@ func EndTry(store TryEnder, id, why string, now time.Time) (Try, error) {
return t, nil
}
// IsTryGroup says a candidates key is the group of a try (the indexer makes the candidate of a group try's previews, for the
// manifest validator and the composition): the try judges it, so the checker's check of candidates leaves it alone, writes
// no verdict and owes the forge no status for it.
func IsTryGroup(key string) bool { return tryKeyOK.MatchString(key) }
// TryOptions narrow what a try runs; a pull request's check has none.
type TryOptions struct {
Level string
Run string
}
// Whole says the try runs everything a group's pull-request check runs (both levels of every member, the manifests judged by
// the manifest validator, the module graph, the composition on every placed node): it is narrowed to neither a level nor
// some tests. Such a try judges by the same code as the check, so the two cannot drift (issue 1921).
func (o *TryOptions) Whole() bool { return o != nil && o.Level == "" && o.Run == "" }
// TryCase is the case a try is judged as: one pull of the group per member at its exact commit, with no pull request number
// (no ref: the commit is fetched as it is), the group named by the try's key, and the heads hash its candidate is keyed by.
func TryCase(key string, t Try, watch func(string, *delivery.Step)) *Case {
cs := &Case{Key: key, Group: key, Watch: watch, Trunk: map[string]string{}, Queued: t.Asked, Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
var heads []delivery.Head
for _, m := range t.MembersOf() {
cs.Pulls = append(cs.Pulls, Pull{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
cs.Trunk[m.Repository] = m.Trunk
heads = append(heads, delivery.Head{Repository: m.Repository, Head: m.Head})
}
cs.HeadsHash, _ = delivery.HeadsHash(heads)
cs.Heads = cs.HeadsHash
return cs
}
// compositionMade says whether a composition was made; notMadeWords what it says when it was not.
func compositionMade(c *delivery.Composition) bool {
return c != nil && c.State != delivery.CompositionNotRun
}
func notMadeWords(c *delivery.Composition) string {
if c == nil || c.Words == "" {
return "no reason was given"
}
return c.Words
}
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR
// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written.
type Tryer struct {
@@ -396,7 +531,13 @@ type Tryer struct {
Now func() time.Time
Logf func(format string, args ...any)
// Say publishes an event of the checker's stream (tryevent.Started, tryevent.Ended) with its body; nil says nothing. The
// indexer, which hears them, writes and deletes the candidate of a whole try's group.
// It answers the publish's error, nil only when the stream acknowledged it: an end not acknowledged is said again.
Say func(ctx context.Context, event string, body tryevent.Event) error
mu sync.Mutex
ending map[string]bool // tries whose end is being said now
wg sync.WaitGroup
inflight map[string]context.CancelFunc
seen map[string]bool // looked at since this process started: a try left running is taken over the first time
@@ -436,6 +577,7 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
stop()
}
tr.mu.Unlock()
tr.sendEnd(ctx, key) // ended by hand while it ran, or its end was written and not said: the indexer drops its candidate
return
}
if tr.inflight[key] != nil || (t.State == TryRunning && tr.seen[key]) {
@@ -491,6 +633,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
t.State, t.Started = TryRunning, tr.now().UTC()
t.Told = (&TryOptions{Level: t.Level, Run: t.Tests}).Whole() // its start is said below; its end is owed from now on
data, err := encodeTry(t)
if err != nil {
return
@@ -502,12 +645,15 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
progress := &tryWriter{tr: tr, ctx: ctx, key: key, t: t, rev: rev}
cs := &Case{Key: key, Group: key, Watch: progress.heard,
Pulls: []Pull{{Repository: t.Repository, Head: t.Head, Trunk: t.Trunk}},
Trunk: map[string]string{t.Repository: t.Trunk},
Queued: t.Asked,
Progress: &Progress{},
Try: &TryOptions{Level: t.Level, Run: t.Tests}}
if t.Told {
started := tryevent.Event{ID: key}
for _, m := range t.MembersOf() {
started.Members = append(started.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
}
tr.say(ctx, tryevent.Started, started)
}
cs := TryCase(key, t, progress.heard)
cs.Wait = progress.waiting
judging := ctx
if tr.NotTaken > 0 {
judging = buildrun.NotTakenBound(ctx, tr.NotTaken)
@@ -517,12 +663,18 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return // left running: the checker takes it over when it starts; or ended by hand, which wrote its end
}
t, rev = progress.end()
t.Waiting = ""
v := out.Verdict
t.Ended = tr.now().UTC()
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
t.Levels, t.Machines, t.Asks = v.Levels, v.Machines, cs.Asks()
t.Composition = v.Composition
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
for _, n := range cs.Notes() {
if len(cs.Pulls) > 1 {
t.Said = sayMembers(t.MembersOf()) + ": " + t.Said
}
t.Notes = cs.Notes()
for _, n := range t.Notes {
t.Said += " | " + n
}
switch {
@@ -532,6 +684,16 @@ func (tr *Tryer) run(ctx context.Context, key string) {
if IsLostAsk(v) || IsNotTaken(v) {
t.Failed = []string{v.Failed.Words}
}
case v.State == delivery.VerdictPass && cs.Try.Whole() && compositionMade(v.Composition) && strings.Contains(v.Composition.Words, delivery.ComposeNoAnswer):
// A node that gave no answer passes a group's check with a warning; a try that is to say the group composes says it
// did not, naming the node (ADR 0637 §3).
t.State = TryError
t.Failed = []string{"the composition was not made on every placed node: " + v.Composition.Words}
case v.State == delivery.VerdictPass && cs.Try.Whole() && !compositionMade(v.Composition):
// A whole try that judged the tests and not the manifests and the composition has not tried what its group's check
// judges: it is no pass, and it says why (failure must be loud). It is an error, not the change's failure.
t.State = TryError
t.Failed = []string{"the composition was not made: " + notMadeWords(v.Composition)}
case v.State == delivery.VerdictPass:
t.State = TryPass
default:
@@ -549,6 +711,97 @@ func (tr *Tryer) run(ctx context.Context, key string) {
return
}
tr.logf("%s: %s", key, t.State)
tr.sendEnd(ctx, key)
}
// sendEnd says the end of an ended try whose start was said (Told) with the state it ended in, then clears Told in the try's
// own state; a checker that stopped between the two says it again at its start. Every end is said so: pass, fail, error and
// by hand.
func (tr *Tryer) sendEnd(ctx context.Context, key string) {
tr.mu.Lock()
if tr.ending[key] {
tr.mu.Unlock()
return
}
if tr.ending == nil {
tr.ending = map[string]bool{}
}
tr.ending[key] = true
tr.mu.Unlock()
defer func() { tr.mu.Lock(); delete(tr.ending, key); tr.mu.Unlock() }()
raw, rev, found, err := tr.Tries.Get(ctx, key)
if err != nil || !found {
return
}
t, err := DecodeTry(raw)
if err != nil || !t.Over() || !t.Told {
return
}
if tr.say(ctx, tryevent.Ended, tryevent.Event{ID: key, State: t.State}) != nil {
return // not acknowledged: Told stays, and the next Put or Reconsider says the end again
}
t.Told = false
data, err := encodeTry(t)
if err == nil {
_, err = tr.Tries.Update(ctx, key, data, rev)
}
if err != nil && !errors.Is(err, ErrConflict) {
tr.logf("%s: its end was said and Told could not be cleared: %v", key, err)
}
}
// say publishes a try event; a failure is said in the log (the indexer finds the open tries again at its start).
func (tr *Tryer) say(ctx context.Context, event string, body tryevent.Event) error {
if tr.Say == nil {
return nil
}
err := tr.Say(ctx, event, body)
if err != nil {
tr.logf("%s of %s could not be said: %v", event, body.ID, err)
}
return err
}
// OpenTries are every try that waits or runs now: what checker_tries answers, so a part that missed an event (the indexer at
// its start) finds them without reading the checker's state. All the keys are read (an old open try is as open as a new one), in
// parallel.
func OpenTries(store TryStore) ([]tryevent.Open, error) {
keys, err := store.Keys()
if err != nil {
return nil, fmt.Errorf("the tries cannot be listed: %w", err)
}
sort.Slice(keys, func(i, j int) bool {
if len(keys[i]) != len(keys[j]) {
return len(keys[i]) > len(keys[j])
}
return keys[i] > keys[j]
})
found := make([]*tryevent.Open, len(keys))
if err := parallel(len(keys), func(i int) error {
raw, err := store.Get(keys[i])
if err != nil || len(raw) == 0 {
return nil
}
t, err := DecodeTry(raw)
if err != nil || t.Over() {
return nil
}
o := tryevent.Open{ID: t.ID, State: t.State, Whole: (&TryOptions{Level: t.Level, Run: t.Tests}).Whole()}
for _, m := range t.MembersOf() {
o.Members = append(o.Members, tryevent.Member{Repository: m.Repository, Head: m.Head, Trunk: m.Trunk})
}
found[i] = &o
return nil
}); err != nil {
return nil, err
}
var out []tryevent.Open
for _, o := range found {
if o != nil {
out = append(out, *o)
}
}
return out, nil
}
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
@@ -579,6 +832,15 @@ func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (O
return out, true
}
// sayMembers names a try's members, each repository at its short commit.
func sayMembers(ms []TryMember) string {
var out []string
for _, m := range ms {
out = append(out, fmt.Sprintf("%s@%.12s", m.Repository, m.Head))
}
return strings.Join(out, " + ")
}
// orNone joins the tries said, or says none.
func orNone(said []string) string {
if len(said) == 0 {
@@ -3,11 +3,14 @@ package check
import (
"context"
"encoding/json"
"errors"
"fmt"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -70,6 +73,25 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
}
// Issue 1890: tries on a build node (a step of theirs running, or none named yet) and tries waiting for one (queued, or
// their asks not taken) are bounded apart, MaxTries and MaxTriesWaiting; the refusal names both counts.
// A group try is the same try whatever the order of its members; the commits asked are the commits tried.
group := TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2}, {Repository: repoA, Head: head1}}}
gid, err := AskTry(store, tryTrunks, group, at.Add(time.Second))
if err != nil || gid == id {
t.Fatalf("the group try = %q, %v, want a try of its own", gid, err)
}
group.Members[0], group.Members[1] = group.Members[1], group.Members[0]
if again, err := AskTry(store, tryTrunks, group, at.Add(time.Minute)); err != nil || again != gid {
t.Errorf("the group asked in another order answered %q, %v, want %q", again, err, gid)
}
if g := readTry(t, kv, gid); len(g.Members) != 2 || g.Members[0].Repository != repoA || g.Members[0].Head != head1 || g.Members[1].Trunk != trunkS {
t.Errorf("group try = %+v, want its members by repository at the commits asked", g.Members)
}
if open, err := OpenTries(store); err != nil || len(open) != 2 || !open[0].Whole || len(open[1].Members) != 2 && len(open[0].Members) != 2 {
t.Errorf("open tries = %+v, %v, want the two queued", open, err)
}
if !IsTryGroup(gid) || IsTryGroup("feat/x") || IsTryGroup("fix/1921-try-1") {
t.Errorf("which candidates keys are a try's group: %s is, a branch is not", gid)
}
onNode := func(i int) Try {
return Try{ID: fmt.Sprintf("try-%d", i), Repository: repoA, Head: fmt.Sprintf("%040x", i), Trunk: trunkA, State: TryRunning,
Steps: []TryStep{{Ask: "a", State: delivery.StepRunning}}}
@@ -112,59 +134,152 @@ func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agr
}
}
for _, bad := range []TryAsk{{Repository: repoA, Head: "abc"}, {Repository: repoA, Head: head1, Level: "e2e"},
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"}} {
{Repository: repoA, Head: head1, Tests: "a b"}, {Repository: "x", Head: head1}, {Repository: repoA, Head: head1, Trunk: "-x"},
// A group try (issue 1921): members or one repository, never both or none; one commit per repository; every member checked.
{},
{Repository: repoA, Head: head1, Members: []TryAskMember{{Repository: repoSDK, Head: head2}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoA, Head: head2}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: repoSDK, Head: "main"}}},
{Members: []TryAskMember{{Repository: repoA, Head: head1}, {Repository: "novox/no-trunk", Head: head2}}}} {
if _, err := AskTry(store, tryTrunks, bad, time.Now()); err == nil {
t.Errorf("%+v was taken", bad)
}
}
}
func TestATryIsJudgedAsAPullRequestsHeadAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5
// fakeTryCandidates answers the indexer's candidate of a try's group from memory, recording the keys asked.
type fakeTryCandidates struct {
cand delivery.Candidate
err error
asked []string
// during is called while the try waits for the candidate.
during func()
}
func (f *fakeTryCandidates) TryCandidate(_ context.Context, group, heads string, _ time.Duration) (delivery.Candidate, error) {
if f.during != nil {
f.during()
}
f.asked = append(f.asked, group+"."+heads)
return f.cand, f.err
}
// wholeBench makes the bench judge a whole try as the checker does: the composition on every placed node and the
// indexer's candidate of the try's group.
func wholeBench(b *bench, cand delivery.Candidate) (*fakeCompositions, *fakeTryCandidates) {
comps := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14", "shanks"}, Took: 40}}
cands := &fakeTryCandidates{cand: cand}
b.ex.Compositions, b.ex.Candidates = comps, cands
return comps, cands
}
// A group try (issue 1921): several repositories each at the exact commit asked, judged by the code of a group's pull-request
// check; the two cannot drift, and a later check of the same heads keeps what the try passed.
func TestATryIsJudgedAsAGroupsCheckAndOwesTheForgeNothing(t *testing.T) { // ADR 0501 §5, issue 1921
ctx := context.Background()
kv := newMemKV()
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Trunk: trunkA}, time.Now())
id, err := AskTry(memTries{kv}, tryTrunks, TryAsk{Members: []TryAskMember{{Repository: repoSDK, Head: head2, Trunk: trunkS}, {Repository: repoA, Head: head1, Trunk: trunkA}}}, time.Now())
if err != nil {
t.Fatal(err)
}
b := newBench(t)
comps, cands := wholeBench(b, delivery.Candidate{FPSet: "fps-of-the-try"})
written := &writtenTries{memKV: kv}
tr := &Tryer{Tries: written, Judge: b.ex}
var waiting string
cands.during = func() { waiting = readTry(t, kv, id).Waiting } // ADR 0639 §5: the stored try says what it waits for
var said []string
tr := &Tryer{Tries: written, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
said = append(said, fmt.Sprintf("%s %s %d %s", event, e.ID, len(e.Members), e.State))
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
// The indexer hears the start with the members at their commits, and the end (it writes and deletes the group's candidate).
if len(said) != 2 || said[0] != "try.started "+id+" 2 " || said[1] != "try.ended "+id+" 0 "+TryPass {
t.Errorf("events = %v, want the start with 2 members and the end", said)
}
// What the indexer reads: tryevent.Decode accepts the events as said and refuses a body that is no try's.
for _, row := range []struct {
name, event, body string
ok bool
}{
{"a start with its members", tryevent.Started, `{"id":"` + id + `","members":[{"repository":"` + repoA + `","head":"` + head1 + `","trunk":"` + trunkA + `"}]}`, true},
{"an end with its state", tryevent.Ended, `{"id":"` + id + `","state":"pass"}`, true},
{"an id that is no try's", tryevent.Ended, `{"id":"pull-7","state":"pass"}`, false},
{"a start with no member", tryevent.Started, `{"id":"` + id + `"}`, false},
{"a body that is no JSON", tryevent.Ended, `try-7`, false},
} {
if e, err := tryevent.Decode(row.event, []byte(row.body)); (err == nil) != row.ok || (row.ok && e.ID != id) {
t.Errorf("Decode of %s = %+v, %v, want ok=%v", row.name, e, err, row.ok)
}
}
got := readTry(t, kv, id)
if got.State != TryPass || got.Took == "" || got.Head != head1 || got.Trunk != trunkA || got.Said == "" || got.Levels[delivery.LevelUnit].Run != 2 {
if !strings.Contains(waiting, "candidate") || got.Waiting != "" {
t.Errorf("waiting while the candidate was awaited = %q, after the end %q: want the reason, then none", waiting, got.Waiting)
}
if got.Told { // the end was said: nothing is owed
t.Errorf("the ended try still owes its end: %+v", got)
}
if open, err := OpenTries(memTries{kv}); err != nil || len(open) != 0 {
t.Errorf("open tries after the end = %v, %v", open, err)
}
if got.State != TryPass || got.Took == "" || len(got.Members) != 2 || got.Members[0].Repository != repoA || got.Members[1].Head != head2 ||
got.Said == "" || got.Levels[delivery.LevelUnit].Run+got.Levels[delivery.LevelUnit].Kept != 4 { // the bench gives both members the same digests
t.Fatalf("try = %+v", got)
}
// Issue 1890: while it runs, the try names its ask as soon as it is put, and then where its step stands; the end
// keeps each ask's node and state.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked ||
written.running[2].Steps[0].Node != "shanks" {
t.Errorf("the running try's writes = %+v, want its ask named while it ran, then its node", written.running)
if got.Composition == nil || got.Composition.State != delivery.CompositionPass || len(got.Machines) == 0 {
t.Errorf("the try's composition = %+v, want the composition on every placed node, as a check's", got.Composition)
}
if len(got.Steps) != 1 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed {
t.Errorf("the ended try's steps = %+v", got.Steps)
// The same key the indexer's candidate of the group has: the try's id and the heads of its members.
if len(comps.asked) != 1 || comps.asked[0] != id || len(cands.asked) != 1 || !strings.HasPrefix(cands.asked[0], id+".") {
t.Errorf("composed %v, candidate asked %v, want the try's key %s", comps.asked, cands.asked, id)
}
// The try names the asks it put, so the delivery view counts their steps as a try's (build_usage).
if len(got.Asks) != 1 {
t.Errorf("asks = %v, want the one check ask the try put", got.Asks)
} else if _, err := delivery.AskStepKey("shanks", got.Asks[0]); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", got.Asks[0], err)
// Issue 1890: while it runs, the try names its asks as soon as they are put, and then where each step stands.
if len(written.running) < 3 || len(written.running[1].Asks) != 1 || written.running[1].Steps[0].State != TryAsked {
t.Errorf("the running try's writes = %+v, want its ask named while it ran", written.running)
}
if len(got.Steps) != 2 || got.Steps[0].Node != "shanks" || got.Steps[0].State != delivery.StepPassed || len(got.Asks) != 2 {
t.Errorf("the ended try's steps = %+v, asks %v", got.Steps, got.Asks)
}
for _, a := range got.Asks { // the delivery view counts their steps as a try's (build_usage)
if _, err := delivery.AskStepKey("shanks", a); err != nil {
t.Errorf("the try names %q, which is no ask id: %v", a, err)
}
}
// Each member is fetched at the exact commit asked, merged onto its own trunk, with no pull request's ref.
fetch := b.steps.tool("fetch")
if len(fetch) != 1 || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "" {
t.Errorf("fetch = %+v, want the head merged onto the trunk, with no pull request's ref", fetch)
if len(fetch) != 2 {
t.Fatalf("fetch = %+v, want one per member", fetch)
}
if keys, _ := b.prev.Keys(ctx); len(keys) != 0 {
t.Errorf("a try wrote the previews %v", keys)
for _, f := range fetch {
want := map[string][2]string{repoA: {head1, trunkA}, repoSDK: {head2, trunkS}}[f.args["repository"].(string)]
if f.args["commit"] != want[0] || f.args["base"] != want[1] || f.args["ref"] != "" {
t.Errorf("fetch = %+v, want the exact commit %s merged onto %s", f.args, want[0], want[1])
}
}
// A whole try writes the merge previews the indexer makes its candidate from; it writes no status and asks no Approve.
if keys, _ := b.prev.Keys(ctx); len(keys) != 2 {
t.Errorf("a whole try wrote the previews %v, want one per member", keys)
}
// An ended try is not run again.
raw, _, _, _ = kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if n := len(b.steps.tool("fetch")); n != 1 {
if n := len(b.steps.tool("fetch")); n != 2 {
t.Errorf("an ended try was fetched again: %d fetches", n)
}
// The kept verdicts: what the try passed is kept under the digests of the merged trees, so the pull-request check of
// the same head on the same trunk runs none of it again (and a push of other code runs only what changed).
before := len(b.steps.tool("test"))
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictPass || out.Verdict.Levels[delivery.LevelUnit].Kept == 0 || out.Verdict.Levels[delivery.LevelUnit].Run != 0 ||
out.Verdict.Levels[delivery.LevelIntegration].Run != 0 {
t.Errorf("the check of the tried head = %+v, want everything the try passed kept and nothing run", out.Verdict.Levels)
}
if after := len(b.steps.tool("test")); after != before {
t.Errorf("the check of the tried head ran %d test steps, want none: the try's passes are kept", after-before)
}
}
func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
@@ -175,10 +290,16 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
t.Fatal(err)
}
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(context.Context, string, tryevent.Event) error {
t.Error("a try of one level and some tests said an event: it has no candidate")
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if open, _ := OpenTries(memTries{kv}); len(open) != 0 {
t.Errorf("open = %v", open)
}
tests := b.steps.tool("test")
if len(tests) != 1 || tests[0].args["level"] != delivery.LevelIntegration || tests[0].args["run"] != "TestBus|TestStore" {
t.Fatalf("tests = %+v, want the integration level alone with the run pattern", tests)
@@ -192,6 +313,24 @@ func TestATryOfOneLevelAndSomeTestsRunsThemAloneAndKeepsNothing(t *testing.T) {
if got := readTry(t, kv, id); got.State != TryPass || !strings.Contains(got.Said, "only the tests TestBus|TestStore") {
t.Errorf("try = %+v", got)
}
t.Run("a try of one level keeps only that level", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1, Level: delivery.LevelUnit}, time.Now())
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
keys, _ := b.kept.Keys(ctx)
if got := readTry(t, kv, id); got.State != TryPass || len(keys) == 0 {
t.Fatalf("try = %+v, kept %v", got, keys)
}
for _, k := range keys {
if !strings.HasPrefix(k, delivery.LevelUnit+".") {
t.Errorf("a try of the unit level kept %s", k)
}
}
})
}
func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) {
@@ -208,26 +347,135 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
!strings.Contains(got.Failed[1], "integration: fixture/failing/bad.TestThreeIsFour") { // both levels ran (the operator, 2026-11-02)
t.Errorf("try = %+v", got)
}
// A whole try that could not judge the manifests and the composition is no pass; a failed composition is named beside
// the failed tests (the group's check does as much), and a candidate of other heads is not the try's.
for _, row := range []struct {
name string
cand *fakeTryCandidates
comp delivery.CompositionVerdict
state, said string
}{
{name: "the indexer wrote no candidate", cand: &fakeTryCandidates{err: errors.New("the indexer wrote no candidate within 5m0s")}, state: TryError, said: "the indexer wrote no candidate within 5m0s"},
{name: "a node that gave no composition answer", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x", Warnings: []string{"g14: " + delivery.ComposeNoAnswer}},
state: TryError, said: "g14: " + delivery.ComposeNoAnswer},
{name: "a composition refused on a node", cand: &fakeTryCandidates{}, comp: delivery.CompositionVerdict{State: delivery.CompositionFail, Machine: "g14", Words: "not composed on g14: web: needs a state mail does not keep"},
state: TryFail, said: "not composed on g14: web: needs a state mail does not keep"},
} {
kv := newMemKV()
b := newBench(t)
comps, _ := wholeBench(b, delivery.Candidate{})
b.ex.Candidates, comps.v = row.cand, row.comp
if row.comp.State == "" {
comps.v.State = delivery.CompositionPass
}
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
var ends []string
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
if event == tryevent.Ended {
ends = append(ends, e.State)
}
return nil
}}
raw, _, _, _ := kv.Get(ctx, id)
tr.Put(ctx, id, raw)
tr.Wait()
if len(ends) != 1 || ends[0] != row.state { // every end says its state: error as well as fail
t.Errorf("%s: ends said = %v, want once %s", row.name, ends, row.state)
}
if got := readTry(t, kv, id); got.State != row.state || !strings.Contains(strings.Join(got.Failed, " | ")+got.Said, row.said) {
t.Errorf("%s: try = %s, failed %v, said %q, want %s naming %q", row.name, got.State, got.Failed, got.Said, row.state, row.said)
}
}
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
}
func TestATryLeftRunningIsTakenOver(t *testing.T) {
ctx := context.Background()
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
// Left running by a checker that stopped.
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
cur.State = TryRunning
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
b := newBench(t)
tr := &Tryer{Tries: kv, Judge: b.ex}
tr.Reconsider(ctx)
tr.Wait()
if got := readTry(t, kv, id); got.State != TryPass {
t.Errorf("the try left running = %+v, want taken over and passed", got)
// Whether the start was said is in the try's own state (Told), so a checker that starts later says the end of a try it did
// not start, whatever way the try ended.
for _, row := range []struct {
name string
told bool
failFirst bool // the stream does not acknowledge the first end said
after func(t *testing.T, kv *memKV, id string)
ends []string
}{
{name: "the first publish of the end is not acknowledged", told: true, failFirst: true, ends: []string{TryPass}},
{name: "left running after its start was said", told: true, ends: []string{TryPass}},
{name: "left running with no start said", told: false, ends: []string{TryPass}},
{name: "ended by hand after the start was said", told: true, ends: []string{TryError}, after: func(t *testing.T, kv *memKV, id string) {
if _, err := EndTry(endable{kv}, id, "stuck", time.Now()); err != nil {
t.Fatal(err)
}
}},
} {
t.Run(row.name, func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
// Left running by a checker that stopped.
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
cur.State, cur.Told = TryRunning, row.told
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
if row.after != nil {
row.after(t, kv, id)
}
b := newBench(t)
wholeBench(b, delivery.Candidate{})
var ends []string
failed := false
tr := &Tryer{Tries: kv, Judge: b.ex, Say: func(_ context.Context, event string, e tryevent.Event) error {
if event == tryevent.Ended {
if row.failFirst && !failed {
failed = true
return errors.New("the stream did not acknowledge")
}
ends = append(ends, e.State)
}
return nil
}}
tr.Reconsider(ctx)
tr.Wait()
if row.failFirst && !readTry(t, kv, id).Told {
t.Errorf("an end the stream did not acknowledge cleared Told: it is never said again")
}
tr.Reconsider(ctx) // said once: the second look finds nothing owed
tr.Wait()
if row.after == nil {
if got := readTry(t, kv, id); got.State != TryPass {
t.Errorf("the try left running = %+v, want taken over and passed", got)
}
}
if !slices.Equal(ends, row.ends) || readTry(t, kv, id).Told {
t.Errorf("ends said = %v, want %v, and nothing owed after", ends, row.ends)
}
})
}
// An ended try is kept TryKeep and removed by the next Reconsider after.
t.Run("an ended try is kept seven days", func(t *testing.T) {
kv := newMemKV()
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
raw, rev, _, _ := kv.Get(ctx, id)
cur, _ := DecodeTry(raw)
ended := time.Now()
cur.State, cur.Ended = TryPass, ended
data, _ := encodeTry(cur)
if _, err := kv.Update(ctx, id, data, rev); err != nil {
t.Fatal(err)
}
now := ended.Add(TryKeep - time.Hour)
tr := &Tryer{Tries: kv, Now: func() time.Time { return now }}
tr.Reconsider(ctx)
if _, _, found, _ := kv.Get(ctx, id); !found {
t.Fatal("the try was removed before TryKeep")
}
now = ended.Add(TryKeep + time.Hour)
tr.Reconsider(ctx)
if _, _, found, _ := kv.Get(ctx, id); found {
t.Error("the try was kept past TryKeep")
}
})
}
+29 -3
View File
@@ -130,14 +130,40 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
if m.SecretsOwner != "mesh-delivery-checker" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) {
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3). A whole try says try.started with its members and try.ended, so the indexer writes and deletes the candidate of its group (issue 1921).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**", "try.**"}}}; !reflect.DeepEqual(m.Streams, want) {
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// Only the checker's account publishes try.* on its stream checks: the stream is declared by its owner alone, and no other
// module of the repository declares a stream carrying a try event (the indexer and the others only consume them).
siblings, err := filepath.Glob("../*/module.json")
if err != nil || len(siblings) < 2 {
t.Fatalf("the sibling manifests = %v, %v", siblings, err)
}
for _, f := range siblings {
raw, err := os.ReadFile(f)
if err != nil {
t.Fatal(err)
}
var other struct {
Module string `json:"module"`
Streams []stream `json:"streams"`
}
if err := json.Unmarshal(raw, &other); err != nil {
t.Fatalf("%s: %v", f, err)
}
for _, st := range other.Streams {
for _, e := range st.Events {
if strings.HasPrefix(e, "try.") && other.Module != m.Module {
t.Errorf("%s declares the stream %s with %s: only the checker's account publishes try events", other.Module, st.Name, e)
}
}
}
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
// The try's tools (ADR 0501 §5; the end by hand, issue 734) beside them: mesh-controller's try calls them. A lost check
// is asked again by hand (checker_ask_again).
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
if want := []string{"checker_ask_again", "checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_tries", "checker_try", "checker_try_end", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// The try is the checker's own seat's verbs (ADR 0547 §10, issue 774), served beside the tools by the same code.
+2 -1
View File
@@ -18,7 +18,7 @@
"mesh-build-node.toolchain.reported"
],
"streams": [
{ "name": "checks", "events": ["preview.**", "owed.**"] }
{ "name": "checks", "events": ["preview.**", "owed.**", "try.**"] }
],
"state": [
{ "name": "asks", "history": 1 },
@@ -56,6 +56,7 @@
"checker_previews",
"checker_pull",
"checker_status",
"checker_tries",
"checker_try",
"checker_try_end",
"checker_try_read",
@@ -216,7 +216,8 @@ var errReadOnly = errors.New("the tools bundle only reads this state")
func main() {
ask := askTakeIn(stdio.State(index.TakeInsState), 25*time.Second)
tools := append(Tools(ask, time.Now, stdio.State(index.VerdictsState), heldCounts(stdio.State(index.OldSectionsState))), catalogTools()...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, ask, time.Now)...)
tools = append(tools, index.PendingTools(keptRead{stdio.State(index.PendingState)}, keptRead{stdio.State(index.OwnBuildStateName)}, ask, time.Now)...)
tools = append(tools, index.TryCandidateTools(keptRead{stdio.State("candidates")}, keptRead{stdio.State(index.TryRecoveryState)}, ask, time.Now)...)
if err := stdio.Serve("", tools); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-indexer-tools] %v\n", err)
os.Exit(1)
@@ -283,7 +283,7 @@ func TestIndexerWhole(t *testing.T) { // 1238/AC2, AC10; 1585/T14, T15
}
waits := func() []takein.PendingWait {
t.Helper()
answer, err := index.PendingTools(kvStore{pendingKV}, nil, time.Now)[0].Run(map[string]any{})
answer, err := index.PendingTools(kvStore{pendingKV}, nil, nil, time.Now)[0].Run(map[string]any{})
if err != nil {
t.Fatal(err)
}
@@ -343,16 +343,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
oldSections := process.AwaitBucket(ctx, js, conn, index.OldSectionsBucket, "the count of the old sections is not kept, so the judge refuses no use of one", logger.Printf)
// The module graph (ADR 0603 §2): a state this delivery brings, bound when it appears; until then no graph is written.
graph := process.AwaitBucket(ctx, js, conn, graph.Bucket, "no module graph is written", logger.Printf)
ix := &index.Indexer{OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
ownBuildState := process.AwaitBucket(ctx, js, conn, index.OwnBuildBucket, "which build this indexer runs is not kept", logger.Printf)
tryRecovery := process.AwaitBucket(ctx, js, conn, index.TryRecoveryBucket, "how the recovery of the open tries went is not kept", logger.Printf)
ix := &index.Indexer{OwnBuildState: awaitedStore{ownBuildState}, TryRecovery: awaitedStore{tryRecovery}, OldSections: awaitedStore{oldSections}, Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Taken: awaitedStore{taken}, Builds: known,
Verdicts: awaitedStore{verdicts}, SayRefused: sayRefused(js, cred, logger.Printf), SayAccepted: sayAccepted(js, cred, logger.Printf),
Predefined: awaitedStore{predefined}, ConfigCandidates: awaitedStore{configCandidates}, Logf: logger.Printf,
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call), Read: readVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), Graph: awaitedStore{graph}, Message: messageVia(calls.Call),
ReadOpen: openVia(calls.Call), Packages: awaitedStore{packages}, Files: filesVia(calls.Call), GoSum: goSumVia(calls.Call), GoMod: goModVia(calls.Call), ReadFile: readFileVia(calls.Call),
OwnBuild: ownBuildVia(calls.Call, cred.Node), OpenTries: openTriesVia(calls.Call), Graph: awaitedStore{graph}, Message: messageVia(calls.Call),
// The take-in step (ADR 0633): its state pending and its stream take-in, both this delivery's, bound when they appear;
// until then every merge is taken in at once, as before.
Pending: awaitedStore{process.AwaitBucket(ctx, js, conn, index.PendingBucket, "no merge is held for the indexer its revision names: each is taken in at once", logger.Printf)},
SayTakeIn: sayTakeIn(js, cred)}
// Which build this program is, fixed here once, before it hears any merge and never asked again (index.FixOwnBuild): the
// node-engine names the successor while it replaces this process, and an old process that learnt it later would take
// revisions in with its old judge.
ix.FixOwnBuild(ctx)
// The log's entries at the start are read before any fingerprint is computed or registered module indexed (below); each
// new one indexes the module it names when that module is registered and the bucket holds none (registered.go).
var started atomic.Bool
@@ -494,15 +500,11 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer tc.Stop()
}
switch pcons, err := cred.EventConsumer(ctx, js, PreviewsStream); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
default:
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.Written(m) }, logger.Printf)
if err != nil {
return err
}
defer pc.Stop()
// The consumer on the checker's stream exists before the open tries are listed and is heard after (ADR 0638 §3): an event
// said between the call and the hearing waits in it, and a try that started or ended meanwhile is found either way.
pcons, perr := cred.EventConsumer(ctx, js, PreviewsStream)
if perr != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews and tries are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, perr)
}
// The pending revisions that name this build are taken in now, oldest first, before the consumer on the forge bridge's
// stream starts (ADR 0633 §2). When the build this program runs is not known yet, nothing waits for it: the consumer
@@ -513,6 +515,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
} else if words != "" {
logger.Printf("take-in at start: %s", words)
}
// The open tries are found again through the checker's tool: a try event missed while the indexer was down would leave a
// try with no candidate, or a candidate with no try. Not fatal: the events come, and a preview written writes the candidates.
// A recovery that fails is said (journal, and indexer_try_candidates) and the events are heard all the same.
_ = ix.RecoverTries(ctx)
if perr == nil {
pc, err := process.HearEvents(pcons, func(m jetstream.Msg) { previews.heardChecks(m) }, logger.Printf)
if err != nil {
return err
}
defer pc.Stop()
}
cons, err := cred.EventConsumer(ctx, js, forgeStream)
if err != nil {
// Essential: the forge's merges are what the indexer indexes.
@@ -10,6 +10,7 @@ import (
"strings"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
@@ -120,6 +121,12 @@ func goModVia(call toolCaller) func(ctx context.Context, repository, sha, dir st
return goFileVia(call, "go.mod")
}
// readFileVia reads one file of a repository at a commit by its path (gitea_get_file); a file the commit does not hold is
// none. The seat definitions of a revision are read with it (seatsof.go).
func readFileVia(call toolCaller) func(ctx context.Context, repository, sha, path string) ([]byte, error) {
return goFileVia(call, "")
}
// goFileVia reads one Go file of a module directory at a commit through the forge's get-file tool; a file the commit does
// not hold is none.
func goFileVia(call toolCaller, name string) func(ctx context.Context, repository, sha, dir string) ([]byte, error) {
@@ -129,7 +136,10 @@ func goFileVia(call toolCaller, name string) func(ctx context.Context, repositor
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
}
p := name
if dir != "" && dir != delivery.RootPath {
switch {
case name == "": // readFileVia: dir is the file's whole path
p = dir
case dir != "" && dir != delivery.RootPath:
p = dir + "/" + name
}
raw, err := call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": p, "ref": sha})
@@ -285,3 +295,23 @@ func openVia(call toolCaller) func(ctx context.Context) ([]forgemsg.OpenPull, er
return out, nil
}
}
// openTriesTool is the checker's tool that lists the tries that wait or run (mesh-delivery-checker checker_tries).
const openTriesTool = "mesh-delivery-checker.checker_tries"
// openTriesVia lists the open tries through the checker's tool, never its state (ADR 0596).
func openTriesVia(call toolCaller) func(ctx context.Context) ([]tryevent.Open, error) {
return func(context.Context) ([]tryevent.Open, error) {
raw, err := call(openTriesTool, map[string]any{})
if err != nil {
return nil, err
}
var o struct {
Tries *[]tryevent.Open `json:"tries"`
}
if err := json.Unmarshal(raw, &o); err != nil || o.Tries == nil {
return nil, fmt.Errorf("the answer of %s is not {tries}: %v", openTriesTool, err)
}
return *o.Tries, nil
}
}
@@ -11,6 +11,7 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
@@ -73,7 +74,10 @@ func (w *previewWatch) ensure() error {
}
})
}()
return w.ix.RecomputeCandidates(w.ctx)
if err := w.ix.RecomputeCandidates(w.ctx); err != nil {
return err
}
return w.ix.WriteTryCandidates(w.ctx)
}
// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the
@@ -90,3 +94,44 @@ func (w *previewWatch) Written(m delivered) {
}
_ = m.Ack()
}
// tryEventSubject is the subject of an event of the checker's stream checks about a try.
func tryEventSubject(event string) string {
return "mesh.stream." + delivery.CheckerModule + ".event." + event
}
// heardChecks takes one event of the checker's stream in: a preview written (Written), or a whole try's start or end, from
// which the candidate of its group is written or deleted (index.Indexer.StartTry, EndTry). A failure stays unacknowledged, and
// the server delivers it again; a body that is no try event is terminated.
func (w *previewWatch) heardChecks(m delivered) {
var event string
switch m.Subject() {
case tryEventSubject(tryevent.Started):
event = tryevent.Started
case tryEventSubject(tryevent.Ended):
event = tryevent.Ended
default:
w.Written(m)
return
}
e, err := tryevent.Decode(event, m.Data())
if err != nil {
w.logger.Printf("%s: not a try event, dropped: %v", m.Subject(), err)
_ = m.Term()
return
}
if event == tryevent.Started {
var ms []index.TryMember
for _, x := range e.Members {
ms = append(ms, index.TryMember{Repository: x.Repository, Head: x.Head, Trunk: x.Trunk})
}
err = w.ix.StartTry(w.ctx, e.ID, ms)
} else {
err = w.ix.EndTry(w.ctx, e.ID)
}
if err != nil {
w.logger.Printf("%s of %s: left for redelivery: %v", event, e.ID, err)
return
}
_ = m.Ack()
}
@@ -19,6 +19,8 @@ type takerIn interface {
TakeInPendingByHand(ctx context.Context, key, by, why string) index.TakenIn
DropPending(ctx context.Context, key, by, why string) index.TakenIn
TakeInMerge(ctx context.Context, repository, merge string, seq uint64, by, why string) index.TakenIn
DropTryCandidate(ctx context.Context, id, by, why string) index.TakenIn
RecoverTriesByHand(ctx context.Context, by, why string) index.TakenIn
}
// takenIn answers one value of the takeins state (the tool indexer_take_in, issue 1945): a value not yet answered takes the
@@ -37,7 +39,13 @@ func takenIn(ctx context.Context, ix takerIn, kv jetstream.KeyValue, e jetstream
return
}
asked, r := "take-in", index.TakenIn{}
if a.Merge != "" {
if a.RecoverTries {
asked, r = "recovery of the tries", ix.RecoverTriesByHand(ctx, a.By, a.Why)
a.Module = "tries"
} else if a.TryCandidate != "" && a.Drop {
asked, r = "drop of a try's candidate", ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
a.Module = a.TryCandidate
} else if a.Merge != "" {
asked, r = "hand take-in of a merge", ix.TakeInMerge(ctx, a.MergeRepository, a.Merge, a.Sequence, a.By, a.Why)
a.Module = r.Module
} else if a.Pending != "" && a.Drop {
@@ -46,7 +46,19 @@ func run(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
}
v, err := judge.Request(r)
// The request's seat definitions (ADR 0636 §4): the composer's request type of this build does not decode them.
var seats struct {
Seats []json.RawMessage `json:"seats"`
}
if err := json.Unmarshal(raw, &seats); err != nil {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
}
files := make([][]byte, len(seats.Seats))
for i, s := range seats.Seats {
files[i] = s
}
v, err := judge.RequestWith(r, files)
if err != nil {
fmt.Fprintln(stderr, "mesh-judge:", err)
return 1
@@ -13,6 +13,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
@@ -82,6 +83,9 @@ type Indexer struct {
// repository it replaces by a directory (fingerprint.ReplacedSiblings), which the version line takes by their
// content; nil reads none.
GoMod func(ctx context.Context, repository, sha, dir string) ([]byte, error)
// ReadFile reads one file of a repository at a commit (gitea.gitea_get_file); a file the commit does not hold is nil.
// With Files it reads the seat definitions a revision carries (seatsof.go); nil reads none.
ReadFile func(ctx context.Context, repository, sha, path string) ([]byte, error)
// Now is the clock of a taken value; nil is time.Now.
Now func() time.Time
Builds *Builds
@@ -118,11 +122,17 @@ type Indexer struct {
fillAgain atomic.Bool // a kick arrived while it ran
atHeads map[string]map[string]delivery.PublishedPackages // a member head's published Go modules, by repository@head (candidatePublished)
// OwnBuild answers the fingerprint of the build this program runs, from the node-engine's seat; nil knows none.
OwnBuild func(ctx context.Context) (string, error)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
ownFP string // the fingerprint of the build this program runs, asked at the start pass (learnOwnBuild)
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
OwnBuild func(ctx context.Context) (string, error)
underSeats bool // a dry run inside judge.WithSeats: its judge passes do not hold the seat list (verdicts.go)
judgeMu sync.Mutex // one judge pass at a time (verdicts.go)
ownMu sync.Mutex // guards ownFP
ownFP string // the fingerprint of the build this program runs, fixed once at the start (FixOwnBuild)
ownFixed bool // FixOwnBuild asked already: it asks once
ownSince time.Time // when the build was asked and not known, with ownWhy
ownWhy string
// OwnBuildState keeps which build this process knows it runs, or since when and why not (FixOwnBuild); nil keeps none.
OwnBuildState Store
judgePending atomic.Bool // the start's judge pass failed: judged again at the next message (verdicts.go)
// Message asks the forge bridge for the merged message of a merge, repository and merge commit in, the message whole out (a
// pending value keeps a reference, not a copy: issue 1585); an error is the bridge's words. Nil asks none.
Message func(ctx context.Context, repository, merge string) ([]byte, error)
@@ -131,9 +141,16 @@ type Indexer struct {
// Pending is the state of merges held for the indexer their revision names (pending.go, novox/hq ADR 0633 §1); nil
// takes every merge in at once, as before the take-in step.
Pending Store
// SayTakeIn publishes a take-in event on the indexer's own stream take-in (ADR 0633 §5); nil says none.
// SayTakeIn publishes a take-in event (take-in.pending, take-in.taken; ADR 0633 §5); nil says none.
SayTakeIn func(ctx context.Context, t takein.TakeIn) error
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
// OpenTries lists the tries that wait or run, from the checker's tool (checker_tries); nil recovers none (trycandidates.go).
OpenTries func(ctx context.Context) ([]tryevent.Open, error)
// TryRecovery keeps how the last recovery of the open tries went (trycandidates.go); nil keeps none.
TryRecovery Store
tryMu sync.Mutex // one try event or recovery at a time (trycandidates.go)
recoveryFailed atomic.Bool // the last recovery of the open tries failed (trycandidates.go)
tries map[string][]TryMember // the whole tries that wait or run, by try id (trycandidates.go); guarded by mu
pendingMu sync.Mutex // one take-in step at a time: a merge, a group's end, the start and a hand act (pending.go)
}
func (ix *Indexer) logf(format string, args ...any) {
@@ -279,6 +296,9 @@ func (ix *Indexer) applyOpen(ctx context.Context, f forgemsg.Forge) error {
return fmt.Errorf("listing the candidates: %w", err)
}
for _, key := range keys {
if IsTryGroup(key) {
continue // a try's candidate is no pull request's: the checker's try owns it (trycandidates.go)
}
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
@@ -244,6 +244,9 @@ func (ix *Indexer) configCandidatesAfterMerge(ctx context.Context, f forgemsg.Fo
}
var errs []error
for _, g := range groups {
if IsTryGroup(g) {
continue // a try's candidate holds no pull request's config declarations (trycandidates.go)
}
raw, _, found, err := ix.Store.Get(ctx, g)
if err != nil || !found {
continue
@@ -85,5 +85,22 @@ func TestTheControlRecords(t *testing.T) {
if _, ok := cands.vals[key]; ok {
t.Error("a closed pull request's declarations stayed")
}
// A group try's candidate (number 0) holds no pull request's declarations: a merge that touches its module writes none
// for it (fails when configCandidatesAfterMerge no longer skips a try's group).
tryPull := delivery.CandidatePull{Head: delivery.Head{Repository: catalogueRepo, Head: h1}, Modules: []string{"avahi"},
Fps: map[string]delivery.CandidateModule{"avahi": {Manifest: delivery.ManifestDigest([]byte(trunk)), None2: NoneNoPreview}}}
tryCand := delivery.Candidate{V: delivery.Version, Group: "try-5", Pulls: []delivery.CandidatePull{tryPull}}
tryCand.HeadsHash, _ = delivery.HeadsHash([]delivery.Head{tryPull.Head})
tryCand.FPSet = delivery.FingerprintSet(tryCand.Pulls)
data, err := delivery.EncodeCandidate(tryCand)
if err != nil {
t.Fatal(err)
}
if err := ix.Store.Create(ctx, "try-5", data); err != nil {
t.Fatal(err)
}
if err := ix.configCandidatesAfterMerge(ctx, mergedMsg(catalogueRepo, map[string]string{"modules/avahi": "t2"}), []string{"avahi"}); err != nil || len(cands.vals) != 0 {
t.Errorf("a merge touching a try's module wrote %v for it (%v), want none", cands.vals, err)
}
})
}
@@ -15,6 +15,8 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-indexer/judge"
"git.novox.be/novox/mesh-delivery/sdk/graph"
)
@@ -70,6 +72,7 @@ type Pending struct {
Ref *PendingRef `json:"ref,omitempty"`
Forge json.RawMessage `json:"forge,omitempty"`
Lost string `json:"lost,omitempty"`
Refused string `json:"refused,omitempty"`
Group string `json:"group,omitempty"`
Ended bool `json:"ended,omitempty"`
Since time.Time `json:"since"`
@@ -336,20 +339,27 @@ func (ix *Indexer) pendingOnBus(ctx context.Context) bool {
return !errors.Is(err, process.ErrNotYet)
}
// ownBuildNow is the build this program runs: the one learnt, else asked of the node-engine now.
func (ix *Indexer) ownBuildNow(ctx context.Context) string {
ix.ownMu.Lock()
fp := ix.ownFP
ix.ownMu.Unlock()
if fp != "" || ix.OwnBuild == nil {
return fp
}
ix.learnOwnBuild(ctx)
// ownBuildNow is the build this program runs, as FixOwnBuild fixed it at the start: empty while unknown, and never asked again.
func (ix *Indexer) ownBuildNow(context.Context) string {
ix.ownMu.Lock()
defer ix.ownMu.Unlock()
return ix.ownFP
}
// sayOwnBuildUnknown says, for a merge this process cannot take in because it does not know its own build (ADR 0633 §1), a
// take-in.pending that names why and since when, so the watchman's take-in-waits bound covers it. The merge is not written to
// pending; the stream delivers it again to the process that knows its build.
func (ix *Indexer) sayOwnBuildUnknown(ctx context.Context, seq uint64, f forgemsg.Forge) {
ix.ownMu.Lock()
since, why := ix.ownSince, ix.ownWhy
ix.ownMu.Unlock()
if since.IsZero() { // never fixed: the start did not run (no engine wired)
since, why = ix.now(), "the build this program runs was never asked"
}
ix.sayTakeIn(ctx, takein.TakeIn{Kind: takein.TakeInPending, Key: takein.PendingKey(seq), What: takein.TakeInRevision,
Merges: []takein.Merge{{Repository: f.Repository, Number: f.Number, Merge: f.Merge}}, Since: since, Unknown: why})
}
// sayTakeIn publishes a take-in event and keeps what it said on the entries, so indexer_pending reads the last event of
// each wait from the state. A publish that fails is said in the log: the next evaluation says it again.
func (ix *Indexer) sayTakeIn(ctx context.Context, t takein.TakeIn) bool {
@@ -398,6 +408,7 @@ func (ix *Indexer) Arrive(ctx context.Context, seq uint64, f forgemsg.Forge) (Ap
}
own := ix.ownBuildNow(ctx)
if own == "" {
ix.sayOwnBuildUnknown(ctx, seq, f)
return Applied{}, "", ErrOwnBuildUnknown
}
ix.pendingMu.Lock()
@@ -688,7 +699,7 @@ func (ix *Indexer) dry() *dryRun {
d := &dryRun{}
x := &Indexer{Manifests: ix.Manifests, Packages: ix.Packages, JudgeID: ix.JudgeID, Files: ix.Files, GoSum: ix.GoSum,
GoMod: ix.GoMod, Now: ix.Now, Builds: ix.Builds, Named: ix.Named, Trunk: ix.Trunk, Read: ix.Read, Contains: ix.Contains,
ReadOpen: nil, OwnBuild: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil}
ReadOpen: nil, OwnBuild: nil, OwnBuildState: nil, Message: ix.Message, SayRefused: nil, SayAccepted: nil, SayTakeIn: nil, Pending: nil, ReadFile: ix.ReadFile, underSeats: true}
x.Logf = func(format string, args ...any) { ix.logf("take-in, computed and not written: "+format, args...) }
x.Store, d.candidates = over(ix.Store)
x.Modules, d.modules = over(ix.Modules)
@@ -721,16 +732,29 @@ func (ix *Indexer) dry() *dryRun {
// is left out, as its take-in will drop it.
func (ix *Indexer) computeOver(ctx context.Context, ready []PendingEntry) (string, *dryRun, error) {
d := ix.dry()
for _, e := range ready {
if e.Covered {
continue
}
if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil {
if errors.Is(err, ErrUnreadable) {
// The manifests are judged against the seat definitions the revision carries (seatsof.go, ADR 0633 §1), so the indexer
// it names is not refused for a verb or an event the built-in list lacks.
seats, err := ix.revisionSeats(ctx, ready)
if err != nil {
return "", d, fmt.Errorf("reading the seat definitions of %s: %w", ready[0].Key, err)
}
var applyErr error
judge.WithSeats(seats, func() {
for _, e := range ready {
if e.Covered {
continue
}
return "", d, fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
if _, err := d.ix.ApplyMessage(ctx, e.Forge); err != nil {
if errors.Is(err, ErrUnreadable) {
continue
}
applyErr = fmt.Errorf("computing the take-in of %s (%s#%d): %w", e.Key, e.Forge.Repository, e.Forge.Number, err)
return
}
}
})
if applyErr != nil {
return "", d, applyErr
}
key, _ := delivery.ModulesKey(delivery.IndexerModule)
value, _, found, err := d.ix.Modules.Get(ctx, key)
@@ -915,24 +939,33 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own
return fmt.Errorf("the indexer's key of the module graph: %w", err)
}
}
// Why the manifest of the indexer the revision names was refused, when it was (judged with the seat definitions the
// revision carries): said with the wait, as a merge the forge bridge could not give is (holdLost).
refused := ""
if value, _, found, err := d.ix.Modules.Get(ctx, key); err == nil && found {
if m, err := delivery.DecodeModule(value); err == nil && m.Manifest != "" {
if vk, err := delivery.VerdictsKey(m.Manifest); err == nil {
if err := copyKey(ctx, d.verdicts, ix.Verdicts, vk); err != nil && !errors.Is(err, process.ErrNotYet) {
return fmt.Errorf("the verdict on the indexer's manifest: %w", err)
}
if vv, _, ok, err := d.ix.Verdicts.Get(ctx, vk); err == nil && ok {
if v, err := delivery.DecodeManifestVerdict(vv); err == nil && v.Refused() {
refused = strings.Join(v.Problems, "; ")
}
}
}
}
}
t := revisionWait(takein.TakeInPending, ready, names)
t.Refused = refused
var ended, unmarked []PendingEntry
said, changed := false, false
for _, e := range ready {
switch {
case e.Value.Said == t.Key && e.Value.SaidKind == takein.TakeInPending:
said = true
changed = changed || e.Value.WaitsFor != names
if e.Value.WaitsFor != names {
changed = changed || e.Value.WaitsFor != names || e.Value.Refused != refused
if e.Value.WaitsFor != names || e.Value.Refused != refused {
unmarked = append(unmarked, e)
}
case e.Value.Said != "" && e.Value.SaidKind == takein.TakeInPending:
@@ -947,13 +980,16 @@ func (ix *Indexer) holdFor(ctx context.Context, ready []PendingEntry, names, own
}
if !said || changed {
ix.logf("take-in: %d merges, oldest %s, name the indexer build %s, not %s, which runs: they wait for it", len(ready), t.Key, names, own)
if refused != "" {
ix.logf("take-in: the manifest of the indexer they name is refused, judged with the seat definitions they carry, so no such indexer runs: %s", refused)
}
if !ix.sayTakeIn(ctx, t) {
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own })
ix.markSaid(ctx, unmarked, nil, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
unmarked = ready
}
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter = names, own })
ix.markSaid(ctx, unmarked, &t, func(p *Pending) { p.WaitsFor, p.LastWriter, p.Refused = names, own, refused })
return nil
}
@@ -1015,6 +1051,9 @@ func waitOf(entries []PendingEntry, what, behind string) takein.PendingWait {
if e.Value.Lost != "" {
pw.Lost = append(pw.Lost, e.Key)
}
if e.Value.Refused != "" {
pw.Refused = e.Value.Refused
}
if e.Value.Said != "" && e.Value.SaidAt.After(pw.SaidAt) {
pw.Said, pw.SaidAt = "take-in."+e.Value.SaidKind, e.Value.SaidAt
}
@@ -12,6 +12,7 @@ import (
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
"git.novox.be/novox/mesh-sdk/go/buscompose"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/process"
)
@@ -60,6 +61,8 @@ func newTakeInBed(t *testing.T) *takeInBed {
t.Fatalf("the bed's modules have no fingerprint")
}
ix.OwnBuild = func(context.Context) (string, error) { return own, nil }
ix.OwnBuildState = newMem()
ix.FixOwnBuild(context.Background()) // fixed once, at the start
return b
}
@@ -130,11 +133,26 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
ctx := context.Background()
t.Run("T6 a merge while the own build is unknown writes nothing", func(t *testing.T) {
b := newTakeInBed(t)
b.ix.ownFP, b.ix.ownFixed = "", false // this process is started again and asks the engine, which does not answer
b.ix.OwnBuild = func(context.Context) (string, error) { return "", errors.New("the engine does not answer") }
b.ix.FixOwnBuild(ctx)
b.said = nil
_, _, err := b.ix.Arrive(ctx, 1, b.merge(20, sha('b'), "tree-avahi-2", ""))
if !errors.Is(err, ErrOwnBuildUnknown) || len(b.keys()) != 0 || b.commit(t, "avahi") == sha('b') {
t.Errorf("err %v, pending %v, avahi at %s: want ErrOwnBuildUnknown and nothing written", err, b.keys(), b.commit(t, "avahi"))
}
// The merge is said pending with why and since when (so the watchman's bound covers it), and indexer_pending says the
// build is unknown, from the own-build state.
if len(b.said) != 1 || b.said[0].Kind != takein.TakeInPending || b.said[0].Key != takein.PendingKey(1) || !strings.Contains(b.said[0].Unknown, "does not answer") ||
b.said[0].Since.IsZero() || b.said[0].WaitsFor != "" {
t.Errorf("said %+v, want one take-in.pending of m.1 saying why the own build is unknown", b.said)
} else if _, err := takein.EncodeTakeIn(b.said[0]); err != nil {
t.Errorf("the unknown-build event is refused by its own contract: %v", err)
}
answer, err := PendingTools(b.pending, b.ix.OwnBuildState, nil, nil)[0].Run(map[string]any{})
if a, _ := answer.(takein.PendingAnswer); err != nil || !strings.Contains(a.OwnBuildUnknown, "does not answer") || a.OwnBuildSince.IsZero() {
t.Errorf("indexer_pending = %+v %v, want the build unknown with why and since", answer, err)
}
// Before the pending state is on the bus nothing waits: the start asks no build, and the merge is taken in at once,
// as before ADR 0633, though the build is unknown (TestIndexerWhole found the start waiting on it for ever).
b.ix.Pending = &memStore{unbound: process.ErrNotYet}
@@ -188,6 +206,63 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
t.Errorf("said %v, want %v", b.events(), want)
}
})
t.Run("K1 the indexer a revision names is judged with the seat definitions the revision carries", func(t *testing.T) {
// A successor's manifest invokes a verb of the node-engine's seat that the built-in list lacks and the revision's
// definitions add (ADR 0633 §1, §6): refused without them, with its reason said on the wait and in indexer_pending;
// accepted with them, and the revision waits for the build it names.
const verb, contracts = "manifest-reads", "modules/seat-contracts"
for _, row := range []struct {
name string
defined bool
refused bool
}{{"definitions the revision does not carry", false, true}, {"definitions the revision carries", true, false}} {
b := newTakeInBed(t)
f := b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2")
f.Manifests = map[string]string{}
for dir, raw := range b.base.Manifests {
f.Manifests[dir] = raw
}
dir := "modules/" + delivery.IndexerModule
f.Manifests[dir] = strings.Replace(f.Manifests[dir], `"module": "`+delivery.IndexerModule+`"`,
`"module": "`+delivery.IndexerModule+`", "invokes": ["seat:node-engine.`+verb+`"]`, 1)
if row.defined {
f.Trees[contracts] = "tree-seat-contracts"
f.Locks = map[string]map[string]string{}
for d, l := range b.base.Locks {
f.Locks[d] = l
}
f.Locks[contracts] = map[string]string{}
f.Manifests[contracts] = `{"module":"seat-contracts","version":"1","build":{"artifacts":[{"name":"go-seats","kind":"package","language":"go","paths":["go"],"generated-from":"."}]}}`
b.ix.Files = func(_ context.Context, repository, commit string, dirs []string) (map[string]map[string]string, error) {
if repository != f.Repository || commit != f.Merge || !slices.Equal(dirs, []string{contracts + "/seats"}) {
t.Errorf("listed %v of %s at %s", dirs, repository, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1", "node-engine/fixtures/a.json": "2"}}, nil
}
b.ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case contracts + "/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil
case contracts + "/parts/reads.json":
return []byte(`{"format":1,"part":"reads","verbs":[{"name":"` + verb + `"}]}`), nil
}
return nil, nil
}
}
b.arrive(t, 2, f)
waits := PendingWaits(readPending(t, b))
refused := ""
if len(b.said) > 0 {
refused = b.said[0].Refused
}
if got := strings.Contains(refused, `serves no verb "`+verb+`"`); got != row.refused || (len(waits) == 1 && waits[0].Refused != refused) {
t.Errorf("%s: said %q, indexer_pending %+v: want a refusal naming the verb %v", row.name, refused, waits, row.refused)
}
if _, ok := catalogue.SeatNamed("node-engine"); !ok || judgeSeatsHave("node-engine", verb) {
t.Errorf("%s: the seats are not put back after the dry run", row.name)
}
}
})
t.Run("T10 the named indexer takes the revision in at its start; a gathered group stays", func(t *testing.T) {
b := newTakeInBed(t)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
@@ -196,31 +271,36 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
g.Members = []delivery.Head{{Repository: "novox/mesh-sdk", Number: 30, Head: g.Head}, {Repository: "novox/mesh-host", Number: 4, Head: sha('9')}}
b.arrive(t, 5, g)
named := b.fp(t, delivery.IndexerModule)
// The new indexer starts; its build is not known at first: a hand ask is answered meanwhile (R5), and the start waits.
next := &Indexer{}
*next = Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum}
next.toolchains = b.ix.toolchains
asked := 0
next.OwnBuild = func(context.Context) (string, error) {
if asked++; asked < 3 { // the hand ask's and the first start's
return "", errors.New("not yet")
}
return named, nil
// The new indexer starts; its build is not known (the engine does not answer at its start): a hand ask is answered
// meanwhile (R5), and the start waits for nothing. It is never asked again, so it stays unknown until it is started again.
mk := func(ask func(context.Context) (string, error)) *Indexer {
n := &Indexer{Store: b.ix.Store, Modules: b.ix.Modules, Manifests: b.ix.Manifests, Graph: b.ix.Graph, Verdicts: b.ix.Verdicts,
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: b.ix.Message, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf, GoSum: b.ix.GoSum}
n.toolchains = b.ix.toolchains
n.OwnBuild = ask
n.FixOwnBuild(ctx)
return n
}
asked := 0
next := mk(func(context.Context) (string, error) { asked++; return "", errors.New("not yet") })
if r := next.TakeInPendingByHand(ctx, "m.nothing", "lead@shanks", "a test"); r.Stored || !strings.Contains(r.Answer, "nothing waits") {
t.Errorf("a hand ask while the own build is unknown: %+v", r)
}
// The first start, its build not known, waits for nothing (no sleep, no timer): it answers at once, takes nothing
// in, and the forge consumer starts after it.
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 2 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("a start with its build unknown: %q %v, asked %d, pending %v: want it to answer at once and take nothing in", words, err, asked, b.keys())
if _, words, err := next.TakeInAtStart(ctx); err != nil || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("a start with its build unknown: %q %v, pending %v: want it to answer at once and take nothing in", words, err, b.keys())
}
// The engine answers now, and the same process does not ask it again: it learnt nothing it did not know at its start.
next.OwnBuild = func(context.Context) (string, error) { asked++; return named, nil }
if _, words, err := next.TakeInAtStart(ctx); err != nil || asked != 1 || len(b.keys()) != 2 || !strings.Contains(words, "not known") {
t.Fatalf("the window of a replacement: %q %v, asked %d, pending %v: want the build fixed at the start, never learnt later", words, err, asked, b.keys())
}
// The module started again fixes the build at its start and takes the revision in.
next = mk(func(context.Context) (string, error) { return named, nil })
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
}
if asked < 3 || !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
t.Errorf("start: asked %d, pending %v, avahi at %s: want the revision taken in, the group left", asked, b.keys(), b.commit(t, "avahi"))
if !reflect.DeepEqual(b.keys(), []string{takein.PendingKey(5)}) || b.commit(t, "avahi") != sha('c') {
t.Errorf("start: pending %v, avahi at %s: want the revision taken in, the group left", b.keys(), b.commit(t, "avahi"))
}
})
t.Run("T8 T9 T11 a group gathered until its end, a merge behind it, one alone, an end again", func(t *testing.T) {
@@ -394,7 +474,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
b.arrive(t, 2, b.merge(21, sha('c'), "tree-avahi-3", "tree-indexer-2"))
b.arrive(t, 3, b.merge(22, sha('d'), "tree-avahi-4", "tree-indexer-2"))
var answer takein.PendingAnswer
for _, tool := range PendingTools(b.pending, nil, nil) {
for _, tool := range PendingTools(b.pending, nil, nil, nil) {
if tool.Name == PendingTool {
out, err := tool.Run(map[string]any{})
if err != nil {
@@ -464,6 +544,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
Packages: b.ix.Packages, Builds: b.ix.Builds, Pending: b.pending, Message: bridgeDown, SayTakeIn: b.ix.SayTakeIn, Logf: b.ix.Logf,
GoSum: b.ix.GoSum, OwnBuild: func(context.Context) (string, error) { return named, nil }}
next.toolchains = b.ix.toolchains
next.FixOwnBuild(ctx)
b.said = nil
if _, _, err := next.TakeInAtStart(ctx); err != nil {
t.Fatal(err)
@@ -520,7 +601,7 @@ func TestTakeIn(t *testing.T) { // 1585/T3-T13 (ADR 0633)
named := map[string]bool{}
for _, f := range []string{"Store", "Modules", "Manifests", "Predefined", "ConfigCandidates", "Taken", "Verdicts", "OldSections",
"JudgeID", "SayRefused", "SayAccepted", "Packages", "Graph", "Files", "GoSum", "GoMod", "Now", "Builds", "Named", "Trunk",
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn"} {
"Read", "Contains", "Logf", "ReadOpen", "OwnBuild", "Message", "Pending", "SayTakeIn", "ReadFile", "OpenTries", "TryRecovery", "OwnBuildState"} {
named[f] = true
}
ty := reflect.TypeOf(Indexer{})
@@ -540,3 +621,19 @@ func mustReadPending(t *testing.T, b *takeInBed) []PendingEntry {
}
return entries
}
// judgeSeatsHave says whether the catalogue's seat list holds the verb on the seat now.
func judgeSeatsHave(seat, verb string) bool {
s, _ := catalogue.SeatNamed(seat)
return slices.ContainsFunc(s.Serves, func(v catalogue.Verb) bool { return v.Name == verb })
}
// readPending is the bed's pending state as indexer_pending reads it.
func readPending(t *testing.T, b *takeInBed) []PendingEntry {
t.Helper()
entries, err := ReadPending(context.Background(), b.pending, nil)
if err != nil {
t.Fatal(err)
}
return entries
}
@@ -2,9 +2,13 @@ package index
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-indexer/takein"
stdio "git.novox.be/novox/mesh-sdk/go"
)
@@ -12,7 +16,7 @@ import (
// PendingTools are the indexer's tools of what waits for a take-in (novox/hq ADR 0633 §5): indexer_pending reads the
// pending state, and indexer_take_in_pending and indexer_drop_pending steer it by an ask in takeins, as indexer_take_in
// does. pending is the pending state as the tools bundle reads it; ask writes an ask and waits for the indexer's answer.
func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
func PendingTools(pending, ownBuild Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
str := func(args map[string]any, k string) string { s, _ := args[k].(string); return s }
hand := func(drop bool) func(map[string]any) (any, error) {
return func(args map[string]any) (any, error) {
@@ -49,7 +53,8 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk,
"merges, since when it waits, the modules it holds back (holds), the merges whose message the forge bridge's stream no " +
"longer holds (lost: they are not taken in; indexer_drop_pending drops them) and the last take-in event said of it. Empty waits: " +
"every merge is taken in. The watchman raises take-in-waits past 60 minutes for a revision, 30 for a group; " +
"indexer_take_in_pending and indexer_drop_pending steer a wait.",
"indexer_take_in_pending and indexer_drop_pending steer a wait. own-build-unknown, when set, says the indexer cannot learn the build it " +
"runs (since when): it fixes it once at its start, so every merge waits (a take-in.pending that says so) until the module is started again.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
var unread []string
@@ -59,7 +64,17 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk,
if err != nil {
return nil, err
}
return takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}, nil
answer := takein.PendingAnswer{Waits: PendingWaits(entries), Unread: unread}
// Whether this indexer knows the build it runs: when it does not, every merge waits, and this says since when and why.
if ownBuild != nil {
if v, _, found, err := ownBuild.Get(context.Background(), OwnBuildKey); err == nil && found {
var o OwnBuild
if json.Unmarshal(v, &o) == nil && !o.Known {
answer.OwnBuildUnknown, answer.OwnBuildSince = o.Why, o.Since
}
}
}
return answer, nil
}},
{Name: TakeInPendingTool,
Description: "Take a wait in by hand with the running indexer and its judge (ADR 0633 §5): the revision under its m. key, " +
@@ -92,3 +107,94 @@ func PendingTools(pending Store, ask func(key string, value []byte) (TakeInAsk,
Run: merge},
}
}
// TryCandidateTools are the tools of a group try's candidate (issue 1921, trycandidates.go): indexer_try_candidates reads the
// candidates the indexer holds for tries (the try, its heads, each member's modules and whether each has its fingerprint),
// and indexer_drop_try_candidate drops one by an ask in takeins. candidates is the candidates state as the tools bundle reads it.
func TryCandidateTools(candidates, recovery Store, ask func(key string, value []byte) (TakeInAsk, error), now func() time.Time) []stdio.Tool {
return []stdio.Tool{
{Name: TryCandidatesTool,
Description: "The candidates the indexer holds for group tries (a try of several repositories at exact commits, mesh-delivery-checker try): " +
"per try its heads hash, each member's repository and commit, the modules the try touches and, per module, its manifest digest, its " +
"fingerprint or why it has none yet (a merge preview not yet written). The checker's manifest validator and the node-engines' " +
"compose-candidate read this candidate. Read-only; an empty answer means no whole try waits or runs.",
Input: map[string]any{"type": "object", "properties": map[string]any{}},
Run: func(map[string]any) (any, error) {
ctx := context.Background()
keys, err := candidates.Keys(ctx)
if err != nil {
return nil, err
}
var out []delivery.Candidate
for _, k := range keys {
if !IsTryGroup(k) {
continue
}
v, _, found, err := candidates.Get(ctx, k)
if err != nil || !found {
continue
}
if c, err := delivery.DecodeCandidate(v); err == nil {
out = append(out, c)
}
}
// How the recovery of the open tries went: loud when it is not done (since when, why).
said := map[string]any{"done": false, "why": "no recovery has been written yet"}
if v, _, found, err := recovery.Get(ctx, TryRecoveryKey); err == nil && found {
var r TryRecovery
if json.Unmarshal(v, &r) == nil {
said = map[string]any{"done": r.Done}
if !r.Done {
said["why"], said["since"] = r.Why, r.Since
said["says"] = "recovery not done, since " + r.Since.Format(time.RFC3339) + ": " + r.Why
}
}
}
return map[string]any{"tries": out, "recovery": said}, nil
}},
{Name: RecoverTriesTool,
Description: "Run the recovery of the open tries now: the indexer lists them through the checker's checker_tries, writes the candidate of each " +
"whole try that waits or runs and deletes the candidate of a try no longer listed. It runs by itself at the indexer's start; when it failed " +
"(indexer_try_candidates says recovery not done, since when and why) this runs it again. A hand act: who and why are said in the journal.",
Input: map[string]any{"type": "object", "required": []string{"why"}, "properties": map[string]any{
"why": map[string]any{"type": "string", "description": "why it is run now, in plain words"}}},
Run: func(args map[string]any) (any, error) {
why, _ := args["why"].(string)
if strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("say why: a hand act is said in the journal")
}
data, err := json.Marshal(TakeInAsk{V: 1, RecoverTries: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("tries.recover."+now().UTC().Format("20060102T150405.000000000"), data)
if err != nil {
return nil, err
}
return a.Result, nil
}},
{Name: DropTryCandidateTool,
Description: "Drop a group try's candidate by hand (the indexer deletes it itself when the try ends): for a candidate left for a try that " +
"is gone, or to make the indexer write it again from the previews at the try's next change. A hand act: who and why are said in the journal.",
Input: map[string]any{"type": "object", "required": []string{"try", "why"}, "properties": map[string]any{
"try": map[string]any{"type": "string", "description": "the try's id, try-<digits>"},
"why": map[string]any{"type": "string", "description": "why it is dropped, in plain words"}}},
Run: func(args map[string]any) (any, error) {
id, _ := args["try"].(string)
why, _ := args["why"].(string)
id = strings.TrimSpace(id)
if !IsTryGroup(id) || strings.TrimSpace(why) == "" {
return nil, fmt.Errorf("name the try (try-<digits>) and say why")
}
data, err := json.Marshal(TakeInAsk{V: 1, TryCandidate: id, Drop: true, Why: strings.TrimSpace(why), By: stdio.CallerOf(args), At: now().UTC()})
if err != nil {
return nil, err
}
a, err := ask("try-candidate.drop."+id, data)
if err != nil {
return nil, err
}
return a.Result, nil
}},
}
}
@@ -72,7 +72,10 @@ func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, d
if !ix.PutPreview(key, value, deleted) {
return nil
}
return ix.RecomputeCandidates(ctx)
if err := ix.RecomputeCandidates(ctx); err != nil {
return err
}
return ix.WriteTryCandidates(ctx) // a try that waited for this preview gets its candidate
}
// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer
@@ -3,11 +3,16 @@ package index
import (
"context"
"encoding/json"
"errors"
"os"
"runtime"
"slices"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-delivery/sdk/tryevent"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/fingerprint"
@@ -182,6 +187,223 @@ func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsW
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview {
t.Errorf("a new head: %+v, want no preview", got)
}
// Issue 1921: a whole group try has no pull request, yet the indexer writes its candidate from the checker's try events and
// the previews its fetch wrote (number 0, the heads hash of its members), keeps it apart from the pull requests' candidates,
// and deletes it when the try ends. Its manifests are judged with the seat definitions its members carry, in one locked
// pass, whether the contracts module's manifest changed or only its seat.json did (the wait is bounded: a lock held twice
// would show as a failure here, not as a stuck run).
for _, row := range []struct {
name string
unchanged bool // the contracts module's module.json is the trunk's: only its definitions changed
}{
{name: "a changed manifest that carries definitions"},
{name: "an unchanged manifest that carries definitions", unchanged: true},
} {
t.Run("a group try's candidate: "+row.name, func(t *testing.T) {
ix, mods, cands := previewBed(t, true)
ctx := context.Background()
ix.Verdicts = newMem()
var changed map[string]any
if err := json.Unmarshal([]byte(catalogueManifest(t, "avahi")), &changed); err != nil {
t.Fatal(err)
}
changed["build"] = map[string]any{"artifacts": []any{map[string]any{"name": "go-seats", "kind": "package", "language": "go", "paths": []any{"go"}, "generated-from": "defs"}}}
raw, _ := json.Marshal(changed)
pv := avahiPreview(t, trunkNow, h1)
pm := delivery.PreviewModule{Tree: "tree-avahi", Manifest: delivery.ManifestDigest(raw), Locks: map[string]string{"go.sum": avahiGoSum}}
if row.unchanged {
m, found := moduleOf(t, mods, "avahi")
_ = found
pm.Manifest = m.Manifest
key, _ := delivery.ManifestsKey(m.Manifest)
data, _ := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(raw)})
if _, _, _, err := ix.Manifests.Get(ctx, key); err != nil {
t.Fatal(err)
}
_ = ix.Manifests.(*memStore).Update(ctx, key, data, ix.Manifests.(*memStore).revs[key])
}
pv.Modules["modules/avahi"] = pm
ix.Read = func(_ context.Context, _, _, dir string) (RepoRead, error) {
return RepoRead{Manifests: map[string]string{dir: string(raw)}}, nil
}
listed := 0
ix.Files = func(_ context.Context, _, commit string, dirs []string) (map[string]map[string]string, error) {
listed++
if commit != h1 || !slices.Equal(dirs, []string{"modules/avahi/defs/seats"}) {
t.Errorf("listed %v at %s, want the member's head", dirs, commit)
}
return map[string]map[string]string{dirs[0]: {"node-engine/seat.json": "1"}}, nil
}
ix.ReadFile = func(_ context.Context, _, _, p string) ([]byte, error) {
switch p {
case "modules/avahi/defs/seats/node-engine/seat.json":
return []byte(`{"format":1,"seat":"node-engine","scope":"node","includes":["reads"]}`), nil
case "modules/avahi/defs/parts/reads.json":
return []byte(`{"format":1,"part":"reads","verbs":[{"name":"try-verb"}]}`), nil
}
return nil, nil
}
started := `{"id":"try-7"}`
_ = started
members := []TryMember{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}
held := func() []string { k, _ := cands.Keys(ctx); return k }
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 0 {
t.Fatalf("a try before its preview: %v, candidates %v", err, held())
}
done := make(chan struct{})
go func() { defer close(done); applyPreview(t, ix, pv) }()
select {
case <-done:
case <-time.After(10 * time.Second):
panic("writing the try's candidate hangs: the seat lock is held twice (a locked pass cannot be left by t.Fatal: the lock stays held)")
}
if listed == 0 {
t.Error("the member's definitions were not read")
}
key, _ := delivery.CandidatesKey("try-7")
rawc, _, found, _ := cands.Get(ctx, key)
if !found {
t.Fatalf("no candidate for the try after its preview: %v", held())
}
c, err := delivery.DecodeCandidate(rawc)
pull := delivery.CandidatePull{}
if len(c.Pulls) == 1 {
pull = c.Pulls[0]
}
if row.unchanged {
if err != nil || c.Group != "try-7" || len(pull.Modules) != 0 {
t.Fatalf("candidate = %+v, %v, want the try's group with no module touched", c, err)
}
} else if err != nil || c.Group != "try-7" || pull.Number != 0 || pull.Head.Head != h1 || pull.Fps["avahi"].Manifest == "" { // no fingerprint asserted: this mesh-sdk pin does not read generated-from yet
t.Fatalf("candidate = %+v, %v, want one pull of number 0 with avahi's manifest", c, err)
}
if err := ix.Apply(ctx, forgemsg.Forge{Kind: forgemsg.ForgeOpen}); err != nil || len(held()) != 1 {
t.Errorf("the open message left %v (%v), want the try's candidate alone", held(), err)
}
// The tools: one reads the candidates held for tries, one drops by an ask the indexer answers.
asks := func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
r := ix.DropTryCandidate(ctx, a.TryCandidate, a.By, a.Why)
a.Result = &r
return a, err
}
recovery := newMem()
ix.TryRecovery = recovery
tools := TryCandidateTools(cands, recovery, asks, time.Now)
if out, err := tools[0].Run(map[string]any{}); err != nil || len(out.(map[string]any)["tries"].([]delivery.Candidate)) != 1 {
t.Errorf("%s = %v, %v, want the try's candidate", tools[0].Name, out, err)
}
if _, err := tools[2].Run(map[string]any{"try": "feat/x", "why": "w"}); err == nil {
t.Error("a branch was dropped as a try")
}
if out, err := tools[2].Run(map[string]any{"try": "try-7", "why": "by hand"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 {
t.Errorf("drop = %v, %v, candidates %v", out, err, held())
}
// The start recovered from the checker's tool: an open whole try is written again, one the checker no longer
// lists is deleted, and a try narrowed to a level is none.
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}},
{ID: "try-8", Whole: false, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h2, Trunk: trunkNow}}}}, nil
}
if err := ix.RecoverTries(ctx); err != nil || len(held()) != 1 {
t.Fatalf("recovered: %v, candidates %v, want try-7's alone", err, held())
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
if err := ix.RecoverTries(ctx); err != nil || len(held()) != 0 {
t.Errorf("recovered with no open try: %v, candidates %v, want none", err, held())
}
// A recovery whose call fails is loud and deletes nothing: the candidate stays, indexer_try_candidates says recovery not
// done since when and why, and the next try event runs the recovery again, which clears it; the tool runs it by hand.
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 {
t.Fatalf("started: %v, candidates %v", err, held())
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
return nil, errors.New("checker_tries does not answer")
}
if err := ix.RecoverTries(ctx); err == nil || len(held()) != 1 {
t.Fatalf("a failed call: %v, candidates %v, want the error and the candidate kept", err, held())
}
if out, err := tools[0].Run(map[string]any{}); err != nil || !strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "recovery not done, since") ||
!strings.Contains(out.(map[string]any)["recovery"].(map[string]any)["says"].(string), "checker_tries does not answer") {
t.Errorf("%s = %v, %v, want recovery not done with since and why", tools[0].Name, out, err)
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
if err := ix.StartTry(ctx, "try-11", members); err != nil || len(held()) != 1 || held()[0] != "try-11" {
t.Fatalf("a try event after a failed recovery: %v, candidates %v, want the recovery run (try-7 not listed, gone) and try-11 written", err, held())
}
if out, _ := tools[0].Run(map[string]any{}); out.(map[string]any)["recovery"].(map[string]any)["done"] != true {
t.Errorf("recovery still said not done after a success: %v", out)
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, errors.New("down") }
recoverAsk := func(key string, value []byte) (TakeInAsk, error) {
a, err := DecodeTakeIn(value)
r := ix.RecoverTriesByHand(ctx, a.By, a.Why)
a.Result = &r
return a, err
}
byHand := TryCandidateTools(cands, recovery, recoverAsk, time.Now)
if out, err := byHand[1].Run(map[string]any{"why": "the checker was replaced"}); err != nil || out.(*TakenIn).Stored || !strings.Contains(out.(*TakenIn).Answer, "recovery not done") {
t.Errorf("%s with the checker down = %v, %v", byHand[1].Name, out, err)
}
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) { return nil, nil }
if out, err := byHand[1].Run(map[string]any{"why": "the checker is back"}); err != nil || !out.(*TakenIn).Stored || len(held()) != 0 {
t.Errorf("%s = %v, %v, candidates %v, want it recovered (try-11 not listed)", byHand[1].Name, out, err, held())
}
if err := ix.StartTry(ctx, "try-7", members); err != nil {
t.Fatal(err)
}
// ADR 0638 §3, §4: the consumer exists before the call, so what happens between the call and the hearing is heard after
// it. A try the call listed and whose start is then heard gets one candidate; a try started after the list gets one; a
// try that ended during the call, listed as open, leaves none once its end is heard.
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
return []tryevent.Open{{ID: "try-7", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}},
{ID: "try-9", Whole: true, Members: []tryevent.Member{{Repository: catalogueRepo, Head: h1, Trunk: trunkNow}}}}, nil
}
if err := ix.RecoverTries(ctx); err != nil {
t.Fatal(err)
}
for _, step := range []struct {
what string
do func() error
want int
}{
{"the start heard after the call, of a try the call listed", func() error { return ix.StartTry(ctx, "try-7", members) }, 2},
{"a start after the list", func() error { return ix.StartTry(ctx, "try-10", members) }, 3},
{"the end heard of a try ended during the call", func() error { return ix.EndTry(ctx, "try-9") }, 2},
} {
if err := step.do(); err != nil || len(held()) != step.want {
t.Fatalf("%s: %v, candidates %v, want %d", step.what, err, held(), step.want)
}
}
for _, id := range []string{"try-10"} {
if err := ix.EndTry(ctx, id); err != nil {
t.Fatal(err)
}
}
if err := ix.StartTry(ctx, "try-7", members); err != nil || len(held()) != 1 {
t.Fatalf("started again: %v, candidates %v", err, held())
}
if err := ix.EndTry(ctx, "try-7"); err != nil || len(held()) != 0 {
t.Errorf("an ended try: %v, candidates %v, want none", err, held())
}
// The hand recovery never overlaps a try event: a start heard while the call to the checker is out writes its candidate
// before or after the sweep, never between the call and the sweep (which would delete it, the try not being listed).
startDone := make(chan error, 1)
ix.OpenTries = func(context.Context) ([]tryevent.Open, error) {
go func() { startDone <- ix.StartTry(ctx, "try-12", members) }()
for i := 0; i < 2000 && len(held()) == 0; i++ {
runtime.Gosched() // lets the start write if nothing holds it back
}
return nil, nil
}
if r := ix.RecoverTriesByHand(ctx, "lead", "a test"); !r.Stored {
t.Fatalf("the hand recovery = %+v", r)
}
if err := <-startDone; err != nil || len(held()) != 1 || held()[0] != "try-12" {
t.Errorf("a start heard during the hand recovery: %v, candidates %v, want try-12 kept", err, held())
}
})
}
}
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
@@ -0,0 +1,154 @@
package index
// The seat definitions a revision carries (novox/hq ADR 0633 §1, ADR 0635 §3, ADR 0636 §4): when a take-in works out which
// indexer a revision names, the manifests it writes are judged against the seats of that revision, never against the
// list this program was built with, since the indexer the revision names may serve or invoke what the revision's
// definitions add (a verb of a seat, an event). They are read from the merges' trees: each contracts module a merge's
// manifests name (a package artifact with `generated-from`) holds `seats/<seat>/seat.json` and the `parts/` they
// include, read through the forge's listing and get-file tools. A delivery group's merges are one revision, so a
// group's members' definitions are read together.
import (
"context"
"encoding/json"
"errors"
"fmt"
"maps"
"path"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"mesh-delivery-indexer/judge"
)
// definitionsDir is the directory of a contracts module's definitions that the module.json at dir names, from the first
// package artifact naming `generated-from`: ok is false for a module that is no contracts module. A manifest that does
// not decode is none here: the judge says why it is refused.
func definitionsDir(dir, manifest string) (string, bool) {
var m struct {
Build struct {
Artifacts []struct {
GeneratedFrom string `json:"generated-from"`
} `json:"artifacts"`
} `json:"build"`
}
if json.Unmarshal([]byte(manifest), &m) != nil {
return "", false
}
for _, a := range m.Build.Artifacts {
if a.GeneratedFrom == "" {
continue
}
base := dir
if base == "." || base == "/" {
base = ""
}
return path.Clean(path.Join(base, a.GeneratedFrom)), true
}
return "", false
}
// revisionSeats are the seats of the definitions the ready merges carry, read at their merge commits, the newest merge
// of a repository governing; none where the indexer has no forge reader. A read the forge does not answer is an error
// (the take-in is tried again); a definition that does not parse is said and left out, as the indexer that keeps it
// would say it unparsed.
func (ix *Indexer) revisionSeats(ctx context.Context, ready []PendingEntry) ([]catalogue.Seat, error) {
if ix.ReadFile == nil || ix.Files == nil {
return nil, nil
}
var defs []judge.Definition
for _, e := range ready {
if e.Covered || e.Forge.Merge == "" {
continue
}
for _, dir := range slices.Sorted(maps.Keys(e.Forge.Manifests)) {
defsDir, ok := definitionsDir(dir, e.Forge.Manifests[dir])
if !ok {
continue
}
got, err := ix.readDefinitions(ctx, e.Forge.Repository, e.Forge.Merge, defsDir)
if err != nil {
return nil, err
}
defs = append(defs, got...)
}
}
seats, err := judge.Seats(defs)
if err != nil {
return nil, err
}
return seats, nil
}
// readDefinitions reads the definitions under a contracts module's definitions directory at a commit, resolved with their
// parts.
func (ix *Indexer) readDefinitions(ctx context.Context, repository, commit, defsDir string) ([]judge.Definition, error) {
file := func(p string) ([]byte, error) {
if defsDir != "." && defsDir != "" {
p = defsDir + "/" + p
}
raw, err := ix.ReadFile(ctx, repository, commit, p)
if err != nil {
return nil, fmt.Errorf("reading %s of %s at %.12s: %w", p, repository, commit, err)
}
return raw, nil
}
dir := path.Join(defsDir, "seats")
if defsDir == "." || defsDir == "" {
dir = "seats"
}
listed, err := ix.Files(ctx, repository, commit, []string{dir})
if err != nil {
return nil, fmt.Errorf("listing %s of %s at %.12s: %w", dir, repository, commit, err)
}
parts := map[string]judge.Part{}
var out []judge.Definition
for _, p := range slices.Sorted(maps.Keys(listed[dir])) {
seat, ok := strings.CutSuffix(p, "/seat.json")
if !ok || seat == "" || strings.Contains(seat, "/") {
continue
}
raw, err := file("seats/" + seat + "/seat.json")
if err != nil {
return nil, err
}
d, err := judge.ParseDefinition(raw)
if err == nil && d.Seat != seat {
err = fmt.Errorf("it names the seat %q, not %q", d.Seat, seat)
}
for _, name := range d.Includes {
if err != nil {
break
}
if _, have := parts[name]; have {
continue
}
praw, perr := file("parts/" + name + ".json")
if perr != nil {
return nil, perr
}
var part judge.Part
if praw == nil {
perr = errors.New("the commit holds no such part")
} else {
part, perr = judge.ParsePart(praw)
}
if perr != nil {
err = fmt.Errorf("its part %s: %w", name, perr)
break
}
parts[name] = part
}
if err == nil {
d, err = d.Resolve(parts)
}
if err != nil {
ix.logf("take-in: the seat definition seats/%s of %s at %.12s does not parse, so it is left out: %v", seat, repository, commit, err)
continue
}
out = append(out, d)
}
return out, nil
}
@@ -40,14 +40,18 @@ type TakeInAsk struct {
Pending string `json:"pending,omitempty"`
// Merge asks the merge of MergeRepository at this commit to be taken in by hand (indexer_take_in_merge): its merged message
// is asked of the forge bridge and run through the take-in as a delivered one, under its stream sequence Sequence.
Merge string `json:"merge,omitempty"`
MergeRepository string `json:"merge-repository,omitempty"`
Sequence uint64 `json:"sequence,omitempty"`
Drop bool `json:"drop,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
Merge string `json:"merge,omitempty"`
MergeRepository string `json:"merge-repository,omitempty"`
Sequence uint64 `json:"sequence,omitempty"`
// TryCandidate asks the candidate of this try's group to be dropped (indexer_drop_try_candidate), trycandidates.go.
TryCandidate string `json:"try-candidate,omitempty"`
// RecoverTries asks the open tries' candidates to be recovered now (indexer_recover_tries).
RecoverTries bool `json:"recover-tries,omitempty"`
Drop bool `json:"drop,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
Result *TakenIn `json:"result,omitempty"`
Answered time.Time `json:"answered,omitzero"`
}
// Done says the ask is answered.
@@ -62,8 +66,8 @@ func DecodeTakeIn(data []byte) (TakeInAsk, error) {
if a.V != 1 {
return a, fmt.Errorf("a takeins value of version %d, not 1", a.V)
}
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" {
return a, errors.New("a takeins value naming no module, repository, pending wait or merge")
if a.Module == "" && a.Repository == "" && a.Pending == "" && a.Merge == "" && a.TryCandidate == "" && !a.RecoverTries {
return a, errors.New("a takeins value naming no module, repository, pending wait, merge or try")
}
return a, nil
}
@@ -0,0 +1,439 @@
package index
// **A group try's candidate** (issue 1921; novox/hq ADR 0593 §7): a try of several repositories at exact commits has no pull
// request, so the forge bridge sends the indexer no message for it; yet the node-engine's compose-candidate and the checker's
// manifest validator read the indexer's candidate of a group. The indexer therefore watches the checker's state tries (the
// checker keeps it; this module reads it) and, for a try that is not narrowed to a level or to some tests, writes
// `candidates.<try id>`: one pull per member (number 0), its modules and manifests taken from the merge preview the try's
// fetch wrote, its manifests kept and judged with the seat definitions the members carry, as a take-in judges a revision's.
// The fingerprints come with the previews (RecomputeCandidates). The candidate is deleted when the try ends. `try-<digits>`
// is the reserved group name of a try: the bridge's open message never takes it out, and the checker never checks it.
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"regexp"
"slices"
"strings"
"time"
"git.novox.be/novox/mesh-delivery/modules/mesh-delivery-forge-bridge/forgemsg"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// TryCandidateTools: the tools that read, drop and recover a try's candidate.
const (
TryCandidatesTool = "indexer_try_candidates"
DropTryCandidateTool = "indexer_drop_try_candidate"
RecoverTriesTool = "indexer_recover_tries"
// TryRecoveryState is the indexer's state of how the last recovery of the open tries went: the bucket
// mesh-delivery-indexer_try-recovery, one key (TryRecoveryKey).
TryRecoveryState = "try-recovery"
TryRecoveryBucket = delivery.IndexerModule + "_" + TryRecoveryState
TryRecoveryKey = "state"
)
// TryRecovery is the value of the try-recovery state: whether the candidates of the open tries were recovered at the
// indexer's start or by hand, and when it began to fail, why.
type TryRecovery struct {
V int `json:"v"`
Done bool `json:"done"`
Since time.Time `json:"since,omitzero"`
Why string `json:"why,omitempty"`
At time.Time `json:"at"`
}
var tryGroupOK = regexp.MustCompile(`^try-[0-9]{1,20}$`)
// IsTryGroup says a candidates key is the group of a try.
func IsTryGroup(key string) bool { return tryGroupOK.MatchString(key) }
// TryMember is one repository of a try: its exact commit and the trunk commit it is merged onto.
type TryMember struct{ Repository, Head, Trunk string }
// StartTry takes a whole try in, as the checker said it started (tryevent.Started): its members at their commits are kept, and
// its candidate is written once every member's preview is held. The checker's state is never read (ADR 0596).
func (ix *Indexer) StartTry(ctx context.Context, id string, members []TryMember) error {
ix.tryMu.Lock()
defer ix.tryMu.Unlock()
ix.retryRecovery(ctx)
return ix.startTry(ctx, id, members)
}
// retryRecovery runs the recovery again on a try event when the last one failed: the failure stays said until one succeeds.
func (ix *Indexer) retryRecovery(ctx context.Context) {
if ix.recoveryFailed.Load() && ix.OpenTries != nil {
_ = ix.recoverNow(ctx)
}
}
func (ix *Indexer) startTry(ctx context.Context, id string, members []TryMember) error {
if !IsTryGroup(id) || len(members) == 0 {
return fmt.Errorf("%q with %d members is no try to start", id, len(members))
}
ix.mu.Lock()
if ix.tries == nil {
ix.tries = map[string][]TryMember{}
}
ix.tries[id] = slices.Clone(members)
ix.mu.Unlock()
return ix.writeTryCandidate(ctx, id)
}
// EndTry takes a try out, as the checker said it ended (tryevent.Ended), and deletes its candidate.
func (ix *Indexer) EndTry(ctx context.Context, id string) error {
ix.tryMu.Lock()
defer ix.tryMu.Unlock()
ix.retryRecovery(ctx)
return ix.endTry(ctx, id)
}
func (ix *Indexer) endTry(ctx context.Context, id string) error {
ix.mu.Lock()
delete(ix.tries, id)
ix.mu.Unlock()
if !IsTryGroup(id) {
return nil
}
return ix.dropTryCandidate(ctx, id)
}
// RecoverTries is, at the indexer's start, what the events it missed would have said: the candidates of the tries that wait
// or run now are written, and the candidate of a try the checker no longer lists is deleted. The checker's tool lists them
// (OpenTries); its bucket is not read. Nothing to recover from when no part is wired to ask.
//
// A recovery that fails is said loudly and kept (the try-recovery state, read by indexer_try_candidates) until one succeeds:
// the events are still heard after it, and indexer_recover_tries runs it again by hand.
//
// A recovery and a try event never overlap (tryMu): a sweep that deleted the candidate a start event wrote after the call to the
// checker would leave a running try with none.
func (ix *Indexer) RecoverTries(ctx context.Context) error {
ix.tryMu.Lock()
defer ix.tryMu.Unlock()
return ix.recoverNow(ctx)
}
func (ix *Indexer) recoverNow(ctx context.Context) error {
if ix.OpenTries == nil || ix.Store == nil {
return nil
}
err := ix.recoverTries(ctx)
ix.recoveryFailed.Store(err != nil)
ix.noteRecovery(ctx, err)
return err
}
// noteRecovery writes how a recovery went: failing keeps the time it began to fail, and the journal says it each time.
func (ix *Indexer) noteRecovery(ctx context.Context, err error) {
now := ix.now()
r := TryRecovery{V: 1, Done: err == nil, At: now.UTC()}
if err != nil {
r.Why, r.Since = err.Error(), now.UTC()
ix.logf("RECOVERY OF THE TRIES NOT DONE: %v (the try candidates may be stale or missing until indexer_recover_tries or the next start succeeds)", err)
}
if ix.TryRecovery == nil {
return
}
for range writeTries {
cur, rev, found, gerr := ix.TryRecovery.Get(ctx, TryRecoveryKey)
if gerr != nil {
ix.logf("the try-recovery state cannot be read: %v", gerr)
return
}
if err != nil && found {
var was TryRecovery
if json.Unmarshal(cur, &was) == nil && !was.Done && !was.Since.IsZero() {
r.Since = was.Since
}
}
data, _ := json.Marshal(r)
var werr error
if found {
werr = ix.TryRecovery.Update(ctx, TryRecoveryKey, data, rev)
} else {
werr = ix.TryRecovery.Create(ctx, TryRecoveryKey, data)
}
if werr == nil {
return
}
if !errors.Is(werr, ErrConflict) {
ix.logf("the try-recovery state cannot be written: %v", werr)
return
}
}
}
// RecoverTriesByHand runs the recovery now (indexer_recover_tries), a hand act said in the journal.
func (ix *Indexer) RecoverTriesByHand(ctx context.Context, by, why string) TakenIn {
r := TakenIn{Module: "tries"}
ix.logf("recovery of the tries asked by %s (%s)", by, why)
if ix.OpenTries == nil {
r.Answer = "the indexer is wired to list no open try: nothing to recover from"
return r
}
if err := ix.RecoverTries(ctx); err != nil {
r.Answer = "recovery not done: " + err.Error()
return r
}
r.Stored, r.Answer = true, "the candidates of the open tries are recovered"
return r
}
func (ix *Indexer) recoverTries(ctx context.Context) error {
open, err := ix.OpenTries(ctx)
if err != nil {
return fmt.Errorf("the open tries cannot be listed, so a missed try event is not recovered: %w", err)
}
live := map[string]bool{}
for _, o := range open {
if !o.Whole || len(o.Members) == 0 {
continue
}
live[o.ID] = true
var ms []TryMember
for _, m := range o.Members {
ms = append(ms, TryMember{m.Repository, m.Head, m.Trunk})
}
if err := ix.startTry(ctx, o.ID, ms); err != nil {
return err
}
}
keys, err := ix.Store.Keys(ctx)
if err != nil {
return err
}
for _, k := range keys {
if IsTryGroup(k) && !live[k] {
if err := ix.endTry(ctx, k); err != nil {
return err
}
}
}
return nil
}
// WriteTryCandidates writes the candidate of every live try that has none yet and whose previews are all there: after a
// preview was written, the try that waited for it.
func (ix *Indexer) WriteTryCandidates(ctx context.Context) error {
ix.mu.Lock()
ids := make([]string, 0, len(ix.tries))
for id := range ix.tries {
ids = append(ids, id)
}
ix.mu.Unlock()
slices.Sort(ids)
for _, id := range ids {
if err := ix.writeTryCandidate(ctx, id); err != nil {
return err
}
}
return nil
}
func (ix *Indexer) dropTryCandidate(ctx context.Context, id string) error {
if ix.Store == nil {
return nil
}
key, err := delivery.CandidatesKey(id)
if err != nil {
return err
}
return write(ctx, ix.Store, key, func(_ []byte, found bool) ([]byte, bool, error) { return nil, found, nil })
}
// DropTryCandidate deletes a try's candidate by hand (indexer_drop_try_candidate); a live try gets it again at its next
// change.
func (ix *Indexer) DropTryCandidate(ctx context.Context, id, by, why string) TakenIn {
r := TakenIn{Module: id}
switch {
case !IsTryGroup(id):
r.Answer = id + " is no try's group, try-<digits>: nothing dropped"
default:
if err := ix.dropTryCandidate(ctx, id); err != nil {
r.Answer = "the candidate of " + id + " could not be dropped: " + err.Error()
} else {
r.Stored, r.Answer = true, "the candidate of "+id+" is dropped by "+by+" ("+why+")"
ix.logf("%s", r.Answer)
}
}
return r
}
// writeTryCandidate writes the candidate of a live try when it has none at its members' heads and every member's merge
// preview is held.
func (ix *Indexer) writeTryCandidate(ctx context.Context, id string) error {
if ix.Store == nil || ix.Read == nil {
return nil
}
ix.mu.Lock()
members := slices.Clone(ix.tries[id])
ix.mu.Unlock()
if len(members) == 0 {
return nil
}
key, err := delivery.CandidatesKey(id)
if err != nil {
return err
}
slices.SortFunc(members, func(a, b TryMember) int { return strings.Compare(a.Repository, b.Repository) })
heads := make([]delivery.Head, len(members))
for i, m := range members {
heads[i] = delivery.Head{Repository: m.Repository, Head: m.Head}
}
hash, err := delivery.HeadsHash(heads)
if err != nil {
return err
}
if value, _, found, err := ix.Store.Get(ctx, key); err == nil && found {
if c, err := delivery.DecodeCandidate(value); err == nil && c.HeadsHash == hash {
return nil // fingerprints follow the previews (RecomputeCandidates)
}
}
previews := make([]delivery.Preview, len(members))
for i, m := range members {
pk, err := delivery.PreviewKey(m.Repository, m.Head, m.Trunk)
if err != nil {
return fmt.Errorf("the try %s: %w", id, err)
}
ix.mu.Lock()
pv, ok := ix.previews[pk]
ix.mu.Unlock()
if !ok {
return nil // its fetch has not written the preview yet: this is called again when it does
}
previews[i] = pv
}
held, err := ix.heldModules(ctx)
if err != nil {
return err
}
pulls := make([]delivery.CandidatePull, len(members))
var entries []PendingEntry
over := map[string]string{}
for i, m := range members {
pull := delivery.CandidatePull{Head: heads[i], Modules: []string{}}
// defs are every module directory's manifest of the member at its head, the unchanged ones too: a contracts module whose
// seat.json changed while its module.json did not carries definitions all the same (ADR 0636 §4).
defs := map[string]string{}
for _, dir := range slices.Sorted(mapKeys(previews[i].Modules)) {
pm := previews[i].Modules[dir]
name, unchanged := "", false
for n, h := range held {
if delivery.SameRepository(h.Repository, m.Repository) && h.Path == forgeDir(dir) {
name, unchanged = n, h.Manifest == pm.Manifest
break
}
}
if unchanged {
if raw := ix.keptManifest(ctx, pm.Manifest); raw != "" {
defs[dir] = raw
}
continue // the manifest the trunk holds: the try does not touch the module
}
rr, err := ix.Read(ctx, m.Repository, m.Head, forgeDir(dir))
if err != nil {
return fmt.Errorf("the try %s: reading the manifest of %s at %.12s: %w", id, dir, m.Head, err)
}
raw := rr.Manifests[forgeDir(dir)]
if raw == "" {
ix.logf("the try %s: %s holds no manifest at %.12s, so no module of it is taken", id, dir, m.Head)
continue
}
module := manifestModule(raw)
if module == "" {
module = name
}
if module == "" {
module = path.Base(strings.Trim(dir, "/."))
}
digest, err := ix.keepManifest(ctx, module, raw)
if err != nil {
return err
}
if digest != pm.Manifest {
ix.logf("the try %s: the manifest of %s read at %.12s is not the one its preview names, left", id, dir, m.Head)
continue
}
if pull.Fps == nil {
pull.Fps = map[string]delivery.CandidateModule{}
}
pull.Modules = append(pull.Modules, module)
pull.Fps[module] = delivery.CandidateModule{Manifest: digest, None2: NoneNoPreview}
defs[dir] = raw
over[module] = digest
}
pulls[i] = pull
entries = append(entries, PendingEntry{Forge: forgemsg.Forge{Repository: m.Repository, Merge: m.Head, Manifests: defs}})
}
// The manifests are judged with the seat definitions the members carry (seatsof.go, ADR 0636 §4), in one pass that takes the
// seats. Their verdicts are the verdicts state's, keyed by the manifest's digest, as a pull request's candidate's are
// (judgeDigests in ApplyMessage): nothing reads them but the engines' composition of that digest and the module
// value of a trunk module with the same digest.
seats, err := ix.revisionSeats(ctx, entries)
if err != nil {
return fmt.Errorf("the try %s: reading the seat definitions of its members: %w", id, err)
}
ix.judgeDigestsWith(ctx, seats, over, "the try "+id)
c := delivery.Candidate{V: delivery.Version, Group: id, HeadsHash: hash, Pulls: pulls}
c.FPSet = delivery.FingerprintSet(c.Pulls)
ix.withPreviews(ctx, &c)
if err := ix.refuseDisagreement(c); err != nil {
return err
}
data, err := delivery.EncodeCandidate(c)
if err != nil {
return err
}
return write(ctx, ix.Store, key, func(_ []byte, _ bool) ([]byte, bool, error) { return data, false, nil })
}
// keptManifest is the text of the manifest the indexer keeps under a digest, or empty.
func (ix *Indexer) keptManifest(ctx context.Context, digest string) string {
if ix.Manifests == nil || digest == "" {
return ""
}
key, err := delivery.ManifestsKey(digest)
if err != nil {
return ""
}
value, _, found, err := ix.Manifests.Get(ctx, key)
if err != nil || !found {
return ""
}
m, err := delivery.DecodeManifest(value)
if err != nil {
return ""
}
return m.JSON
}
func mapKeys[V any](m map[string]V) func(func(string) bool) {
return func(yield func(string) bool) {
for k := range m {
if !yield(k) {
return
}
}
}
}
// forgeDir is a preview's directory as the forge reader spells it: the root as "".
func forgeDir(dir string) string {
if dir == "." || dir == "/" {
return ""
}
return strings.Trim(dir, "/")
}
// heldAt says the trunk holds a module of the repository in that directory.
func heldAt(held map[string]delivery.Module, repository, dir string) bool {
for _, h := range held {
if delivery.SameRepository(h.Repository, repository) && h.Path == forgeDir(dir) {
return true
}
}
return false
}
@@ -61,6 +61,17 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, over ma
// modules another is rewriting.
ix.judgeMu.Lock()
defer ix.judgeMu.Unlock()
if !ix.underSeats {
// The seat list stands still for this pass: a take-in's dry run may put its revision's seats over it (seatsof.go).
defer judge.HoldSeats()()
}
return ix.judgeOverHeld(ctx, fresh, over)
}
// judgeOverHeld is judgeOver for a caller that holds the judge pass (judgeMu) and keeps the seat list as it wants it
// (judge.HoldSeats, or inside judge.WithSeats): it takes no lock, so a pass that put its seats over the list judges inside
// it without asking the shared lock the exclusive one forbids.
func (ix *Indexer) judgeOverHeld(ctx context.Context, fresh map[string]bool, over map[string]string) (Judged, error) {
var out Judged
if ix.Verdicts == nil || ix.Modules == nil || ix.Manifests == nil {
return out, nil
@@ -200,7 +211,6 @@ func (ix *Indexer) judgeOver(ctx context.Context, fresh map[string]bool, over ma
// digest of the verdicts state that another judge refused or held waiting (one of a candidate, or one back on the trunk
// later) is judged again by this judge (judgeStale), so a refusal an older judge made never stands for good.
func (ix *Indexer) JudgeAtStart(ctx context.Context) (Judged, error) {
ix.learnOwnBuild(ctx)
j, err := ix.JudgeAll(ctx, nil)
ix.judgePending.Store(err != nil)
if err == nil {
@@ -248,6 +258,27 @@ func (ix *Indexer) judgeDigests(ctx context.Context, over map[string]string, why
}
}
// judgeDigestsWith is judgeDigests with the seats put over the catalogue's list for the whole pass (judge.WithSeats): one
// locked pass that takes the seats, and never a second lock under it (the exclusive one of WithSeats and the shared one of
// judgeOver would wait for each other for good).
func (ix *Indexer) judgeDigestsWith(ctx context.Context, seats []catalogue.Seat, over map[string]string, why string) {
if ix.Verdicts == nil || ix.Manifests == nil || ix.Modules == nil {
return
}
ix.judgeMu.Lock()
defer ix.judgeMu.Unlock()
judge.WithSeats(seats, func() {
for _, module := range slices.Sorted(maps.Keys(over)) {
if !ix.staleVerdict(ctx, over[module]) {
continue
}
if _, err := ix.judgeOverHeld(ctx, nil, map[string]string{module: over[module]}); err != nil {
ix.logf("judge: the manifest %s of %s (%s) is not judged: %v", over[module], module, why, err)
}
}
})
}
// judgeStale judges again every digest of the verdicts state another judge refused or held waiting, each as a manifest
// of the module its verdict names (judgeDigests). Never on a timer: at the start pass.
func (ix *Indexer) judgeStale(ctx context.Context) {
@@ -323,21 +354,66 @@ func (ix *Indexer) writeVerdict(ctx context.Context, digest string, v delivery.M
return wrote, err
}
// learnOwnBuild asks where the indexer's program is told which build it runs (OwnBuild: the node-engine's seat, never the
// trunk's value of the indexer, which names a build that may be merged and not delivered). Asked at the start pass; where
// it is not answered the build stays unknown and every unknown field waits, which is the safe side.
func (ix *Indexer) learnOwnBuild(ctx context.Context) {
if ix.OwnBuild == nil {
// OwnBuildState is the indexer's state of the build it runs (the bucket mesh-delivery-indexer_own-build, one key, OwnBuildKey):
// written once, at the process's start by FixOwnBuild, so indexer_pending says whether this process knows its build and, when it
// does not, since when and why.
const (
OwnBuildStateName = "own-build"
OwnBuildBucket = delivery.IndexerModule + "_" + OwnBuildStateName
OwnBuildKey = "state"
)
// OwnBuild is the value of the own-build state.
type OwnBuild struct {
V int `json:"v"`
Known bool `json:"known"`
FP string `json:"fp,omitempty"`
Since time.Time `json:"since"`
Why string `json:"why,omitempty"`
}
// FixOwnBuild fixes, once and at the process's start, which build of the indexer this program is: asked of the node-engine's
// seat (OwnBuild; never the trunk's value of the indexer, which names a build that may be merged and not delivered), before
// this process hears a merge. It is never asked again: while the node-engine replaces the indexer the seat already names the
// successor, and an old process that learnt it later would take revisions in with its old judge. Where the start's ask is not
// answered the build stays unknown for the life of this process: every merge waits, loudly (a take-in.pending that says so, the
// own-build state, indexer_pending), and the module is started again to learn it.
func (ix *Indexer) FixOwnBuild(ctx context.Context) {
ix.ownMu.Lock()
if ix.ownFixed || ix.OwnBuild == nil {
ix.ownMu.Unlock()
return
}
ix.ownFixed = true
ix.ownMu.Unlock()
fp, err := ix.OwnBuild(ctx)
if err != nil || fp == "" {
ix.logf("judge: the build this program runs is not known (%v), so a manifest with a field it does not know waits and is not refused", err)
return
if err == nil && fp == "" {
err = errors.New("the node-engine names no build of the indexer")
}
st := OwnBuild{V: 1, Since: ix.now().UTC()}
if err != nil {
st.Why = err.Error()
ix.logf("judge: the build this program runs is not known (%v), so every merge waits and a manifest with a field it does not know waits and is not refused; it is learnt only at the start, so the indexer is started again", err)
} else {
st.Known, st.FP = true, fp
}
ix.ownMu.Lock()
ix.ownFP = fp
ix.ownFP, ix.ownSince, ix.ownWhy = fp, st.Since, st.Why
ix.ownMu.Unlock()
if ix.OwnBuildState == nil {
return
}
data, _ := json.Marshal(st)
if _, rev, found, gerr := ix.OwnBuildState.Get(ctx, OwnBuildKey); gerr == nil && found {
err = ix.OwnBuildState.Update(ctx, OwnBuildKey, data, rev)
} else if gerr == nil {
err = ix.OwnBuildState.Create(ctx, OwnBuildKey, data)
} else {
err = gerr
}
if err != nil {
ix.logf("the own-build state cannot be written: %v", err)
}
}
// ownBuildIsNewest says whether the build of the indexer this program runs (ownFP, from the node-engine) is the newest the
@@ -478,8 +554,6 @@ func (ix *Indexer) JudgeAgain(ctx context.Context, module, digest string) TakenI
out.Answer = "the modules state cannot be read: " + err.Error()
return out
}
// The build this program runs is asked again, so a start pass that missed the engine's answer is recovered without a restart.
ix.learnOwnBuild(ctx)
if digest != "" {
// One digest (a candidate's, or one the trunk no longer names), judged by this judge in the catalogue the trunk holds.
value, _, found, err := ix.Manifests.Get(ctx, digest)
@@ -273,12 +273,16 @@ func TestTheIndexerJudgesEveryManifestOnce(t *testing.T) {
if rows, _ := ListVerdicts(keys, get, "", true); len(rows) <= len(keys)-3 || rows[0].Module > rows[len(rows)-1].Module {
t.Fatalf("indexer_verdicts, all: %d rows of %d verdicts", len(rows), len(keys))
}
// indexer_judge asks the engine for the running build again: a start pass that missed it is recovered (the trunk's value of
// the indexer is the newer one, so the field waits; asked and answered with that same build it is refused).
ix.ownFP = ""
// The build is fixed once, at the start, and never learnt later: this program started when the engine could not answer, and
// the engine now names the trunk's newest build (as it does while it replaces the indexer by a successor). indexer_judge
// does not take it for this program's own, so the field still waits and the build stays unknown.
ix.ownFP, ix.ownFixed = "", true
ix.OwnBuild = func(context.Context) (string, error) { return "src2:" + strings.Repeat("a", 64), nil }
if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "refused") || ix.ownFP == "" {
t.Fatalf("indexer_judge did not learn the running build: %+v", got)
if got := ix.JudgeAgain(ctx, "later", ""); !got.Stored || !strings.Contains(got.Answer, "waits") || ix.ownFP != "" {
t.Fatalf("indexer_judge learnt the build the engine names now: %+v (own %q)", got, ix.ownFP)
}
if _, err := ix.JudgeAtStart(ctx); err != nil || ix.ownFP != "" || !verdict(later).Waits() {
t.Fatalf("a judge pass learnt the build: %v %q %+v", err, ix.ownFP, verdict(later))
}
ix.ownFP = ownFP
ix.OwnBuild = nil
+18 -1
View File
@@ -204,7 +204,24 @@ const Unnamed = "(between manifests)"
// each case's manifests as one catalogue (Shelf), every manifest accepted or refused with its problems, one per line; the
// problems no module's removal ends under Between. The verdict's SDK names this judge (Identity). A manifest left as a
// digest, or a case named twice, is a request it cannot run.
func Request(r composer.Request) (composer.Verdict, error) {
func Request(r composer.Request) (composer.Verdict, error) { return RequestWith(r, nil) }
// RequestWith is Request judging against the seat definitions the request gives (ADR 0636 §4; the request's `seats`,
// which the mesh-sdk this judge is built with does not decode: cmd/mesh-judge reads them from the request's bytes), put
// over the seats this program was built with, never the built-in list alone: inside a delivery group's check they are
// the members' heads' definitions, so a manifest that serves or invokes what the group's definitions add is judged by
// them. None is Request.
func RequestWith(r composer.Request, seats [][]byte) (composer.Verdict, error) {
over, err := SeatsOfFiles(seats)
if err != nil {
return composer.Verdict{}, fmt.Errorf("a seat definition of the request: %w", err)
}
var v composer.Verdict
WithSeats(over, func() { v, err = request(r) })
return v, err
}
func request(r composer.Request) (composer.Verdict, error) {
v := composer.Verdict{V: composer.Version, SDK: Identity()}
seen := map[string]bool{}
counts := trunkOldSections(r)
@@ -96,6 +96,7 @@ func TestTheJudge(t *testing.T) {
"a": {Raw: json.RawMessage(`{"module":"a","version":"1"}`)},
"b": {Raw: json.RawMessage(`{"module":"b","version":"1","requires":["amqp"]}`)},
}}}}
was := catalogue.Seats()
v, err := Request(r)
if err != nil || v.V != composer.Version || len(v.Cases) != 1 || v.SDK != Identity() || !v.Cases[0].Manifests["a"].Accepted ||
v.Cases[0].Manifests["b"].Accepted || !strings.Contains(v.Cases[0].Manifests["b"].Refused, `requires "amqp"`) {
@@ -122,6 +123,43 @@ func TestTheJudge(t *testing.T) {
!got["newer"].Accepted || !got["user"].Accepted || !got["mesh-grants"].Accepted {
t.Errorf("the change's old sections: %+v", got)
}
// ADR 0633 §6, ADR 0636 §4: a successor's manifest is judged by the seat definitions its request carries, put over the
// built-in seats: a verb the built-in list lacks is refused without them and accepted with them (the request's bytes,
// as cmd/mesh-judge reads them), and the seats stand as before once the request is judged.
succ := composer.Request{V: composer.Version, Cases: []composer.Case{{Name: "change", Manifests: map[string]composer.Manifest{
"successor": {Raw: json.RawMessage(`{"module":"successor","version":"1","invokes":["seat:node-engine.manifest-reads"]}`)}}}}}
defs := [][]byte{[]byte(`{"format":1,"seat":"node-engine","scope":"node","grain":"seat-list","includes":["reads"],"verbs":[{"name":"manifest-reads"}]}`)}
for _, row := range []struct {
name string
seats [][]byte
want string // a word of the refusal; empty: accepted
}{
{"without definitions", nil, `serves no verb "manifest-reads"`},
{"a definition with the verb", [][]byte{[]byte(strings.Replace(string(defs[0]), `"includes":["reads"],`, "", 1))}, ""},
{"a definition naming a part it includes", defs, "includes"},
{"a definition of no seat", [][]byte{[]byte(`{"format":1}`)}, "names its seat"},
} {
sv, err := RequestWith(succ, row.seats)
switch {
case row.want != "" && err != nil:
if !strings.Contains(err.Error(), row.want) {
t.Errorf("%s: %v, want %q", row.name, err, row.want)
}
case err != nil:
t.Errorf("%s: %v", row.name, err)
case (row.want == "") != sv.Cases[0].Manifests["successor"].Accepted ||
!strings.Contains(sv.Cases[0].Manifests["successor"].Refused, row.want):
t.Errorf("%s: %+v, want %q", row.name, sv.Cases[0].Manifests, row.want)
}
if _, ok := catalogue.SeatNamed("node-engine"); !ok || len(catalogue.Seats()) != len(was) {
t.Fatalf("%s: the seats are not put back", row.name)
}
}
parts := map[string]Part{"reads": {Verbs: []DefVerb{{Name: "manifest-reads"}}}}
d, _ := ParseDefinition(defs[0])
if d, err := d.Resolve(parts); err != nil || len(d.Verbs) != 2 || d.Verbs[0].Name != "manifest-reads" {
t.Errorf("a resolved definition: %+v %v", d, err)
}
r.Cases[0].Manifests["c"] = composer.Manifest{Digest: strings.Repeat("a", 64)}
if _, err := Request(r); err == nil {
t.Fatal("a manifest left as a digest was judged")
@@ -0,0 +1,179 @@
package judge
// The seat definitions a revision carries (novox/hq ADR 0633 §1, §6; ADR 0635, ADR 0636 §4): a manifest is judged
// against the seats of the revision it belongs to, never against the list this program was built with. A definition is
// the contracts module's `seats/<seat>/seat.json` (design 26 *A seat's definition*), with the parts it includes from
// `parts/<part>.json`. This reader takes of it only what a manifest is judged by (the seat's scope, what it delivers,
// its verbs and events) and reads the rest leniently: the format's one strict reader (mesh-sdk seatdef) is not in the
// mesh-sdk this judge is built with, and the indexer of the revision keeps the definitions whole.
import (
"encoding/json"
"fmt"
"slices"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// Definition is the part of one seat's definition a manifest is judged by.
type Definition struct {
Format int `json:"format"`
Seat string `json:"seat"`
Scope string `json:"scope"`
Decision string `json:"decision"`
HolderDeclares bool `json:"holder-declares"`
Delivers string `json:"delivers"`
Replicated bool `json:"replicated"`
PulledByKind bool `json:"pulled-by-kind"`
Yields bool `json:"yields"`
HeldByTheEngine bool `json:"held-by-the-engine"`
Includes []string `json:"includes"`
Verbs []DefVerb `json:"verbs"`
Accepts []DefName `json:"accepts"`
Events []DefName `json:"events"`
}
// DefVerb is a verb of a definition.
type DefVerb struct {
Name string `json:"name"`
Description string `json:"description"`
Optional bool `json:"optional"`
Replaces []string `json:"replaces"`
}
// DefName is an accepted message or an event of a definition.
type DefName struct {
Name string `json:"name"`
}
// Part is a part a definition includes: verbs and events written once for the seats that share them.
type Part struct {
Part string `json:"part"`
Verbs []DefVerb `json:"verbs"`
Events []DefName `json:"events"`
}
// ParseDefinition reads one definition; a file that is not one names a seat and a scope.
func ParseDefinition(raw []byte) (Definition, error) {
var d Definition
if err := json.Unmarshal(raw, &d); err != nil {
return Definition{}, fmt.Errorf("a seat definition is not JSON: %w", err)
}
if d.Seat == "" || (d.Scope != catalogue.ScopeMesh && d.Scope != catalogue.ScopeNode) {
return Definition{}, fmt.Errorf("a seat definition names its seat and its scope (mesh or node), not %q and %q", d.Seat, d.Scope)
}
return d, nil
}
// ParsePart reads one part.
func ParsePart(raw []byte) (Part, error) {
var p Part
if err := json.Unmarshal(raw, &p); err != nil {
return Part{}, fmt.Errorf("a part of a seat definition is not JSON: %w", err)
}
return p, nil
}
// Resolve is the definition with the verbs and events of the parts it includes put first, as the indexer keeps it.
func (d Definition) Resolve(parts map[string]Part) (Definition, error) {
out := d
out.Includes, out.Verbs, out.Events = nil, nil, nil
for _, name := range d.Includes {
p, ok := parts[name]
if !ok {
return Definition{}, fmt.Errorf("the seat %s includes the part %q, which is not given", d.Seat, name)
}
out.Verbs = append(out.Verbs, p.Verbs...)
out.Events = append(out.Events, p.Events...)
}
out.Verbs = append(out.Verbs, d.Verbs...)
out.Events = append(out.Events, d.Events...)
return out, nil
}
// Seats are the seats of resolved definitions, in the catalogue's types; a seat its holder declares itself is left out.
// A definition still naming parts it includes is refused: resolve it first.
func Seats(defs []Definition) ([]catalogue.Seat, error) {
var out []catalogue.Seat
for _, d := range defs {
if d.HolderDeclares {
continue
}
if len(d.Includes) > 0 {
return nil, fmt.Errorf("the seat %s names parts it includes: resolve it first", d.Seat)
}
s := catalogue.Seat{Name: d.Seat, Scope: d.Scope, Delivers: d.Delivers, Replicated: d.Replicated,
PulledByKind: d.PulledByKind, Yields: d.Yields, HeldByTheEngine: d.HeldByTheEngine, Decision: d.Decision}
for _, a := range d.Accepts {
s.Accepts = append(s.Accepts, a.Name)
}
for _, e := range d.Events {
s.Emits = append(s.Emits, e.Name)
}
for _, v := range d.Verbs {
s.Serves = append(s.Serves, catalogue.Verb{Name: v.Name, Description: v.Description, Optional: v.Optional, Replaces: v.Replaces})
}
out = append(out, s)
}
return out, nil
}
// SeatsOfFiles are the seats of resolved definitions given as files, as a request carries them.
func SeatsOfFiles(raws [][]byte) ([]catalogue.Seat, error) {
defs := make([]Definition, 0, len(raws))
for _, raw := range raws {
d, err := ParseDefinition(raw)
if err != nil {
return nil, err
}
defs = append(defs, d)
}
return Seats(defs)
}
// merged is base with each seat of over put in place of the seat of its name, or added: a definition is a seat's whole
// contract. A verb the base seat holds keeps its input and output (a definition read here carries none).
func merged(base, over []catalogue.Seat) []catalogue.Seat {
out := slices.Clone(base)
for _, s := range over {
at := slices.IndexFunc(out, func(b catalogue.Seat) bool { return b.Name == s.Name })
if at < 0 {
out = append(out, s)
continue
}
for i, v := range s.Serves {
if j := slices.IndexFunc(out[at].Serves, func(b catalogue.Verb) bool { return b.Name == v.Name }); j >= 0 {
s.Serves[i].Input, s.Serves[i].Output = out[at].Serves[j].Input, out[at].Serves[j].Output
}
}
out[at] = s
}
return out
}
// seatsMu keeps the seat list the catalogue holds (a list of the process) still while one pass judges with another:
// WithSeats takes it whole; a judge pass of the running indexer that uses the list as it stands takes it shared.
var seatsMu sync.RWMutex
// HoldSeats keeps the catalogue's seat list as it stands until the returned function is called, so a pass that judges
// with it never sees another's (WithSeats). The one pass inside WithSeats does not take it.
func HoldSeats() func() {
seatsMu.RLock()
return seatsMu.RUnlock
}
// WithSeats runs fn with seats put over the catalogue's seat list (merged), and puts the list back after. Nothing
// else judges meanwhile (HoldSeats). fn must not call HoldSeats. No seats runs fn as it is, without the exclusion.
func WithSeats(seats []catalogue.Seat, fn func()) {
if len(seats) == 0 {
fn()
return
}
seatsMu.Lock()
defer seatsMu.Unlock()
was := catalogue.Seats()
catalogue.UseSeats(merged(was, seats))
defer catalogue.UseSeats(was)
fn()
}
@@ -108,7 +108,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
t.Errorf("emits = %v, want manifest-refused, manifest-accepted and the two take-in events", m.Emits)
}
// 1126 note 76: heard keeps the last 64 FORGE messages the indexer handled, for indexer_heard.
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory},
if want := []state{{Name: "candidates", History: 10}, {Name: "heard", History: 64}, {Name: "manifests", History: 1}, {Name: "modules", History: 20}, {Name: "predefined", History: 5}, {Name: "config-candidates", History: 5}, {Name: "taken", History: 1}, {Name: "takeins", History: 1}, {Name: "verdicts", History: 5}, {Name: "oldsections", History: 1}, {Name: "try-recovery", History: 1}, {Name: "own-build", History: 1}, {Name: delivery.PackagesState, History: delivery.PackagesHistory},
{Name: graph.State, History: graph.History}, {Name: index.PendingState, History: 1}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
@@ -117,7 +117,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
}
// "Reads FORGE" is a consumes of the indexer's stream: the broker grants only buckets by reads (mesh-control
// internal/broker/state.go:62-93).
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-checker.preview.written", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-delivery-checker.preview.written", "mesh-delivery-checker.try.ended", "mesh-delivery-checker.try.started", "mesh-delivery-forge-bridge.forge.**"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
@@ -129,7 +129,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// Three invokes: the file of a registered module's manifest and lock files (registered.go), the forge's trees (the named paths' objects, since the indexer is the one reader of a manifest's paths:
// ADR 0499 clauses 2 and 3, ADR 0495 clause 1; and the module directories of the trunk, issue 1900) and whether a
// commit is in another (issue 1900).
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition"}; !reflect.DeepEqual(m.Invokes, want) {
if want := []string{"gitea.gitea_commit_trees", "gitea.gitea_contains", "gitea.gitea_get_file", "gitea.gitea_open_pulls", "mesh-delivery-checker.checker_tries", "mesh-delivery-forge-bridge.bridge_merged_message", "seat:node-engine.composition"}; !reflect.DeepEqual(m.Invokes, want) {
t.Errorf("invokes = %v, want %v", m.Invokes, want)
}
// 1945: indexer_take_in, served by its tools bundle through the takeins state; the process serves none. The same bundle
@@ -137,7 +137,7 @@ func TestManifestDeclarations(t *testing.T) { // 1238/AC1, 1444 (reads the build
// (indexer_judge; issue 774, clause 3). 1966/AC11: indexer_old_sections reads the count of the old sections (state oldsections).
// 1585/T12, T13 (ADR 0633 §5): indexer_pending reads the pending state; indexer_take_in_pending and indexer_drop_pending
// ask through takeins; indexer_take_in_merge takes one merge in, its message asked of the forge bridge (issue 1585).
if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TakeInMergeTool}; !reflect.DeepEqual(m.Tools, want) {
if want := []string{"indexer_take_in", "indexer_old_sections", "indexer_verdicts", "indexer_judge", index.PendingTool, index.TakeInPendingTool, index.DropPendingTool, index.TryCandidatesTool, index.DropTryCandidateTool, index.RecoverTriesTool, index.TakeInMergeTool}; !reflect.DeepEqual(m.Tools, want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
// ADR 0604 §2: it holds the seat mesh-catalog, serving its four verbs from its tools bundle (which stays without a
@@ -22,12 +22,17 @@
"indexer_pending",
"indexer_take_in_pending",
"indexer_drop_pending",
"indexer_try_candidates",
"indexer_drop_try_candidate",
"indexer_recover_tries",
"indexer_take_in_merge"
],
"consumes": [
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported",
"mesh-delivery-checker.preview.written",
"mesh-delivery-checker.try.started",
"mesh-delivery-checker.try.ended",
"mesh-delivery-forge-bridge.forge.**"
],
"state": [
@@ -41,6 +46,8 @@
{ "name": "takeins", "history": 1 },
{ "name": "verdicts", "history": 5 },
{ "name": "oldsections", "history": 1 },
{ "name": "try-recovery", "history": 1 },
{ "name": "own-build", "history": 1 },
{ "name": "packages", "history": 5 },
{ "name": "graph", "history": 5 },
{ "name": "pending", "history": 1 }
@@ -64,6 +71,7 @@
"gitea.gitea_contains",
"gitea.gitea_get_file",
"gitea.gitea_open_pulls",
"mesh-delivery-checker.checker_tries",
"mesh-delivery-forge-bridge.bridge_merged_message",
"seat:node-engine.composition"
],
+15 -2
View File
@@ -136,6 +136,13 @@ type TakeIn struct {
// Lost are the keys of merges whose message the forge bridge's stream no longer holds: they are not taken in, and the
// revision waits until indexer_drop_pending drops them (issue 1585).
Lost []string `json:"lost,omitempty"`
// Refused is why the manifest of the indexer the revision names was refused when the revision was worked out: with
// the seat definitions the revision carries (ADR 0633 §1), the manifest validator's problems. The revision waits for
// an indexer whose manifest is accepted, and says why here, as it says what it lost.
Refused string `json:"refused,omitempty"`
// Unknown is why the indexer holds a merge it does not take in although no indexer build is named: it cannot learn the build
// it runs itself (ADR 0633 §1), so Since is when it began not to know. A pending revision names WaitsFor or Unknown.
Unknown string `json:"own-build-unknown,omitempty"`
}
func (t TakeIn) check() error {
@@ -164,8 +171,8 @@ func (t TakeIn) check() error {
if !strings.HasPrefix(t.Key, "m.") {
return fmt.Errorf("delivery: a revision's take-in key is its oldest merge's m.<sequence>, not %q", t.Key)
}
if t.Kind == TakeInPending && t.WaitsFor == "" {
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for", t.Key)
if t.Kind == TakeInPending && t.WaitsFor == "" && t.Unknown == "" {
return fmt.Errorf("delivery: a pending revision %s names the indexer build it waits for, or why the indexer does not know its own", t.Key)
}
case TakeInGroup:
if t.Group == "" {
@@ -235,12 +242,18 @@ type PendingWait struct {
SaidAt time.Time `json:"said-at,omitzero"`
// Lost are the keys of its merges whose message the forge bridge's stream no longer holds (TakeIn.Lost).
Lost []string `json:"lost,omitempty"`
// Refused is why the manifest of the indexer it names was refused (TakeIn.Refused).
Refused string `json:"refused,omitempty"`
}
// PendingAnswer is indexer_pending's answer: every wait, oldest first, and the pending keys that do not read.
type PendingAnswer struct {
Waits []PendingWait `json:"waits"`
Unread []string `json:"unread,omitempty"`
// OwnBuildUnknown is why the running indexer does not know its own build, and since when: while it is set, every merge waits
// (a take-in.pending with the same words) and the indexer is started again to learn it. Empty when the build is known.
OwnBuildUnknown string `json:"own-build-unknown,omitempty"`
OwnBuildSince time.Time `json:"own-build-since,omitzero"`
}
// DecodePendingAnswer reads indexer_pending's answer, refusing a wait that is not a revision or a group, or names no
@@ -20,6 +20,7 @@ func sampleTakeIns(since time.Time) []TakeIn {
Members: []delivery.Head{{Repository: "novox/mesh-delivery", Number: 9, Head: "ffff"}, {Repository: "novox/mesh-sdk", Number: 37, Head: "dddd"}}, Since: since},
{Kind: TakeInTaken, Key: PendingKey(12), What: TakeInRevision, Merges: m, WaitsFor: "src2:abc", Since: since,
By: "lead-47@shanks", Why: "the named indexer can never run"},
{Kind: TakeInPending, Key: PendingKey(13), What: TakeInRevision, Merges: m, Unknown: "the engine does not answer", Since: since}, // the indexer cannot learn its own build
}
}
@@ -106,6 +106,24 @@ func TestTheWatchmansFindings(t *testing.T) {
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound-10*time.Minute)
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §1 (issue 1921): a merge held because the indexer cannot learn its own build is raised past the revision bound from its since, naming why", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp()
body, err := takein.EncodeTakeIn(takein.TakeIn{Kind: takein.TakeInPending, Key: revKey, What: takein.TakeInRevision, Since: c.t,
Merges: []takein.Merge{{Repository: "novox/a", Number: 1, Merge: "c0ffee"}}, Unknown: "the engine does not answer"})
if err != nil {
t.Fatal(err)
}
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), body)
if o.ttls[signalKey("take-in", revKey)] != TakeInRevisionBound || len(o.findings) != 0 {
t.Fatalf("an unknown own build: ttls %v, findings %v", o.ttls, o.findings)
}
expire(w, c, signalKey("take-in", revKey), TakeInRevisionBound)
for _, f := range o.findings {
if !strings.Contains(f.Summary, "cannot learn the build it runs") {
t.Errorf("finding %q does not say why", f.Summary)
}
}
}, map[string]string{FindingKey(ScopeDelivery, revKey, KindTakeInWaits): Warning}},
{"ADR 0633 §5 (1585/T21): a drop clears the wait, and the merges left under it said pending again with the same since are raised from it", func(t *testing.T, w *Watchman, o *fakeOut, c *clock) {
w.TakeInCaughtUp() // a live message, heard after the replay
w.TakeIn(takeInEvent(t, takein.TakeInPending, revKey), takeInBody(t, takein.TakeInPending, revKey, t0))
@@ -72,6 +72,9 @@ func (w *Watchman) takeIn(kind, key string, t takein.TakeIn) {
return
}
bound, what := TakeInRevisionBound, "a source revision waits for the indexer build "+t.WaitsFor
if t.Unknown != "" {
what = "a source revision is held because the indexer cannot learn the build it runs (" + t.Unknown + ")"
}
if t.What == takein.TakeInGroup {
bound, what = TakeInGroupBound, "a delivery group is gathered and its end has not come"
}
+57
View File
@@ -0,0 +1,57 @@
// Package tryevent is what the checker says on its stream checks when a whole try (a try of a delivery group, not narrowed
// to a level or to some tests) starts and ends (issue 1921), and what the indexer, which hears it, reads: a try has no pull
// request, so the indexer writes the candidate of its group from the members at their commits, and deletes it at the end.
// The events carry the members, so no part reads the checker's state.
package tryevent
import (
"encoding/json"
"fmt"
"regexp"
)
// The events of the checker's stream checks.
const (
Started = "try.started"
Ended = "try.ended"
)
// IDOK is the shape of a try's id, which is also the name of its group (the key of its candidate).
var IDOK = regexp.MustCompile(`^try-[0-9]{1,20}$`)
// Member is one repository of the try: the exact commit and the trunk commit it is merged onto.
type Member struct {
Repository string `json:"repository"`
Head string `json:"head"`
Trunk string `json:"trunk"`
}
// Event is the body of both events: the try, and for Started its members; for Ended the state it ended in.
type Event struct {
ID string `json:"id"`
Members []Member `json:"members,omitempty"`
State string `json:"state,omitempty"`
}
// Decode reads an event body; a started event names its members.
func Decode(event string, body []byte) (Event, error) {
var e Event
if err := json.Unmarshal(body, &e); err != nil {
return e, err
}
if !IDOK.MatchString(e.ID) {
return e, fmt.Errorf("%q is not a try's id, try-<digits>", e.ID)
}
if event == Started && len(e.Members) == 0 {
return e, fmt.Errorf("the try %s starts with no member", e.ID)
}
return e, nil
}
// Open is one try that waits or runs, as the checker's tool checker_tries answers it.
type Open struct {
ID string `json:"id"`
State string `json:"state"`
Whole bool `json:"whole"`
Members []Member `json:"members"`
}