Merge branch 'feat/r4g-asks' into feat/r4g
# Conflicts: # modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go # modules/mesh-delivery-checker/manifest_test.go # modules/mesh-delivery-checker/module.json
This commit is contained in:
@@ -50,7 +50,7 @@ func TryTools(tr Trying) []stdio.Tool {
|
||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||
"the checker checks no level of (hq, one whose trunk the gate does not name) is refused.",
|
||||
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||
@@ -104,8 +104,7 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
return map[string]any{"verdicts": vs}, err
|
||||
}},
|
||||
{Name: check.PullReadTool,
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check; for an hq pull request, that hq is judged by " +
|
||||
"merge-check.sh until issue 1296. Read-only.",
|
||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
|
||||
@@ -143,32 +142,16 @@ func Tools(r check.Reader) []stdio.Tool {
|
||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||
}},
|
||||
{Name: check.StatusReadTool,
|
||||
Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
|
||||
"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
|
||||
"candidates without a verdict for their heads (a group of hq's pull requests alone is left out: merge-check.sh judges it), and the " +
|
||||
"candidates whose check waits by kind (waiting_by_kind: new, nodes, and trunk, which start after the other two). A key a build " +
|
||||
"module left running (started past the step bound, or its build module not running on its node) is listed apart under left_running " +
|
||||
"with why, and not counted. The places the checker holds are in its process, not in a bucket, and the answer says so. Read-only.",
|
||||
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||
}
|
||||
}
|
||||
|
||||
func capacity() (string, error) {
|
||||
path := os.Getenv("MESH_CHECKER_SETTINGS")
|
||||
if path == "" {
|
||||
return os.Getenv("MESH_CHECKER_CAPACITY"), nil
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var s struct {
|
||||
Capacity string `json:"capacity"`
|
||||
}
|
||||
return s.Capacity, json.Unmarshal(raw, &s)
|
||||
}
|
||||
|
||||
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
|
||||
// duration.
|
||||
func stepBound() time.Duration {
|
||||
@@ -180,9 +163,9 @@ func stepBound() time.Duration {
|
||||
|
||||
func main() {
|
||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||
PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
Capacity: capacity, HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
PreviewState: read(stdio.State("previews")),
|
||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||
tries := stdio.State(check.TriesState)
|
||||
source := stdio.State("mesh-delivery-gate.source")
|
||||
trunks := func() (map[string]string, error) {
|
||||
|
||||
@@ -105,43 +105,6 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key
|
||||
}
|
||||
}
|
||||
|
||||
// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the
|
||||
// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every
|
||||
// change after. No timer: the caller bounds the wait with its context.
|
||||
type stepsWaiter struct{ js bucketOpener }
|
||||
|
||||
func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
||||
kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||
if err != nil {
|
||||
return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||
}
|
||||
watch, err := kv.Watch(ctx, key)
|
||||
if err != nil {
|
||||
return delivery.Step{}, err
|
||||
}
|
||||
defer watch.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return delivery.Step{}, ctx.Err()
|
||||
case e, ok := <-watch.Updates():
|
||||
if !ok {
|
||||
return delivery.Step{}, errors.New("the watch of the step ended")
|
||||
}
|
||||
if e == nil || !opOf(e) {
|
||||
continue
|
||||
}
|
||||
step, err := delivery.DecodeStep(e.Value())
|
||||
if err != nil {
|
||||
return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err)
|
||||
}
|
||||
if step.State != delivery.StepRunning {
|
||||
return step, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
||||
type manifestReader struct{ kv jetstream.KeyValue }
|
||||
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// heardCaller is the build runner's caller with the calls that reached nobody marked: no responders, or none answering within
|
||||
// the call's bound. The step never began there, so the check may give it to another node (issue 1820). A call that was
|
||||
// answered, with a failure or not, is left as it is.
|
||||
type heardCaller struct {
|
||||
call func(node, key string, body any) (json.RawMessage, error)
|
||||
}
|
||||
|
||||
func (h heardCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
|
||||
raw, err := h.call(node, key, body)
|
||||
if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) {
|
||||
return nil, &check.Unheard{Err: err}
|
||||
}
|
||||
return raw, err
|
||||
}
|
||||
@@ -1,30 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/nats-io/nats.go"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
func TestACallThatReachedNobodyIsMarkedAndAnAnsweredFailureIsNot(t *testing.T) { // 1820/AC6
|
||||
for _, err := range []error{nats.ErrNoResponders, nats.ErrTimeout, context.DeadlineExceeded} {
|
||||
h := heardCaller{call: func(string, string, any) (json.RawMessage, error) { return nil, fmt.Errorf("call: %w", err) }}
|
||||
_, got := h.CallOn("g14", "m.t", nil)
|
||||
var u *check.Unheard
|
||||
if !errors.As(got, &u) {
|
||||
t.Errorf("%v: %v is not marked unheard", err, got)
|
||||
}
|
||||
}
|
||||
h := heardCaller{call: func(string, string, any) (json.RawMessage, error) { return nil, errors.New("the module refused") }}
|
||||
_, got := h.CallOn("g14", "m.t", nil)
|
||||
var u *check.Unheard
|
||||
if errors.As(got, &u) {
|
||||
t.Errorf("an answered failure %v is marked unheard", got)
|
||||
}
|
||||
}
|
||||
@@ -1,101 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"log"
|
||||
"sync"
|
||||
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
)
|
||||
|
||||
// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): whether one does is the list of the keys of
|
||||
// the controller's bucket of the delivery plans in their build phase, read each time a check is about to start. Its keys age
|
||||
// out (a time to live, and no delete marker), which a watch never hears, so a watch cannot decide it (mesh-controller
|
||||
// internal/link/holds.go lists the keys too); the watch is only the wake-up, for a plan that ended by a delete. The bucket
|
||||
// is read as empty where it is absent (issue 1652 makes it) or cannot be read, saying so once. It may come into being after
|
||||
// the checker started and the checker holds no timer (ADR 0437 §8), so each time a check is about to start and the bucket is
|
||||
// not known yet, the checker looks for it.
|
||||
type holdWatch struct {
|
||||
ctx context.Context
|
||||
js bucketOpener
|
||||
keys func(ctx context.Context) ([]string, error)
|
||||
changed func(ctx context.Context) // a plan key changed: the checks that waited may start
|
||||
logger *log.Logger
|
||||
errs chan error
|
||||
mu sync.Mutex
|
||||
watched bool
|
||||
said bool
|
||||
}
|
||||
|
||||
func newHoldWatch(ctx context.Context, js bucketOpener, changed func(context.Context), logger *log.Logger) *holdWatch {
|
||||
return &holdWatch{ctx: ctx, js: js, changed: changed, logger: logger, errs: make(chan error, 1)}
|
||||
}
|
||||
|
||||
// Held says whether a delivery builds now.
|
||||
func (w *holdWatch) Held() bool {
|
||||
w.ensure()
|
||||
w.mu.Lock()
|
||||
keys := w.keys
|
||||
w.mu.Unlock()
|
||||
if keys == nil {
|
||||
return false
|
||||
}
|
||||
list, err := keys(w.ctx)
|
||||
if err != nil {
|
||||
w.say("the keys of %s cannot be listed, so it is read as empty: %v", check.PlansBuildingBucket, err)
|
||||
return false
|
||||
}
|
||||
return len(list) > 0
|
||||
}
|
||||
|
||||
func (w *holdWatch) say(format string, args ...any) {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if !w.said {
|
||||
w.said = true
|
||||
w.logger.Printf(format, args...)
|
||||
}
|
||||
}
|
||||
|
||||
// ensure finds the bucket if it is not known and exists, and starts the wake-up watch.
|
||||
func (w *holdWatch) ensure() {
|
||||
w.mu.Lock()
|
||||
if w.watched || w.js == nil {
|
||||
w.mu.Unlock()
|
||||
return
|
||||
}
|
||||
w.mu.Unlock()
|
||||
kv, err := w.js.KeyValue(w.ctx, check.PlansBuildingBucket)
|
||||
switch {
|
||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||
w.say("the bucket %s does not exist yet: it is read as empty, so no check is held back", check.PlansBuildingBucket)
|
||||
return
|
||||
case err != nil:
|
||||
w.say("the bucket %s cannot be read, so it is read as empty: %v", check.PlansBuildingBucket, err)
|
||||
return
|
||||
}
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
if w.watched {
|
||||
return
|
||||
}
|
||||
w.watched, w.said = true, false
|
||||
w.keys = func(ctx context.Context) ([]string, error) {
|
||||
list, err := kv.Keys(ctx)
|
||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
||||
return nil, nil
|
||||
}
|
||||
return list, err
|
||||
}
|
||||
go func() {
|
||||
wake, err := kv.WatchAll(w.ctx)
|
||||
if err != nil {
|
||||
w.errs <- err
|
||||
return
|
||||
}
|
||||
w.errs <- follow(w.ctx, wake, func(jetstream.KeyValueEntry) { w.changed(w.ctx) })
|
||||
}()
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// The controller's plans-building keys age out of the bucket (a time to live, no delete marker), which a watch never hears:
|
||||
// whether a delivery builds is the list of the keys now, and the watch is only a wake-up (review D3).
|
||||
func TestADeliveryHoldsWhileItsKeyIsListedAndNotWhenItAgedOutWithNoEvent(t *testing.T) {
|
||||
listed := []string{"plan-1"}
|
||||
w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return listed, nil }}
|
||||
if !w.Held() {
|
||||
t.Fatal("a plan key is listed and nothing holds")
|
||||
}
|
||||
listed = nil // the key aged out: no delete reached the watch
|
||||
if w.Held() {
|
||||
t.Error("the key is gone from the bucket and the checker is still held")
|
||||
}
|
||||
}
|
||||
|
||||
func TestABucketThatCannotBeListedIsReadAsEmpty(t *testing.T) {
|
||||
w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return []string{"plan-1"}, errors.New("the bus is down") }}
|
||||
if w.Held() {
|
||||
t.Error("an unreadable bucket held the checker")
|
||||
}
|
||||
var absent holdWatch
|
||||
absent.logger = log.New(io.Discard, "", 0)
|
||||
if absent.Held() {
|
||||
t.Error("a watch with no bucket held the checker")
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
@@ -20,6 +21,7 @@ import (
|
||||
"github.com/nats-io/nats.go/jetstream"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||
|
||||
"mesh-delivery-checker/internal/check"
|
||||
@@ -75,7 +77,6 @@ func readCaptured(t *testing.T, name string) map[string]any {
|
||||
type fakeBuildModules struct {
|
||||
js jetstream.JetStream
|
||||
ctx context.Context
|
||||
n atomic.Int64
|
||||
conflict atomic.Bool
|
||||
mu sync.Mutex
|
||||
calls map[string]int // by `<module>.<tool>`
|
||||
@@ -112,45 +113,55 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
||||
return data
|
||||
}
|
||||
|
||||
func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
|
||||
steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
||||
// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on n1 as the real module
|
||||
// does since R4g (ADR 0525; issue 1887 A5): each ask's job runs with step bodies answering the captured results, and a
|
||||
// check takes the kept verdicts of the checker's kept bucket by their digests.
|
||||
func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.KeyValue) {
|
||||
kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) {
|
||||
tokens := strings.Split(m.Subject, ".") // mesh mod <module> call <tool> <node> <caller>
|
||||
tool, node := tokens[4], tokens[5]
|
||||
f.mu.Lock()
|
||||
f.calls[module+"."+tool]++
|
||||
f.mu.Unlock()
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(m.Data, &args)
|
||||
id := fmt.Sprintf("s%d", f.n.Add(1))
|
||||
key, _ := delivery.StepKey(node, id)
|
||||
put := func(s delivery.Step) {
|
||||
data, err := delivery.EncodeStep(s)
|
||||
if err == nil {
|
||||
_, err = steps.Put(f.ctx, key, data)
|
||||
language := strings.TrimPrefix(module, "mesh-build-")
|
||||
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
prefix := map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_"}[module]
|
||||
call := func(name string) buildstep.StepFunc {
|
||||
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||
tool := prefix + name
|
||||
f.mu.Lock()
|
||||
f.calls[module+"."+tool]++
|
||||
f.mu.Unlock()
|
||||
return f.result(t, tool, args), nil
|
||||
}
|
||||
}
|
||||
steps := buildstep.AskSteps{Fetch: call("fetch"), Test: call("test"),
|
||||
Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
|
||||
key, err := delivery.KeptKey(level, digest)
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
e, err := kept.Get(ctx, key)
|
||||
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||
return 0, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
t.Errorf("writing step %s: %v", key, err)
|
||||
return 0, false, err
|
||||
}
|
||||
}
|
||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()}
|
||||
put(step)
|
||||
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node})
|
||||
_ = m.Respond(answer)
|
||||
go func() {
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
step.Ended = time.Now().UTC()
|
||||
step.State, step.Result = delivery.StepPassed, f.result(t, tool, args)
|
||||
put(step)
|
||||
}()
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
v, err := delivery.DecodeKept(e.Value())
|
||||
return v.Tests, err == nil, err
|
||||
}}
|
||||
if module == "mesh-build-go" {
|
||||
steps.Digest = call("digest")
|
||||
}
|
||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||
put := func(key string, value any) (uint64, error) {
|
||||
raw, _ := value.(json.RawMessage)
|
||||
return kv.Put(f.ctx, key, raw)
|
||||
}
|
||||
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
|
||||
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
|
||||
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||
go func() { _ = p.Pull(f.ctx) }()
|
||||
}
|
||||
|
||||
// serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured):
|
||||
@@ -283,8 +294,8 @@ func startBed(t *testing.T) *bed {
|
||||
}
|
||||
b.serveForge(t, ctl)
|
||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||
b.fake.serve(t, ctl, "mesh-build-go")
|
||||
b.fake.serve(t, ctl, "mesh-build-node")
|
||||
b.fake.serve(t, "mesh-build-go", b.buckets[delivery.CheckerKeptBucket])
|
||||
b.fake.serve(t, "mesh-build-node", b.buckets[delivery.CheckerKeptBucket])
|
||||
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
||||
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
||||
b.setTrunk(t, map[string]string{repo: trunk})
|
||||
@@ -520,14 +531,25 @@ func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { /
|
||||
|
||||
// TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in
|
||||
// the same run is.
|
||||
func TestAnHqPullRequestGetsNoVerdictAndAnotherDoes(t *testing.T) { // ADR 0484 clause 8
|
||||
// 1887/A10, ADR 0527 §2: a pull request into hq gets a verdict and a mesh/check from the checker, as any repository's.
|
||||
func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) {
|
||||
b := startBed(t)
|
||||
b.setTrunk(t, map[string]string{repo: trunk, "novox/hq": trunk})
|
||||
hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1))
|
||||
other := b.candidate(t, "feat/x", pull(repo, 7, head2))
|
||||
b.start(t, nil)
|
||||
b.verdict(t, other, ended)
|
||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, hq); err == nil {
|
||||
t.Error("hq's pull request has a verdict")
|
||||
if v := b.verdict(t, hq, ended); v.State != delivery.VerdictPass {
|
||||
t.Fatalf("hq's verdict = %+v", v)
|
||||
}
|
||||
key, _ := delivery.CheckerOwedKey("novox/hq", 3, head1)
|
||||
for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||
if e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key); err == nil {
|
||||
if o, err := delivery.DecodeCheckerOwed(e.Value()); err == nil && o.State == "success" {
|
||||
return
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("hq#3 owes no success within 20 s")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -550,34 +572,6 @@ func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/
|
||||
}
|
||||
}
|
||||
|
||||
// TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds: the controller's bucket of the plans in their build phase
|
||||
// holds a plan's key; the candidate waits, no step is called; the key deleted, the check starts and passes (ADR 0484 clause 12).
|
||||
func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1
|
||||
b := startBed(t)
|
||||
plans, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_plans-building", History: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := plans.Put(b.ctx, "plan-1", []byte(`{"plan":"plan-1","since":"2026-10-21T19:00:00Z","seen":1}`)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||
b.start(t, nil)
|
||||
time.Sleep(1500 * time.Millisecond) // a wait for nothing to happen: there is no event to wait on
|
||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key); err == nil {
|
||||
t.Fatal("a check started while a delivery builds")
|
||||
}
|
||||
if n := b.fake.count("mesh-build-go.build_go_fetch"); n != 0 {
|
||||
t.Fatalf("%d fetches while a delivery builds", n)
|
||||
}
|
||||
if err := plans.Delete(b.ctx, "plan-1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
|
||||
t.Errorf("verdict = %+v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not
|
||||
// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9).
|
||||
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2
|
||||
|
||||
@@ -15,11 +15,8 @@ import (
|
||||
|
||||
// judgeDeps are the parts the judge of a check is made of.
|
||||
type judgeDeps struct {
|
||||
runner buildrun.Runner
|
||||
asks check.Asks
|
||||
tools *buildrun.ToolchainSet
|
||||
presence *buildrun.PresenceSet
|
||||
have *buildrun.HaveSet
|
||||
places *buildrun.Places
|
||||
trunks *check.Trunks
|
||||
kept check.KV
|
||||
previews check.KV
|
||||
@@ -33,7 +30,7 @@ type judgeDeps struct {
|
||||
}
|
||||
|
||||
func newJudge(d judgeDeps) check.Judge {
|
||||
return &check.Examiner{Compositions: d.compositions, Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places,
|
||||
return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
|
||||
}
|
||||
|
||||
@@ -144,17 +141,28 @@ func (r *repoModules) PackagePaths(repository string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildModules are the build modules that judge a repository, sorted.
|
||||
// BuildModules are the build modules that judge a repository, sorted: the Go build module always, and the Node build
|
||||
// module when a module of the repository builds TypeScript. A repository of which the mesh knows no module (mesh-lab, hq)
|
||||
// is judged by both, each finding its own directories in the tree (go.mod files; package.json files), so a repository
|
||||
// with tests in either language and no manifest is judged whole (issue 1887 A12).
|
||||
func (r *repoModules) BuildModules(repository string) []string {
|
||||
out := []string{buildrun.GoBuildModule}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
known := false
|
||||
for module, repo := range r.repoOf {
|
||||
if repo == repository && r.langs[r.typed[module]]["typescript"] {
|
||||
if repo != repository {
|
||||
continue
|
||||
}
|
||||
known = true
|
||||
if r.langs[r.typed[module]]["typescript"] {
|
||||
out = append(out, buildrun.NodeBuildModule)
|
||||
break
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
out = append(out, buildrun.NodeBuildModule)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -69,8 +69,10 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test
|
||||
if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" {
|
||||
t.Errorf("the commit the mesh runs = %q", got)
|
||||
}
|
||||
if got := r.BuildModules("novox/mesh-sdk"); strings.Join(got, ",") != buildrun.GoBuildModule {
|
||||
t.Errorf("a repository with no module: %v, want the Go build module", got)
|
||||
// 1887/A12: a repository with no module the mesh knows (mesh-lab: nested go.mod files and a root package.json) is
|
||||
// judged by both build modules.
|
||||
if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule {
|
||||
t.Errorf("a repository with no module: %v, want both build modules", got)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
||||
if !equal(manifest, announced) {
|
||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||
}
|
||||
if want := []string{"owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
||||
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||
t.Errorf("readable %v, want %v", readable, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -114,7 +114,9 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
||||
var keptStates = []announce.State{
|
||||
{Name: "kept"},
|
||||
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||
{Name: "kept", Readable: true},
|
||||
{Name: "owed", Readable: true},
|
||||
{Name: "previews", Readable: true},
|
||||
{Name: "tries"},
|
||||
@@ -189,10 +191,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_CHECKER_SETTINGS")), getenv("MESH_CHECKER_CAPACITY"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the capacity setting: %w", err)
|
||||
}
|
||||
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
|
||||
if link == "" {
|
||||
link = defaultLink
|
||||
@@ -201,8 +199,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
presence := &buildrun.PresenceSet{}
|
||||
have := &buildrun.HaveSet{}
|
||||
trunks := &check.Trunks{}
|
||||
places := &buildrun.Places{Capacity: capacity}
|
||||
runner := buildrun.Runner{Caller: heardCaller{caller.CallOn}, Steps: stepsWaiter{js}, Bound: stepBound}
|
||||
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
||||
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
||||
asks := buildrun.JetStreamAsks{JS: js}
|
||||
ch := &check.Checker{
|
||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||
@@ -241,9 +240,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}}
|
||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
||||
holds := newHoldWatch(ctx, js, func(ctx context.Context) { ch.Reconsider(ctx); tryer.Reconsider(ctx) }, logger)
|
||||
ch.Hold = holds.Held
|
||||
tryer.Hold = holds.Held
|
||||
announce := func(ctx context.Context, event, key string) {
|
||||
body, _ := json.Marshal(map[string]string{"key": key})
|
||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||
@@ -252,7 +248,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
}
|
||||
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
||||
ch.Judge = newJudge(judgeDeps{
|
||||
runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks,
|
||||
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||
announce: announce, compositions: compositions,
|
||||
@@ -394,7 +390,5 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
||||
return err
|
||||
case err := <-watch.errs:
|
||||
return err
|
||||
case err := <-holds.errs:
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `<node>=<steps at
|
||||
// once>` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a
|
||||
// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch.
|
||||
func capacityFrom(path, fallback string) (map[string]int, error) {
|
||||
if path == "" {
|
||||
return buildrun.ParseCapacity(fallback)
|
||||
}
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the settings file: %w", err)
|
||||
}
|
||||
var s struct {
|
||||
Capacity string `json:"capacity"`
|
||||
}
|
||||
if err := json.Unmarshal(raw, &s); err != nil {
|
||||
return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
|
||||
}
|
||||
return buildrun.ParseCapacity(s.Capacity)
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||
)
|
||||
|
||||
func settingsFile(t *testing.T, body string) string {
|
||||
t.Helper()
|
||||
p := filepath.Join(t.TempDir(), "settings.json")
|
||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// ADR 0484 clause 12: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere.
|
||||
func TestCapacityIsReadFromTheSettingsFile(t *testing.T) {
|
||||
for body, want := range map[string]map[string]int{
|
||||
`{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2},
|
||||
`{"capacity": ""}`: {},
|
||||
`{}`: {},
|
||||
} {
|
||||
got, err := capacityFrom(settingsFile(t, body), "")
|
||||
if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] {
|
||||
t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want)
|
||||
}
|
||||
}
|
||||
p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
|
||||
if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 {
|
||||
t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace"))
|
||||
}
|
||||
}
|
||||
|
||||
func mustCapacity(t *testing.T, path string) map[string]int {
|
||||
t.Helper()
|
||||
c, err := capacityFrom(path, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) {
|
||||
if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil {
|
||||
t.Error("shanks=-1 accepted")
|
||||
}
|
||||
if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 {
|
||||
t.Errorf("the starting value: %v, %v", c, err)
|
||||
}
|
||||
if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil {
|
||||
t.Error("a file that is not JSON accepted")
|
||||
}
|
||||
if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil {
|
||||
t.Error("a settings file that is not there accepted")
|
||||
}
|
||||
}
|
||||
|
||||
// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's.
|
||||
func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) {
|
||||
got, err := capacityFrom("", "ace=2")
|
||||
if err != nil || got["ace"] != 2 {
|
||||
t.Errorf("capacity %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user