Compare commits
91
Commits
fix/r4e-r2
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
42d880f075 | ||
|
|
3553cbaf08 | ||
|
|
d8b19cb332 | ||
|
|
28ee22ef27 | ||
|
|
f0c5be8c30 | ||
|
|
80a0183567 | ||
|
|
90fd07ac26 | ||
|
|
90a79b1585 | ||
|
|
45ab994777 | ||
|
|
55f486e040 | ||
|
|
7a88eb0a56 | ||
|
|
9261347a8f | ||
|
|
263ff54a4e | ||
|
|
cd3e55e056 | ||
|
|
b303e32a41 | ||
|
|
a31bd7d4a5 | ||
|
|
32542ebc14 | ||
|
|
a697ee86d4 | ||
|
|
0373a3b24e | ||
|
|
fae4d48e09 | ||
|
|
b2c545c40a | ||
|
|
ad26670996 | ||
|
|
fad96ae67b | ||
|
|
b83af04f0c | ||
|
|
c7bddbbeab | ||
|
|
d3561923aa | ||
|
|
d32ec6b7f8 | ||
|
|
52d8b9b753 | ||
|
|
ce51457974 | ||
|
|
bac82bb91e | ||
|
|
357e28f31d | ||
|
|
5ddc221833 | ||
|
|
37860d46a0 | ||
|
|
84b2c92ccf | ||
|
|
c0f24f56b5 | ||
|
|
76bc810bda | ||
|
|
e927469460 | ||
|
|
97b42e3f0d | ||
|
|
1b019f0353 | ||
|
|
95f52afab2 | ||
|
|
e4a40a1cb0 | ||
|
|
dd8e05142d | ||
|
|
66be3dd0e4 | ||
|
|
527f87f94c | ||
|
|
0463f21026 | ||
|
|
a4629bc649 | ||
|
|
50836eb4d6 | ||
|
|
79ef768ea4 | ||
|
|
a8ba5c2fd5 | ||
|
|
d0354574c6 | ||
|
|
69577f739d | ||
|
|
d4977d9c9d | ||
|
|
179664898a | ||
|
|
77af82cfa2 | ||
|
|
f37e5d5fa1 | ||
|
|
d9d1e57e3a | ||
|
|
e4b639c6de | ||
|
|
271d1ca759 | ||
|
|
2227e7a81f | ||
|
|
173178df86 | ||
|
|
9f27e2b1f8 | ||
|
|
85aca46348 | ||
|
|
d5868c178f | ||
|
|
7cbb1c4207 | ||
|
|
4c98841d2a | ||
|
|
c9ee92a4ab | ||
|
|
0f7091e4ac | ||
|
|
ed98a7519e | ||
|
|
d7ba5aa91e | ||
|
|
016385751d | ||
|
|
5091cab652 | ||
|
|
d8599855e1 | ||
|
|
57258b6fa4 | ||
|
|
c2b7b8867e | ||
|
|
a48279b31a | ||
|
|
034a6de9a7 | ||
|
|
e7ed48b81a | ||
|
|
f7dd4ab586 | ||
|
|
57b0030258 | ||
|
|
db5d88f893 | ||
|
|
24753bfc2c | ||
|
|
1dc9d80cfb | ||
|
|
26bfdbb637 | ||
|
|
4ffe5f4d52 | ||
|
|
2bdb8d0e00 | ||
|
|
75a4821eff | ||
|
|
e762028760 | ||
|
|
36bf3b7143 | ||
|
|
9fad90ee89 | ||
|
|
4b8b1c047c | ||
|
|
b714eab995 |
@@ -1,74 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# mesh-check-toolchain: go
|
|
||||||
#
|
|
||||||
# This repository's own check, mesh/repo-check (novox/hq ADR 0238 §4), written with its first commit (ADR 0428 §4):
|
|
||||||
# every Go module in the tree is vetted, its unit tests run, and then its integration tests (-tags integration).
|
|
||||||
# A tree with no Go module has nothing to test, and the check says so and passes.
|
|
||||||
#
|
|
||||||
# No module commits its dependencies (no vendor directory: .gitignore). A module's dependencies are fetched, or a
|
|
||||||
# requirement of the mesh's own forge (the pinned mesh-sdk commit) is taken from its repository's clone beside the check
|
|
||||||
# (MESH_CHECK_BESIDE), through a go.mod of its own (-modfile), by mesh-catalog's helper checks/deps, built from the
|
|
||||||
# clone of mesh-catalog beside this check and not copied here: the build seat forgets the forge credential before the
|
|
||||||
# check runs, so a commit the seat's module cache does not hold cannot be fetched from the forge (novox/hq issue 385). A
|
|
||||||
# module whose dependencies cannot be had **fails the check**, by name and why: its tests could not run, and a test that
|
|
||||||
# could not run is a failure, never a note. Run by hand, without MESH_CHECK_BESIDE, the dependencies are fetched.
|
|
||||||
set -eu
|
|
||||||
cd "$(dirname "$0")"
|
|
||||||
|
|
||||||
# The toolchain the check runs in sets -mod=vendor; there is no vendor directory, so the modules are used as they are.
|
|
||||||
GOFLAGS="$(printf '%s' "${GOFLAGS:-}" | sed 's/-mod=[a-z]*//g') -mod=mod"
|
|
||||||
export GOFLAGS
|
|
||||||
export GOPRIVATE=git.novox.be
|
|
||||||
|
|
||||||
modules=$(find . -name go.mod -not -path '*/vendor/*' -not -path './.git/*' | sort)
|
|
||||||
if [ -z "$modules" ]; then
|
|
||||||
echo "This tree holds no Go module, so there are no tests to run; the check passes."
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
work=$(mktemp -d)
|
|
||||||
trap 'rm -rf "$work"' EXIT
|
|
||||||
deps=""
|
|
||||||
helper="${MESH_CHECK_BESIDE:-}/mesh-catalog/checks/deps"
|
|
||||||
if [ -n "${MESH_CHECK_BESIDE:-}" ] && [ -f "$helper/main.go" ]; then
|
|
||||||
if ! (cd "$helper" && go build -o "$work/deps" .); then
|
|
||||||
echo "FAILED: the dependency helper of mesh-catalog beside this check could not be built"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
deps="$work/deps"
|
|
||||||
elif [ -n "${MESH_CHECK_BESIDE:-}" ]; then
|
|
||||||
echo "NOT TAKEN FROM BESIDE: no mesh-catalog/checks/deps beside this check ($MESH_CHECK_BESIDE); the dependencies are fetched"
|
|
||||||
fi
|
|
||||||
|
|
||||||
untestable=""
|
|
||||||
for mod in $modules; do
|
|
||||||
dir=$(dirname "$mod")
|
|
||||||
modfile=""
|
|
||||||
if [ -n "$deps" ]; then
|
|
||||||
if ! modfile=$("$deps" -beside "$MESH_CHECK_BESIDE" -work "$work" "$dir"); then
|
|
||||||
echo "FAILED: $dir could not be tested: $modfile"
|
|
||||||
untestable="$untestable $dir"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
[ -n "$modfile" ] && echo "$dir takes its forge dependencies from the clones beside the check"
|
|
||||||
elif ! fetched=$(cd "$dir" && go mod download 2>&1); then
|
|
||||||
echo "FAILED: $dir could not be tested: its dependencies cannot be fetched: $(printf '%s\n' "$fetched" | tail -n 1)"
|
|
||||||
untestable="$untestable $dir"
|
|
||||||
continue
|
|
||||||
fi
|
|
||||||
mf=""
|
|
||||||
[ -n "$modfile" ] && mf="-modfile=$modfile"
|
|
||||||
unformatted=$(cd "$dir" && gofmt -l .)
|
|
||||||
if [ -n "$unformatted" ]; then
|
|
||||||
echo "$dir is not gofmt'd: $unformatted"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "== $dir: go vet and the unit tests"
|
|
||||||
(cd "$dir" && go vet $mf ./... && go test $mf ./...)
|
|
||||||
echo "== $dir: the integration tests (-tags integration)"
|
|
||||||
(cd "$dir" && go vet $mf -tags integration ./... && go test $mf -tags integration ./...)
|
|
||||||
done
|
|
||||||
if [ -n "$untestable" ]; then
|
|
||||||
echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
|
)
|
||||||
|
|
||||||
|
// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events
|
||||||
|
// are kept in, as the controller's broker raises them.
|
||||||
|
type bed struct {
|
||||||
|
srv *server.Server
|
||||||
|
js jetstream.JetStream
|
||||||
|
ctx context.Context
|
||||||
|
dir string
|
||||||
|
buckets map[string]jetstream.KeyValue
|
||||||
|
}
|
||||||
|
|
||||||
|
func startBed(t *testing.T, leaveOut string) *bed {
|
||||||
|
t.Helper()
|
||||||
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "assigner")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||||
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go srv.Start()
|
||||||
|
if !srv.ReadyForConnections(10 * time.Second) {
|
||||||
|
t.Fatal("the bus did not come up")
|
||||||
|
}
|
||||||
|
t.Cleanup(srv.Shutdown)
|
||||||
|
nc, err := nats.Connect(srv.ClientURL())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(nc.Close)
|
||||||
|
js, err := jetstream.New(nc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
t.Cleanup(cancel)
|
||||||
|
b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}}
|
||||||
|
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||||
|
if name == leaveOut {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.buckets[name] = kv
|
||||||
|
}
|
||||||
|
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
// start runs the program as the node does, through run with its environment and a broker file; it answers the
|
||||||
|
// channel run's error arrives on.
|
||||||
|
func (b *bed) start(t *testing.T) <-chan error {
|
||||||
|
t.Helper()
|
||||||
|
broker := filepath.Join(b.dir, "broker")
|
||||||
|
cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"})
|
||||||
|
if err := os.WriteFile(broker, cred, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
env := map[string]string{"MESH_BROKER_FILE": broker}
|
||||||
|
ctx, stop := context.WithCancel(b.ctx)
|
||||||
|
done := make(chan error, 1)
|
||||||
|
go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }()
|
||||||
|
t.Cleanup(func() {
|
||||||
|
stop()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
case <-time.After(10 * time.Second):
|
||||||
|
}
|
||||||
|
})
|
||||||
|
return done
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable,
|
||||||
|
// having opened every bucket it names.
|
||||||
|
func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) {
|
||||||
|
b := startBed(t, "")
|
||||||
|
done := b.start(t)
|
||||||
|
events, err := b.js.Stream(b.ctx, "EVENTS")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var said announce.States
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("the assigner stopped: %v", err)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
m, err := events.GetLastMsgForSubject(b.ctx, announce.Subject("mesh-delivery-assigner"))
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(m.Data, &said); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") ||
|
||||||
|
!said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") {
|
||||||
|
t.Fatalf("the assigner said %+v, want its four states, each readable", said)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket.
|
||||||
|
func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) {
|
||||||
|
for _, missing := range readBuckets {
|
||||||
|
b := startBed(t, missing)
|
||||||
|
select {
|
||||||
|
case err := <-b.start(t):
|
||||||
|
if err == nil || !strings.Contains(err.Error(), missing) {
|
||||||
|
t.Errorf("without %s the assigner stopped with %v, want the bucket named", missing, err)
|
||||||
|
}
|
||||||
|
case <-time.After(15 * time.Second):
|
||||||
|
t.Errorf("without %s the assigner started", missing)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// putJSON writes a value into one of the bed's buckets.
|
||||||
|
func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil {
|
||||||
|
t.Fatalf("%s %s: %v", bucket, key, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest.
|
||||||
|
func (b *bed) manifest(t *testing.T, module, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
digest := delivery.ManifestDigest([]byte(body))
|
||||||
|
key, _ := delivery.ManifestsKey(digest)
|
||||||
|
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.putJSON(t, delivery.ManifestsBucket, key, value)
|
||||||
|
return digest
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within
|
||||||
|
// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with.
|
||||||
|
func TestAChangeIsComposedOnTheBus(t *testing.T) {
|
||||||
|
b := startBed(t, "")
|
||||||
|
trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||||
|
changed := b.manifest(t, "web", `{"module":"web","version":"2"}`)
|
||||||
|
mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||||
|
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`))
|
||||||
|
b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`))
|
||||||
|
heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}}
|
||||||
|
hash, _ := delivery.HeadsHash(heads)
|
||||||
|
// 1927/AC1: the candidate carries the fingerprint set the SDK computes, as the indexer writes it.
|
||||||
|
pulls := []delivery.CandidatePull{{Head: heads[0],
|
||||||
|
Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}
|
||||||
|
cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||||
|
b.putJSON(t, delivery.CandidatesBucket, ckey, cand)
|
||||||
|
began := time.Now()
|
||||||
|
b.start(t)
|
||||||
|
key, _ := delivery.CompositionKey("feat/x", hash)
|
||||||
|
var v delivery.CompositionVerdict
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("no composition under %s after 10 s", key)
|
||||||
|
}
|
||||||
|
e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if v, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" {
|
||||||
|
t.Fatalf("composition = %+v, want a pass on g14 alone naming its mesh-sdk", v)
|
||||||
|
}
|
||||||
|
if took := time.Since(began); took > 5*time.Second {
|
||||||
|
t.Errorf("the composition took %s", took)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get waits for a key of one of the bed's buckets and answers its value.
|
||||||
|
func (b *bed) get(t *testing.T, bucket, key string) []byte {
|
||||||
|
t.Helper()
|
||||||
|
for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err := b.buckets[bucket].Get(b.ctx, key); err == nil {
|
||||||
|
return e.Value()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t.Fatalf("no %s %s after 10 s", bucket, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole,
|
||||||
|
// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the
|
||||||
|
// controller sends ace the older fingerprint while ace wants F).
|
||||||
|
func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) {
|
||||||
|
b := startBed(t, "")
|
||||||
|
fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64)
|
||||||
|
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||||
|
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||||
|
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||||
|
built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
bkey, _ := delivery.BuilderBuildKey("web", fNew)
|
||||||
|
b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built)
|
||||||
|
gkey, _ := delivery.ControllerGrantKey("web", fNew)
|
||||||
|
grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{})
|
||||||
|
b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant)
|
||||||
|
for _, node := range []string{"ace", "g14"} {
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
b.putJSON(t, delivery.PresenceBucket, node, on)
|
||||||
|
have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
||||||
|
b.putJSON(t, delivery.HaveBucket, node+".web", have)
|
||||||
|
}
|
||||||
|
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`))
|
||||||
|
b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`))
|
||||||
|
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`))
|
||||||
|
b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`))
|
||||||
|
b.start(t)
|
||||||
|
|
||||||
|
wkey, _ := delivery.WantKey("ace", "web")
|
||||||
|
want, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey))
|
||||||
|
if err != nil || want.FP != fNew {
|
||||||
|
t.Fatalf("the canary ace's want = %+v, %v", want, err)
|
||||||
|
}
|
||||||
|
dkey, _ := delivery.DifferenceKey("g14", "web")
|
||||||
|
d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey))
|
||||||
|
if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew {
|
||||||
|
t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err)
|
||||||
|
}
|
||||||
|
akey, _ := delivery.DifferenceKey("ace", "web")
|
||||||
|
a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey))
|
||||||
|
if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld {
|
||||||
|
t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err)
|
||||||
|
}
|
||||||
|
if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil {
|
||||||
|
t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check
|
||||||
|
// failed on a node and writes the ask about it in its own asks, which the view and the teller read; its asks bucket is
|
||||||
|
// the one its manifest declares (ownBuckets, held equal to the manifest by TestOpensWhatItsManifestNames).
|
||||||
|
func TestAHeldFingerprintIsAskedAbout(t *testing.T) {
|
||||||
|
b := startBed(t, "")
|
||||||
|
fNew := delivery.SourcePrefix2 + strings.Repeat("c", 64)
|
||||||
|
digest := b.manifest(t, "web", `{"module":"web","version":"1"}`)
|
||||||
|
mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath})
|
||||||
|
b.putJSON(t, delivery.ModulesBucket, "web", mod)
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
b.putJSON(t, delivery.PresenceBucket, "ace", on)
|
||||||
|
failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"})
|
||||||
|
b.putJSON(t, delivery.HaveBucket, "ace.web", failed)
|
||||||
|
b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`))
|
||||||
|
b.start(t)
|
||||||
|
hkey, _ := delivery.HeldKey("web", fNew)
|
||||||
|
if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" {
|
||||||
|
t.Fatalf("held = %+v, %v", h, err)
|
||||||
|
}
|
||||||
|
akey, _ := delivery.HoldAskKey("web", fNew)
|
||||||
|
a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey))
|
||||||
|
if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 {
|
||||||
|
t.Fatalf("the ask = %+v, %v", a, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,225 @@
|
|||||||
|
// Command mesh-delivery-assigner is delivery's assigner (novox/hq ADR 0429 §1 and §6, ADR 0446, ADR 0528, issue 1297): a
|
||||||
|
// process on its own bus credential that watches what the indexer, the builder, the controller, the node-engines and
|
||||||
|
// the watchman write, and writes what each node should run (`want.<node>.<module>`, by the six rules), the
|
||||||
|
// fingerprints it holds, every difference between a want and what the controller sends, and the composition check's
|
||||||
|
// verdict of each group the checker judges. In release 5 the wants are only compared: the controller still sends what
|
||||||
|
// each node runs (ADR 0429 §7). It holds no timer: every pass is started by a change it watched.
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"os/signal"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/process"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
|
||||||
|
defer stop()
|
||||||
|
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
|
||||||
|
fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err)
|
||||||
|
os.Exit(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test
|
||||||
|
// holds the two lists equal), so the controller's broker grants exactly these.
|
||||||
|
var readBuckets = []string{
|
||||||
|
provider.AssignmentsBucket,
|
||||||
|
delivery.BuildsBucket,
|
||||||
|
delivery.ControllerGrantsBucket,
|
||||||
|
delivery.BuilderBuildsBucket,
|
||||||
|
delivery.CandidatesBucket,
|
||||||
|
delivery.ManifestsBucket,
|
||||||
|
delivery.ModulesBucket,
|
||||||
|
delivery.PresenceBucket,
|
||||||
|
delivery.HaveBucket,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ownBuckets are the assigner's own states, as its manifest declares them.
|
||||||
|
var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket}
|
||||||
|
|
||||||
|
// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view
|
||||||
|
// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view
|
||||||
|
// and the teller.
|
||||||
|
var keptStates = []announce.State{
|
||||||
|
{Name: "asks", Readable: true},
|
||||||
|
{Name: "compositions", Readable: true},
|
||||||
|
{Name: "differences", Readable: true},
|
||||||
|
{Name: "wants", Readable: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
// buckets is every bucket the assigner opened, by name.
|
||||||
|
type buckets map[string]jetstream.KeyValue
|
||||||
|
|
||||||
|
// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass
|
||||||
|
// without it would write wants from half the mesh.
|
||||||
|
func open(ctx context.Context, js jetstream.JetStream) (buckets, error) {
|
||||||
|
out := buckets{}
|
||||||
|
for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) {
|
||||||
|
kv, err := js.KeyValue(ctx, name)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err)
|
||||||
|
}
|
||||||
|
out[name] = kv
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// run is the program: it returns when ctx ends or the bus cannot be used.
|
||||||
|
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
|
||||||
|
logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags)
|
||||||
|
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
|
||||||
|
if path == "" {
|
||||||
|
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with")
|
||||||
|
}
|
||||||
|
account, err := process.ReadAccount(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
opts := []nats.Option{
|
||||||
|
nats.Name(account.Node + "." + account.Module),
|
||||||
|
nats.UserInfo(account.User, account.Password),
|
||||||
|
nats.CustomInboxPrefix(account.InboxPrefix()),
|
||||||
|
nats.MaxReconnects(-1),
|
||||||
|
nats.RetryOnFailedConnect(true),
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(account.Fingerprint) != "" {
|
||||||
|
opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint)))
|
||||||
|
}
|
||||||
|
conn, err := nats.Connect(account.URL, opts...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("connecting to the bus: %w", err)
|
||||||
|
}
|
||||||
|
defer conn.Close()
|
||||||
|
js, err := jetstream.New(conn)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
kvs, err := open(ctx, js)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// What the assigner keeps, said on the bus (ADR 0495 §2-§3): a reader is granted from it. Not fatal when it cannot
|
||||||
|
// be said now: process.Conn says it again at every reconnect.
|
||||||
|
said, err := process.Connect(account, nats.Name(account.Node+"."+account.Module+".keeps"), nats.RetryOnFailedConnect(true))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer said.Close()
|
||||||
|
if err := said.Keeps(keptStates...); err != nil {
|
||||||
|
logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err)
|
||||||
|
}
|
||||||
|
logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs))
|
||||||
|
return follow(ctx, kvs, logger)
|
||||||
|
}
|
||||||
|
|
||||||
|
// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and
|
||||||
|
// watched only for keys, so a manifest that arrives wakes a composition that waits for it.
|
||||||
|
var watched = []string{
|
||||||
|
provider.AssignmentsBucket,
|
||||||
|
delivery.ControllerGrantsBucket,
|
||||||
|
delivery.BuilderBuildsBucket,
|
||||||
|
delivery.CandidatesBucket,
|
||||||
|
delivery.ModulesBucket,
|
||||||
|
delivery.PresenceBucket,
|
||||||
|
delivery.HaveBucket,
|
||||||
|
delivery.AssignerAsksBucket,
|
||||||
|
delivery.AssignerCompositionsBucket,
|
||||||
|
delivery.AssignerDifferencesBucket,
|
||||||
|
delivery.AssignerWantsBucket,
|
||||||
|
}
|
||||||
|
|
||||||
|
// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after
|
||||||
|
// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends.
|
||||||
|
func follow(ctx context.Context, kvs buckets, logger *log.Logger) error {
|
||||||
|
w := newWorld()
|
||||||
|
dirty := make(chan struct{}, 1)
|
||||||
|
wake := func() {
|
||||||
|
select {
|
||||||
|
case dirty <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
errs := make(chan error, len(watched)+1)
|
||||||
|
settled := make(chan struct{}, len(watched)+1)
|
||||||
|
watch := func(name string, opts ...jetstream.WatchOpt) error {
|
||||||
|
kw, err := kvs[name].WatchAll(ctx, opts...)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("the bucket %s cannot be watched: %w", name, err)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer kw.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
errs <- nil
|
||||||
|
return
|
||||||
|
case e, ok := <-kw.Updates():
|
||||||
|
if !ok {
|
||||||
|
errs <- fmt.Errorf("the watch of %s ended", name)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if e == nil {
|
||||||
|
settled <- struct{}{}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var value []byte
|
||||||
|
if e.Operation() == jetstream.KeyValuePut {
|
||||||
|
value = e.Value()
|
||||||
|
}
|
||||||
|
w.put(name, e.Key(), value)
|
||||||
|
wake()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
for _, name := range watched {
|
||||||
|
var opts []jetstream.WatchOpt
|
||||||
|
if name == delivery.HaveBucket {
|
||||||
|
opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back
|
||||||
|
}
|
||||||
|
if err := watch(name, opts...); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for range len(watched) + 1 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case err := <-errs:
|
||||||
|
return err
|
||||||
|
case <-settled:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p := &passer{kvs: kvs, w: w, logf: logger.Printf}
|
||||||
|
wake()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil
|
||||||
|
case err := <-errs:
|
||||||
|
return err
|
||||||
|
case <-settled:
|
||||||
|
case <-dirty:
|
||||||
|
p.pass(ctx)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them
|
||||||
|
// (`<module>.<state>` read as the bucket `<module>_<state>`), so it is granted what it reads and reads nothing else.
|
||||||
|
func TestOpensWhatItsManifestNames(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile(filepath.Join("..", "..", "module.json"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
var reads []string
|
||||||
|
for _, r := range m.Reads {
|
||||||
|
reads = append(reads, strings.Replace(r, ".", "_", 1))
|
||||||
|
}
|
||||||
|
var states []string
|
||||||
|
for _, s := range m.State {
|
||||||
|
states = append(states, m.Module+"_"+s.Name)
|
||||||
|
}
|
||||||
|
for _, pair := range []struct {
|
||||||
|
what string
|
||||||
|
code, want []string
|
||||||
|
}{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} {
|
||||||
|
got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want))
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var kept []string
|
||||||
|
for _, s := range keptStates {
|
||||||
|
kept = append(kept, m.Module+"_"+s.Name)
|
||||||
|
}
|
||||||
|
if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) {
|
||||||
|
t.Errorf("the program announces %v, keeps %v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B2a: without a credential the program does not start, and says why.
|
||||||
|
func TestNoCredential(t *testing.T) {
|
||||||
|
err := run(context.Background(), func(string) string { return "" }, io.Discard)
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") {
|
||||||
|
t.Errorf("run without a credential = %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/sdkversion"
|
||||||
|
|
||||||
|
"mesh-delivery-assigner/internal/assign"
|
||||||
|
)
|
||||||
|
|
||||||
|
// passer makes the assigner's passes: from what the world holds now, it writes each key whose value differs from the
|
||||||
|
// one written, and only those (the Same functions).
|
||||||
|
type passer struct {
|
||||||
|
kvs buckets
|
||||||
|
w *world
|
||||||
|
logf func(string, ...any)
|
||||||
|
sdk string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *passer) pass(ctx context.Context) {
|
||||||
|
p.compositions(ctx)
|
||||||
|
rules := p.wants(ctx)
|
||||||
|
p.differences(ctx, rules)
|
||||||
|
}
|
||||||
|
|
||||||
|
// differences writes, for each node and module the assigner wants or the controller sends, the difference between the
|
||||||
|
// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that
|
||||||
|
// ended; its history keeps what it was.
|
||||||
|
func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) {
|
||||||
|
_, assignments := p.w.assigned()
|
||||||
|
wanted := p.w.wantsOf()
|
||||||
|
trunk := p.w.trunk()
|
||||||
|
pairs := map[[2]string]bool{}
|
||||||
|
for _, a := range assignments {
|
||||||
|
pairs[[2]string{a.Node, a.Module}] = true
|
||||||
|
}
|
||||||
|
for module, byNode := range wanted {
|
||||||
|
for node := range byNode {
|
||||||
|
pairs[[2]string{node, module}] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
written := p.w.read(delivery.AssignerDifferencesBucket)
|
||||||
|
keep := map[string]bool{}
|
||||||
|
for pair := range pairs {
|
||||||
|
node, module := pair[0], pair[1]
|
||||||
|
key, err := delivery.DifferenceKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var want *delivery.Want
|
||||||
|
if w, ok := wanted[module][node]; ok {
|
||||||
|
want = &w
|
||||||
|
}
|
||||||
|
var sent assign.Sent
|
||||||
|
if a, ok := assignments[node+"."+module]; ok {
|
||||||
|
sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)}
|
||||||
|
}
|
||||||
|
d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node])
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keep[key] = true
|
||||||
|
if before, err := delivery.DecodeDifference(written[key]); err == nil {
|
||||||
|
before.V = 0
|
||||||
|
if delivery.SameDifference(before, d) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) })
|
||||||
|
}
|
||||||
|
for key := range written {
|
||||||
|
if keep[key] {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil {
|
||||||
|
p.logf("the difference %s that ended could not be deleted: %v", key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p.w.put(delivery.AssignerDifferencesBucket, key, nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and
|
||||||
|
// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node
|
||||||
|
// back, for the comparison.
|
||||||
|
func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding {
|
||||||
|
get := kvGetter(p.kvs[delivery.ManifestsBucket])
|
||||||
|
assigned, _ := p.w.assigned()
|
||||||
|
trunk := p.w.trunk()
|
||||||
|
manifests := map[string]catalogue.Manifest{}
|
||||||
|
for name, mod := range trunk {
|
||||||
|
if mod.Manifest == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" {
|
||||||
|
manifests[name] = got.manifest
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodes := p.w.nodes(manifests, assigned)
|
||||||
|
nodesOf := map[string][]string{}
|
||||||
|
for node, modules := range assigned {
|
||||||
|
for _, m := range modules {
|
||||||
|
nodesOf[m] = append(nodesOf[m], node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
wanted := p.w.wantsOf()
|
||||||
|
rules := map[string]map[string]assign.Holding{}
|
||||||
|
names := map[string]bool{}
|
||||||
|
for name := range trunk {
|
||||||
|
names[name] = true
|
||||||
|
}
|
||||||
|
for name := range wanted {
|
||||||
|
names[name] = true
|
||||||
|
}
|
||||||
|
for name := range names {
|
||||||
|
fp := trunk[name].FP2
|
||||||
|
m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]}
|
||||||
|
if fp != "" {
|
||||||
|
held, _ := delivery.HeldKey(name, fp)
|
||||||
|
grant, _ := delivery.ControllerGrantKey(name, fp)
|
||||||
|
m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant)
|
||||||
|
}
|
||||||
|
last := map[string]string{}
|
||||||
|
for _, node := range m.Nodes {
|
||||||
|
last[node] = p.w.lastHealthy(node, name, fp)
|
||||||
|
}
|
||||||
|
out := assign.Wants(m, nodes, wanted[name], last)
|
||||||
|
rules[name] = out.Rules
|
||||||
|
if out.Held != nil {
|
||||||
|
key, err := delivery.HeldKey(name, fp)
|
||||||
|
if err == nil {
|
||||||
|
held := *out.Held
|
||||||
|
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if out.Held != nil || m.Held {
|
||||||
|
p.holdAsk(ctx, name, fp)
|
||||||
|
}
|
||||||
|
for node, want := range out.Wants {
|
||||||
|
key, err := delivery.WantKey(node, name)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("no want of %s on %s is written: %v", name, node, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
want := want
|
||||||
|
p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return rules
|
||||||
|
}
|
||||||
|
|
||||||
|
// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the
|
||||||
|
// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part.
|
||||||
|
func (p *passer) compositions(ctx context.Context) {
|
||||||
|
if p.sdk == "" {
|
||||||
|
p.sdk = sdkversion.Version()
|
||||||
|
}
|
||||||
|
written := p.w.read(delivery.AssignerCompositionsBucket)
|
||||||
|
trunk := p.w.trunk()
|
||||||
|
var mesh *assign.Mesh
|
||||||
|
for key, value := range p.w.read(delivery.CandidatesBucket) {
|
||||||
|
c, err := delivery.DecodeCandidate(value)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("the candidate %s cannot be read, so it is not composed: %v", key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out, err := delivery.CompositionKey(c.Group, c.HeadsHash)
|
||||||
|
if err != nil {
|
||||||
|
p.logf("the candidate %s names no composition key: %v", key, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
began := time.Now()
|
||||||
|
ch, waits := p.w.change(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]), c, trunk)
|
||||||
|
var v delivery.CompositionVerdict
|
||||||
|
if waits != "" {
|
||||||
|
v = delivery.CompositionVerdict{Group: c.Group, Heads: c.HeadsHash, State: delivery.CompositionWaits, Words: waits}
|
||||||
|
} else {
|
||||||
|
if mesh == nil {
|
||||||
|
m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]))
|
||||||
|
mesh = &m
|
||||||
|
}
|
||||||
|
v = assign.Compose(*mesh, ch, p.sdk, time.Since(began))
|
||||||
|
}
|
||||||
|
if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil {
|
||||||
|
before.Took, before.V = v.Took, 0
|
||||||
|
if delivery.SameCompositionVerdict(before, v) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
p.put(ctx, delivery.AssignerCompositionsBucket, out, func() ([]byte, error) { return delivery.EncodeCompositionVerdict(v) })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass.
|
||||||
|
func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) {
|
||||||
|
value, err := encode()
|
||||||
|
if err != nil {
|
||||||
|
p.logf("%s %s is not written: %v", bucket, key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := p.kvs[bucket].Put(ctx, key, value); err != nil && !errors.Is(err, context.Canceled) {
|
||||||
|
p.logf("%s %s could not be written: %v", bucket, key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.w.put(bucket, key, value)
|
||||||
|
}
|
||||||
|
|
||||||
|
// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs
|
||||||
|
// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a
|
||||||
|
// node (wants are only compared, ADR 0429 §7), and the options say so.
|
||||||
|
func (p *passer) holdAsk(ctx context.Context, module, fp string) {
|
||||||
|
key, err := delivery.HoldAskKey(module, fp)
|
||||||
|
if err != nil || p.w.has(delivery.AssignerAsksBucket, key) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
a := assign.HoldAsk(key, module, fp, time.Now())
|
||||||
|
p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) })
|
||||||
|
}
|
||||||
@@ -0,0 +1,370 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
|
|
||||||
|
"mesh-delivery-assigner/internal/assign"
|
||||||
|
)
|
||||||
|
|
||||||
|
// noModuleWords is the indexer's reason for a module of a candidate whose directory the merge preview holds no manifest
|
||||||
|
// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module.
|
||||||
|
const noModuleWords = delivery.CandidateNoneNoModule
|
||||||
|
|
||||||
|
// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view
|
||||||
|
// internal/view/view.go): an assignment stands while it has not ended and has no state.
|
||||||
|
type assignment struct {
|
||||||
|
Node string `json:"node"`
|
||||||
|
Module string `json:"module"`
|
||||||
|
Build string `json:"build"`
|
||||||
|
Ended time.Time `json:"ended"`
|
||||||
|
State string `json:"state"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" }
|
||||||
|
|
||||||
|
// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by
|
||||||
|
// digest. The watches write it; a pass reads it.
|
||||||
|
type world struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
values map[string]map[string][]byte
|
||||||
|
manifests map[string]parsed
|
||||||
|
// healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to.
|
||||||
|
healthy map[string][]string
|
||||||
|
// sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one.
|
||||||
|
sent map[string]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read.
|
||||||
|
type parsed struct {
|
||||||
|
module string
|
||||||
|
manifest catalogue.Manifest
|
||||||
|
err string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newWorld() *world {
|
||||||
|
return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged.
|
||||||
|
func (w *world) put(bucket, key string, value []byte) {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
if w.values[bucket] == nil {
|
||||||
|
w.values[bucket] = map[string][]byte{}
|
||||||
|
}
|
||||||
|
if value == nil {
|
||||||
|
delete(w.values[bucket], key)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.values[bucket][key] = value
|
||||||
|
if bucket == delivery.HaveBucket {
|
||||||
|
if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||||
|
list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP })
|
||||||
|
w.healthy[key] = append(list, h.FP)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty.
|
||||||
|
func (w *world) lastHealthy(node, module, fp string) string {
|
||||||
|
key, err := delivery.HaveKey(node, module)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
list := w.healthy[key]
|
||||||
|
for i := len(list) - 1; i >= 0; i-- {
|
||||||
|
if list[i] != fp {
|
||||||
|
return list[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// read is a copy of what a bucket holds now.
|
||||||
|
func (w *world) read(bucket string) map[string][]byte {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
out := make(map[string][]byte, len(w.values[bucket]))
|
||||||
|
for k, v := range w.values[bucket] {
|
||||||
|
out[k] = v
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// getter reads one key's value; errNotThere when the key does not exist.
|
||||||
|
type getter func(ctx context.Context, key string) ([]byte, error)
|
||||||
|
|
||||||
|
var errNotThere = errors.New("no such key")
|
||||||
|
|
||||||
|
// kvGetter reads keys of a bucket.
|
||||||
|
func kvGetter(kv jetstream.KeyValue) getter {
|
||||||
|
return func(ctx context.Context, key string) ([]byte, error) {
|
||||||
|
e, err := kv.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return nil, errNotThere
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return e.Value(), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes.
|
||||||
|
func (w *world) manifest(ctx context.Context, get getter, digest string) parsed {
|
||||||
|
w.mu.Lock()
|
||||||
|
p, ok := w.manifests[digest]
|
||||||
|
w.mu.Unlock()
|
||||||
|
if ok {
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
key, err := delivery.ManifestsKey(digest)
|
||||||
|
if err != nil {
|
||||||
|
return parsed{err: err.Error()}
|
||||||
|
}
|
||||||
|
value, err := get(ctx, key)
|
||||||
|
if errors.Is(err, errNotThere) {
|
||||||
|
return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"}
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return parsed{err: err.Error()}
|
||||||
|
}
|
||||||
|
m, err := delivery.DecodeManifest(value)
|
||||||
|
if err != nil {
|
||||||
|
p = parsed{err: err.Error()}
|
||||||
|
} else if man, err := catalogue.ParseManifest([]byte(m.JSON)); err != nil {
|
||||||
|
p = parsed{module: m.Module, err: err.Error()}
|
||||||
|
} else {
|
||||||
|
p = parsed{module: m.Module, manifest: man}
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
w.manifests[digest] = p
|
||||||
|
w.mu.Unlock()
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order.
|
||||||
|
func (w *world) assigned() (map[string][]string, map[string]assignment) {
|
||||||
|
out, all := map[string][]string{}, map[string]assignment{}
|
||||||
|
for _, value := range w.read(provider.AssignmentsBucket) {
|
||||||
|
var a assignment
|
||||||
|
if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[a.Node] = append(out[a.Node], a.Module)
|
||||||
|
all[a.Node+"."+a.Module] = a
|
||||||
|
}
|
||||||
|
for _, mods := range out {
|
||||||
|
sort.Strings(mods)
|
||||||
|
}
|
||||||
|
return out, all
|
||||||
|
}
|
||||||
|
|
||||||
|
// trunk is every module the indexer holds at the trunk, with its value.
|
||||||
|
func (w *world) trunk() map[string]delivery.Module {
|
||||||
|
out := map[string]delivery.Module{}
|
||||||
|
for key, value := range w.read(delivery.ModulesBucket) {
|
||||||
|
name, err := delivery.ParseModulesKey(key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if m, err := delivery.DecodeModule(value); err == nil {
|
||||||
|
out[name] = m
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// mesh is the mesh as the trunk holds it, for the composition check.
|
||||||
|
func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh {
|
||||||
|
m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}}
|
||||||
|
m.Assigned, _ = w.assigned()
|
||||||
|
for name, mod := range w.trunk() {
|
||||||
|
if mod.Manifest == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
p := w.manifest(ctx, manifests, mod.Manifest)
|
||||||
|
if p.err != "" {
|
||||||
|
m.Unreadable[name] = p.err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m.Manifests[name] = p.manifest
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// change is a candidate's change for the composition check, or why the composition waits for it.
|
||||||
|
func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) {
|
||||||
|
ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Twice: map[string][]string{}}
|
||||||
|
definedIn := map[string]map[string]bool{}
|
||||||
|
for _, p := range c.Pulls {
|
||||||
|
if p.Repository == assign.SDKRepository {
|
||||||
|
ch.SDKHead = p.Head.Head
|
||||||
|
}
|
||||||
|
for _, name := range p.Modules {
|
||||||
|
fp, ok := p.Fps[name]
|
||||||
|
if !ok || fp.Manifest == "" {
|
||||||
|
if ok && fp.None2 == noModuleWords {
|
||||||
|
ch.Removed = append(ch.Removed, name)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number)
|
||||||
|
}
|
||||||
|
if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 {
|
||||||
|
ch.Rebuilt++
|
||||||
|
}
|
||||||
|
got := w.manifest(ctx, manifests, fp.Manifest)
|
||||||
|
if strings.HasPrefix(got.err, "the indexer holds no manifest") {
|
||||||
|
return ch, got.err
|
||||||
|
}
|
||||||
|
if got.err != "" {
|
||||||
|
ch.Unreadable[name] = got.err
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ch.Manifests[got.module] = got.manifest
|
||||||
|
if definedIn[got.module] == nil {
|
||||||
|
definedIn[got.module] = map[string]bool{}
|
||||||
|
}
|
||||||
|
definedIn[got.module][p.Repository] = true
|
||||||
|
if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository {
|
||||||
|
definedIn[got.module][t.Repository] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, repos := range definedIn {
|
||||||
|
if len(repos) > 1 {
|
||||||
|
for r := range repos {
|
||||||
|
ch.Twice[name] = append(ch.Twice[name], r)
|
||||||
|
}
|
||||||
|
sort.Strings(ch.Twice[name])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ch, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// wantsOf is the wants the assigner wrote, by module then node.
|
||||||
|
func (w *world) wantsOf() map[string]map[string]delivery.Want {
|
||||||
|
out := map[string]map[string]delivery.Want{}
|
||||||
|
for key, value := range w.read(delivery.AssignerWantsBucket) {
|
||||||
|
node, module, err := delivery.ParseWantKey(key)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
want, err := delivery.DecodeWant(value)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if out[module] == nil {
|
||||||
|
out[module] = map[string]delivery.Want{}
|
||||||
|
}
|
||||||
|
out[module][node] = want
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes.
|
||||||
|
func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes {
|
||||||
|
n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}}
|
||||||
|
for key, value := range w.read(delivery.PresenceBucket) {
|
||||||
|
node, _, _, err := delivery.ParsePresenceKey(key)
|
||||||
|
if err != nil || node == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p, err := delivery.DecodePresence(value); err == nil {
|
||||||
|
n.Online[node] = p.Online
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for key, value := range w.read(delivery.HaveBucket) {
|
||||||
|
node, module, isNode, err := delivery.ParseHaveKey(key)
|
||||||
|
if err != nil || isNode {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if h, err := delivery.DecodeHave(value); err == nil {
|
||||||
|
if n.Haves[node] == nil {
|
||||||
|
n.Haves[node] = map[string]delivery.Have{}
|
||||||
|
}
|
||||||
|
n.Haves[node][module] = h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
nodesOf := map[string][]string{}
|
||||||
|
for node, modules := range assigned {
|
||||||
|
for _, m := range modules {
|
||||||
|
nodesOf[m] = append(nodesOf[m], node)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for name, man := range manifests {
|
||||||
|
for _, o := range man.Provides {
|
||||||
|
if n.Providers[o.Name] == nil {
|
||||||
|
n.Providers[o.Name] = map[string][]string{}
|
||||||
|
}
|
||||||
|
n.Providers[o.Name][name] = nodesOf[name]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
|
// has says whether a bucket holds a key now.
|
||||||
|
func (w *world) has(bucket, key string) bool {
|
||||||
|
w.mu.Lock()
|
||||||
|
defer w.mu.Unlock()
|
||||||
|
_, ok := w.values[bucket][key]
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
|
// built says whether the builder's build of a module at a fingerprint is done.
|
||||||
|
func (w *world) built(module, fp string) bool {
|
||||||
|
key, err := delivery.BuilderBuildKey(module, fp)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
value, ok := w.values[delivery.BuilderBuildsBucket][key]
|
||||||
|
w.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
b, err := delivery.DecodeBuilderBuild(value)
|
||||||
|
return err == nil && b.State == delivery.BuildDone
|
||||||
|
}
|
||||||
|
|
||||||
|
// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record
|
||||||
|
// (mesh-controller_builds `<module>.build.<id>`) once and kept; empty when the record names none.
|
||||||
|
func (w *world) sentFP(ctx context.Context, get getter, module, build string) string {
|
||||||
|
if build == "" {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
fp, ok := w.sent[build]
|
||||||
|
w.mu.Unlock()
|
||||||
|
if ok {
|
||||||
|
return fp
|
||||||
|
}
|
||||||
|
value, err := get(ctx, delivery.BuildKey(module, build))
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
var rec delivery.BuildRecord
|
||||||
|
if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
w.mu.Lock()
|
||||||
|
w.sent[build] = rec.SourceFingerprint
|
||||||
|
w.mu.Unlock()
|
||||||
|
return rec.SourceFingerprint
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-assigner/internal/assign"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeManifests is the indexer's manifests bucket in memory.
|
||||||
|
type fakeManifests map[string][]byte
|
||||||
|
|
||||||
|
func (f fakeManifests) add(t *testing.T, module, body string) string {
|
||||||
|
t.Helper()
|
||||||
|
digest := delivery.ManifestDigest([]byte(body))
|
||||||
|
value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
f[digest] = value
|
||||||
|
return digest
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) {
|
||||||
|
if v, ok := f[key]; ok {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
return nil, errNotThere
|
||||||
|
}
|
||||||
|
|
||||||
|
func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||||
|
|
||||||
|
func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull {
|
||||||
|
p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules}
|
||||||
|
for m := range modules {
|
||||||
|
p.Modules = append(p.Modules, m)
|
||||||
|
}
|
||||||
|
slices.Sort(p.Modules)
|
||||||
|
return p
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a
|
||||||
|
// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from
|
||||||
|
// another repository as defined twice.
|
||||||
|
func TestTheChangeOfACandidate(t *testing.T) {
|
||||||
|
f := fakeManifests{}
|
||||||
|
web := f.add(t, "web", `{"module":"web","version":"2"}`)
|
||||||
|
mail := f.add(t, "mail", `{"module":"mail","version":"1"}`)
|
||||||
|
trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}}
|
||||||
|
c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{
|
||||||
|
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}),
|
||||||
|
pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}),
|
||||||
|
pull(assign.SDKRepository, 3, nil),
|
||||||
|
}}
|
||||||
|
ch, waits := newWorld().change(context.Background(), f.get, c, trunk)
|
||||||
|
if waits != "" {
|
||||||
|
t.Fatalf("the change waits: %s", waits)
|
||||||
|
}
|
||||||
|
if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) {
|
||||||
|
t.Errorf("change = %+v", ch)
|
||||||
|
}
|
||||||
|
if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 {
|
||||||
|
t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits,
|
||||||
|
// saying for what; an unreadable manifest is the change's, not a wait.
|
||||||
|
func TestAChangeWaitsForItsManifests(t *testing.T) {
|
||||||
|
f := fakeManifests{}
|
||||||
|
bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`)
|
||||||
|
for _, c := range []delivery.CandidatePull{
|
||||||
|
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}),
|
||||||
|
pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}),
|
||||||
|
} {
|
||||||
|
_, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil)
|
||||||
|
if waits == "" {
|
||||||
|
t.Errorf("a change without its manifest %+v does not wait", c.Fps)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab",
|
||||||
|
Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil)
|
||||||
|
if waits != "" || ch.Unreadable["web"] == "" {
|
||||||
|
t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
module mesh-delivery-assigner
|
||||||
|
|
||||||
|
go 1.26.0
|
||||||
|
|
||||||
|
require (
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
|
github.com/nats-io/nats.go v1.54.0
|
||||||
|
)
|
||||||
|
|
||||||
|
require (
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
|
||||||
|
github.com/google/go-tpm v0.9.8 // indirect
|
||||||
|
github.com/klauspost/compress v1.20.0 // indirect
|
||||||
|
github.com/minio/highwayhash v1.0.4 // indirect
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.1 // indirect
|
||||||
|
github.com/nats-io/nkeys v0.4.16 // indirect
|
||||||
|
github.com/nats-io/nuid v1.0.1 // indirect
|
||||||
|
golang.org/x/crypto v0.57.0 // indirect
|
||||||
|
golang.org/x/sys v0.48.0 // indirect
|
||||||
|
golang.org/x/time v0.15.0 // indirect
|
||||||
|
)
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0=
|
||||||
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
|
||||||
|
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
|
||||||
|
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
|
||||||
|
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
|
||||||
|
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
|
||||||
|
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
|
||||||
|
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
|
||||||
|
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
|
||||||
|
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
|
||||||
|
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
|
||||||
|
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
|
||||||
|
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
|
||||||
|
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
|
||||||
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
|
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
|
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||||
|
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||||
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/asks"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep
|
||||||
|
// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer
|
||||||
|
// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6.
|
||||||
|
func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk {
|
||||||
|
return delivery.AssignerAsk{Ask: asks.Ask{
|
||||||
|
ID: delivery.AskingID(key, 0),
|
||||||
|
Headline: fmt.Sprintf("%s is held at %s", module, short(fp)),
|
||||||
|
Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+
|
||||||
|
"no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)),
|
||||||
|
Options: []asks.Option{
|
||||||
|
{ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge},
|
||||||
|
{ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge},
|
||||||
|
},
|
||||||
|
Who: asks.Operator,
|
||||||
|
Expires: now.Add(7 * 24 * time.Hour),
|
||||||
|
OnExpiry: "the version stays held",
|
||||||
|
About: key,
|
||||||
|
}}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at
|
||||||
|
// its key, of a kind the view may answer.
|
||||||
|
func TestTheHoldAskIsValid(t *testing.T) {
|
||||||
|
key, err := delivery.HoldAskKey("mail", fNew)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
now := time.Now()
|
||||||
|
a := HoldAsk(key, "mail", fNew, now)
|
||||||
|
if err := a.Ask.Check(now); err != nil {
|
||||||
|
t.Errorf("the router refuses the ask: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeAssignerAsk(key, a); err != nil {
|
||||||
|
t.Errorf("the ask cannot be written: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := delivery.ViewAnswerKey(key); err != nil {
|
||||||
|
t.Errorf("the view may not answer it: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its
|
||||||
|
// assignment names, and that build's fingerprint of the new form (empty when its record names none).
|
||||||
|
type Sent struct {
|
||||||
|
Sends bool
|
||||||
|
Build string
|
||||||
|
FP string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4),
|
||||||
|
// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and
|
||||||
|
// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the
|
||||||
|
// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said
|
||||||
|
// in words, for acceptance test 17 of design 47 to read.
|
||||||
|
func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) {
|
||||||
|
var d delivery.Difference
|
||||||
|
removed := want != nil && want.Removed
|
||||||
|
if want != nil && !removed {
|
||||||
|
d.Want = want.FP
|
||||||
|
}
|
||||||
|
d.WantRemoved = removed
|
||||||
|
if !sent.Sends {
|
||||||
|
if want == nil || removed {
|
||||||
|
return delivery.Difference{}, false
|
||||||
|
}
|
||||||
|
d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing"
|
||||||
|
return d, true
|
||||||
|
}
|
||||||
|
d.Sent, d.SentBuild = sent.FP, sent.Build
|
||||||
|
switch {
|
||||||
|
case sent.FP == "":
|
||||||
|
d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build)
|
||||||
|
case removed:
|
||||||
|
d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP))
|
||||||
|
case sent.FP == d.Want:
|
||||||
|
return delivery.Difference{}, false
|
||||||
|
case hold.Rule != 0 && sent.FP == newest:
|
||||||
|
d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words)
|
||||||
|
case d.Want == "":
|
||||||
|
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP))
|
||||||
|
default:
|
||||||
|
d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want))
|
||||||
|
}
|
||||||
|
return d, true
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3.
|
||||||
|
func TestADifferenceExplainedByTheCanary(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
out := Wants(m, n, wantedOld(), nil)
|
||||||
|
want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary}
|
||||||
|
d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"])
|
||||||
|
if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew {
|
||||||
|
t.Fatalf("difference = %+v, %v; want rule 3", d, ok)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||||
|
t.Errorf("the difference cannot be written: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written
|
||||||
|
// unexplained, with words, and is writable.
|
||||||
|
func TestCompare(t *testing.T) {
|
||||||
|
w := delivery.Want{FP: fNew}
|
||||||
|
if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok {
|
||||||
|
t.Errorf("an equal want and send is a difference")
|
||||||
|
}
|
||||||
|
if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok {
|
||||||
|
t.Errorf("no want and nothing sent is a difference")
|
||||||
|
}
|
||||||
|
removed := delivery.Want{Removed: true}
|
||||||
|
if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok {
|
||||||
|
t.Errorf("a removed want and nothing sent is a difference")
|
||||||
|
}
|
||||||
|
old := delivery.Want{FP: fOld}
|
||||||
|
for name, c := range map[string]struct {
|
||||||
|
want *delivery.Want
|
||||||
|
sent Sent
|
||||||
|
}{
|
||||||
|
"sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}},
|
||||||
|
"names no fingerprint": {&w, Sent{Sends: true, Build: "b"}},
|
||||||
|
"sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||||
|
"sends nothing": {&w, Sent{}},
|
||||||
|
"no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||||
|
"no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}},
|
||||||
|
} {
|
||||||
|
d, ok := Compare(c.want, c.sent, fNew, Holding{})
|
||||||
|
if !ok || !d.Unexplained || d.Words == "" {
|
||||||
|
t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeDifference(d); err != nil {
|
||||||
|
t.Errorf("%s: the difference cannot be written: %v", name, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,269 @@
|
|||||||
|
// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a
|
||||||
|
// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should
|
||||||
|
// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads
|
||||||
|
// and writes nothing itself: the program hands it what it watched and writes what it answers.
|
||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/compose"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself.
|
||||||
|
const SDKRepository = "novox/mesh-sdk"
|
||||||
|
|
||||||
|
// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the
|
||||||
|
// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461;
|
||||||
|
// the design limit of R4g, issue 1297): said on every verdict, never passed silently.
|
||||||
|
const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " +
|
||||||
|
"credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " +
|
||||||
|
"node-engine's validator would refuse"
|
||||||
|
|
||||||
|
// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with
|
||||||
|
// why, and each node's open assignments.
|
||||||
|
type Mesh struct {
|
||||||
|
Manifests map[string]catalogue.Manifest
|
||||||
|
Unreadable map[string]string
|
||||||
|
Assigned map[string][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those
|
||||||
|
// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk
|
||||||
|
// member (empty when none), and how many modules it rebuilds.
|
||||||
|
type Change struct {
|
||||||
|
Group, Heads string
|
||||||
|
Manifests map[string]catalogue.Manifest
|
||||||
|
Unreadable map[string]string
|
||||||
|
Removed []string
|
||||||
|
Twice map[string][]string
|
||||||
|
SDKHead string
|
||||||
|
Rebuilt int
|
||||||
|
}
|
||||||
|
|
||||||
|
// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or
|
||||||
|
// removes, those running a consumer of a provision such a module provides (before or after), and those running a
|
||||||
|
// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere
|
||||||
|
// touches no machine.
|
||||||
|
func Touched(m Mesh, c Change) []string {
|
||||||
|
moved := map[string]bool{}
|
||||||
|
for name := range c.Manifests {
|
||||||
|
moved[name] = true
|
||||||
|
}
|
||||||
|
for name := range c.Unreadable {
|
||||||
|
moved[name] = true
|
||||||
|
}
|
||||||
|
for _, name := range c.Removed {
|
||||||
|
moved[name] = true
|
||||||
|
}
|
||||||
|
provided, required := map[string]bool{}, map[string]bool{}
|
||||||
|
for name := range moved {
|
||||||
|
for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} {
|
||||||
|
for _, o := range man.Provides {
|
||||||
|
provided[o.Name] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, r := range c.Manifests[name].Requires {
|
||||||
|
required[r] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for node, modules := range m.Assigned {
|
||||||
|
for _, mod := range modules {
|
||||||
|
man := m.Manifests[mod]
|
||||||
|
if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) ||
|
||||||
|
slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) {
|
||||||
|
out = append(out, node)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned
|
||||||
|
// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the
|
||||||
|
// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made
|
||||||
|
// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing).
|
||||||
|
func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input {
|
||||||
|
in := compose.Input{Catalogue: catalog}
|
||||||
|
nodes := make([]string, 0, len(assigned))
|
||||||
|
for n := range assigned {
|
||||||
|
nodes = append(nodes, n)
|
||||||
|
}
|
||||||
|
sort.Strings(nodes)
|
||||||
|
for _, n := range nodes {
|
||||||
|
in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])})
|
||||||
|
for _, mod := range assigned[n] {
|
||||||
|
for _, cl := range catalog[mod].Claims {
|
||||||
|
scope := cl.Scope
|
||||||
|
if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" {
|
||||||
|
scope = seat.Scope
|
||||||
|
}
|
||||||
|
in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return in
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched
|
||||||
|
// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot
|
||||||
|
// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the
|
||||||
|
// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past
|
||||||
|
// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk
|
||||||
|
// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took.
|
||||||
|
func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict {
|
||||||
|
v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()}
|
||||||
|
var fails []string
|
||||||
|
failAt := func(machine, words string) {
|
||||||
|
if v.Machine == "" && machine != "" {
|
||||||
|
v.Machine = machine
|
||||||
|
}
|
||||||
|
fails = append(fails, words)
|
||||||
|
}
|
||||||
|
for _, name := range sortedKeys(c.Unreadable) {
|
||||||
|
failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name]))
|
||||||
|
}
|
||||||
|
for _, name := range sortedKeys(c.Twice) {
|
||||||
|
failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and ")))
|
||||||
|
}
|
||||||
|
machines := Touched(m, c)
|
||||||
|
v.Machines = machines
|
||||||
|
for _, node := range machines {
|
||||||
|
for _, name := range c.Removed {
|
||||||
|
if slices.Contains(m.Assigned[node], name) {
|
||||||
|
failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
trunk := input(m.Manifests, m.Assigned)
|
||||||
|
changed := map[string]catalogue.Manifest{}
|
||||||
|
for name, man := range m.Manifests {
|
||||||
|
changed[name] = man
|
||||||
|
}
|
||||||
|
for name, man := range c.Manifests {
|
||||||
|
changed[name] = man
|
||||||
|
}
|
||||||
|
for _, name := range c.Removed {
|
||||||
|
delete(changed, name)
|
||||||
|
}
|
||||||
|
after := input(changed, m.Assigned)
|
||||||
|
for _, node := range machines {
|
||||||
|
_, before := compose.Resolve(trunk, node)
|
||||||
|
_, now := compose.Resolve(after, node)
|
||||||
|
switch {
|
||||||
|
case now == nil:
|
||||||
|
case before != nil:
|
||||||
|
v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before))
|
||||||
|
default:
|
||||||
|
failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.Rebuilt > delivery.CompositionRebuildWarning {
|
||||||
|
v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning))
|
||||||
|
}
|
||||||
|
if c.SDKHead != "" {
|
||||||
|
v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+
|
||||||
|
"and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead))
|
||||||
|
}
|
||||||
|
v.Warnings = append(v.Warnings, NotJudged)
|
||||||
|
if could := CouldRun(m, c, machines); len(could) > 0 {
|
||||||
|
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||||
|
"the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", "))
|
||||||
|
}
|
||||||
|
if grants := GrantsAlone(m, c, machines); len(grants) > 0 {
|
||||||
|
v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+
|
||||||
|
"the machines whose grants alone the change alters: "+strings.Join(grants, ", "))
|
||||||
|
}
|
||||||
|
if len(machines) == 0 {
|
||||||
|
v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere")
|
||||||
|
}
|
||||||
|
if len(fails) > 0 {
|
||||||
|
v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ")
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
v.State = delivery.CompositionPass
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
|
||||||
|
// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds
|
||||||
|
// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in
|
||||||
|
// the facts snapshot), so each is named as not judged.
|
||||||
|
func CouldRun(m Mesh, c Change, composed []string) []string {
|
||||||
|
var out []string
|
||||||
|
for node, modules := range m.Assigned {
|
||||||
|
if slices.Contains(composed, node) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for name := range c.Manifests {
|
||||||
|
if !slices.Contains(modules, name) {
|
||||||
|
out = append(out, node)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers.
|
||||||
|
func grantsOf(man catalogue.Manifest) string {
|
||||||
|
var offers []string
|
||||||
|
for _, o := range man.Provides {
|
||||||
|
offers = append(offers, o.Name)
|
||||||
|
}
|
||||||
|
return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants)
|
||||||
|
}
|
||||||
|
|
||||||
|
// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a
|
||||||
|
// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries
|
||||||
|
// every module's bus grants) and those assigned a provider of a provision it requires, before or after.
|
||||||
|
func GrantsAlone(m Mesh, c Change, composed []string) []string {
|
||||||
|
provisions := map[string]bool{}
|
||||||
|
altered := false
|
||||||
|
for name, man := range c.Manifests {
|
||||||
|
before, had := m.Manifests[name]
|
||||||
|
if had && grantsOf(before) == grantsOf(man) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
altered = true
|
||||||
|
for _, r := range append(slices.Clone(man.Requires), before.Requires...) {
|
||||||
|
provisions[r] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !altered {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for node, modules := range m.Assigned {
|
||||||
|
if slices.Contains(composed, node) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, mod := range modules {
|
||||||
|
man := m.Manifests[mod]
|
||||||
|
if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) {
|
||||||
|
out = append(out, node)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func sortedKeys[V any](m map[string]V) []string {
|
||||||
|
out := make([]string, 0, len(m))
|
||||||
|
for k := range m {
|
||||||
|
out = append(out, k)
|
||||||
|
}
|
||||||
|
sort.Strings(out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
func meshOf() Mesh {
|
||||||
|
return Mesh{
|
||||||
|
Manifests: map[string]catalogue.Manifest{
|
||||||
|
"mail": {Module: "mail", Version: "1", Requires: []string{"database"}},
|
||||||
|
"store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}},
|
||||||
|
"web": {Module: "web", Version: "1"},
|
||||||
|
},
|
||||||
|
Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func hasWarning(v delivery.CompositionVerdict, part string) bool {
|
||||||
|
return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) })
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names
|
||||||
|
// the machine and the mesh-sdk it composed with, and says what it did not judge.
|
||||||
|
func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) {
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}}
|
||||||
|
v := Compose(meshOf(), c, "v0.1.14-x", 0)
|
||||||
|
if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" {
|
||||||
|
t.Fatalf("composition = %+v, want a pass on g14 alone", v)
|
||||||
|
}
|
||||||
|
if !hasWarning(v, "issue 1461") {
|
||||||
|
t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings)
|
||||||
|
}
|
||||||
|
if _, err := delivery.EncodeCompositionVerdict(v); err != nil {
|
||||||
|
t.Errorf("the verdict cannot be written: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module.
|
||||||
|
func TestRemovingAModuleAMachineRunsFails(t *testing.T) {
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}}
|
||||||
|
v := Compose(meshOf(), c, "sdk", 0)
|
||||||
|
if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") {
|
||||||
|
t.Fatalf("composition = %+v, want a fail naming g14 and web", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the
|
||||||
|
// consumer's machine is touched when its provider changes.
|
||||||
|
func TestAMachineTheChangeBreaksFails(t *testing.T) {
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}}
|
||||||
|
v := Compose(meshOf(), c, "sdk", 0)
|
||||||
|
if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") {
|
||||||
|
t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3c: what was already broken on the trunk is said and fails nothing.
|
||||||
|
func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) {
|
||||||
|
m := meshOf()
|
||||||
|
m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"}
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}}
|
||||||
|
v := Compose(m, c, "sdk", 0)
|
||||||
|
if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") {
|
||||||
|
t.Fatalf("composition = %+v, want a pass saying ace was already broken", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn.
|
||||||
|
func TestUnreadableTwiceAndWide(t *testing.T) {
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13}
|
||||||
|
v := Compose(meshOf(), c, "sdk", 0)
|
||||||
|
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") {
|
||||||
|
t.Fatalf("composition = %+v", v)
|
||||||
|
}
|
||||||
|
c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12}
|
||||||
|
v = Compose(meshOf(), c, "sdk", 0)
|
||||||
|
if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") {
|
||||||
|
t.Fatalf("composition = %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and
|
||||||
|
// the verdict says so, naming the member's head it did not use; a group without one says nothing of it.
|
||||||
|
func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) {
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"}
|
||||||
|
v := Compose(meshOf(), c, "v0.1.14-own", 0)
|
||||||
|
if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") {
|
||||||
|
t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings)
|
||||||
|
}
|
||||||
|
c.SDKHead = ""
|
||||||
|
if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") {
|
||||||
|
t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module
|
||||||
|
// assigned nowhere touches none, and the verdict says so.
|
||||||
|
func TestTouched(t *testing.T) {
|
||||||
|
m := meshOf()
|
||||||
|
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) {
|
||||||
|
t.Errorf("touched by store = %v", got)
|
||||||
|
}
|
||||||
|
m.Assigned["g14"] = append(m.Assigned["g14"], "mail")
|
||||||
|
if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) {
|
||||||
|
t.Errorf("touched by store with a consumer on g14 = %v", got)
|
||||||
|
}
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}}
|
||||||
|
if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass {
|
||||||
|
t.Errorf("an added module assigned nowhere: %+v", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned
|
||||||
|
// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not
|
||||||
|
// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant
|
||||||
|
// names none of the second.
|
||||||
|
func TestTheMachinesNotJudgedAreNamed(t *testing.T) {
|
||||||
|
m := meshOf()
|
||||||
|
m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}}
|
||||||
|
m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}}
|
||||||
|
m.Assigned["novox"] = []string{"bus"}
|
||||||
|
m.Assigned["shanks"] = []string{"db"}
|
||||||
|
c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{
|
||||||
|
"web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}}
|
||||||
|
v := Compose(m, c, "sdk", 0)
|
||||||
|
if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") {
|
||||||
|
t.Errorf("the machines that could run web are not named: %v", v.Warnings)
|
||||||
|
}
|
||||||
|
if !hasWarning(v, "whose grants alone the change alters: novox") {
|
||||||
|
t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings)
|
||||||
|
}
|
||||||
|
c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"}
|
||||||
|
if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") {
|
||||||
|
t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,205 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"sort"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the
|
||||||
|
// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller
|
||||||
|
// issued its credential for F, its manifest at F, and the nodes it is assigned to.
|
||||||
|
type Module struct {
|
||||||
|
Name string
|
||||||
|
FP string
|
||||||
|
Built bool
|
||||||
|
Held bool
|
||||||
|
Granted bool
|
||||||
|
Manifest catalogue.Manifest
|
||||||
|
Nodes []string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have.
|
||||||
|
type Nodes struct {
|
||||||
|
Online map[string]bool
|
||||||
|
Haves map[string]map[string]delivery.Have
|
||||||
|
// Providers is, by provision, the modules that provide it, with the nodes each is assigned to.
|
||||||
|
Providers map[string]map[string][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
// healthy says whether a have runs fp healthy.
|
||||||
|
func healthy(h delivery.Have, fp string) bool {
|
||||||
|
return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
|
||||||
|
}
|
||||||
|
|
||||||
|
// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by
|
||||||
|
// name, ADR 0528 clause 3). Empty when none is online.
|
||||||
|
func Canary(nodes []string, online map[string]bool) string {
|
||||||
|
sorted := slices.Sorted(slices.Values(nodes))
|
||||||
|
for _, n := range sorted {
|
||||||
|
if online[n] {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not,
|
||||||
|
// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no
|
||||||
|
// limit (ADR 0528 clause 3).
|
||||||
|
func Holds(m Module, node string, n Nodes) (bool, int, string) {
|
||||||
|
switch {
|
||||||
|
case !m.Built:
|
||||||
|
return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP))
|
||||||
|
case m.Held:
|
||||||
|
return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP))
|
||||||
|
}
|
||||||
|
canary := Canary(m.Nodes, n.Online)
|
||||||
|
switch {
|
||||||
|
case canary == "":
|
||||||
|
return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name)
|
||||||
|
case node != canary && !healthy(n.Haves[canary][m.Name], m.FP):
|
||||||
|
return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP))
|
||||||
|
}
|
||||||
|
for _, provision := range m.Manifest.Requires {
|
||||||
|
if ok, why := providedHealthy(provision, node, n); !ok {
|
||||||
|
return false, delivery.RuleProviders, why
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !m.Granted {
|
||||||
|
return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP))
|
||||||
|
}
|
||||||
|
return true, 0, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned
|
||||||
|
// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node
|
||||||
|
// offers it) and holds.
|
||||||
|
func providedHealthy(provision, node string, n Nodes) (bool, string) {
|
||||||
|
providers := n.Providers[provision]
|
||||||
|
if len(providers) == 0 {
|
||||||
|
return true, ""
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(providers))
|
||||||
|
for p := range providers {
|
||||||
|
names = append(names, p)
|
||||||
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
for _, p := range names {
|
||||||
|
nodes := providers[p]
|
||||||
|
at := node
|
||||||
|
if !slices.Contains(nodes, node) {
|
||||||
|
at = Canary(nodes, n.Online)
|
||||||
|
}
|
||||||
|
if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK {
|
||||||
|
return true, ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node)
|
||||||
|
}
|
||||||
|
|
||||||
|
// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable,
|
||||||
|
// valuable or rebuildable; cache and none do not).
|
||||||
|
func keepsData(m catalogue.Manifest) bool {
|
||||||
|
for _, it := range m.DataItems() {
|
||||||
|
if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write.
|
||||||
|
type Outcome struct {
|
||||||
|
Wants map[string]delivery.Want
|
||||||
|
// Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet.
|
||||||
|
Held *delivery.Held
|
||||||
|
// Rules is, by node, the rule holding the node back from m.FP and why, for the comparison.
|
||||||
|
Rules map[string]Holding
|
||||||
|
}
|
||||||
|
|
||||||
|
// Holding is a rule holding a node back, and why.
|
||||||
|
type Holding struct {
|
||||||
|
Rule int
|
||||||
|
Words string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written
|
||||||
|
// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed
|
||||||
|
// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a
|
||||||
|
// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A
|
||||||
|
// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4).
|
||||||
|
func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome {
|
||||||
|
out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}}
|
||||||
|
held := m.Held
|
||||||
|
if !held && m.FP != "" {
|
||||||
|
for _, node := range slices.Sorted(slices.Values(m.Nodes)) {
|
||||||
|
if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed {
|
||||||
|
out.Held = &delivery.Held{Node: node, Words: h.Detail}
|
||||||
|
held = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
mm := m
|
||||||
|
mm.Held = held
|
||||||
|
for _, node := range m.Nodes {
|
||||||
|
if m.FP == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
before, had := wants[node]
|
||||||
|
var next delivery.Want
|
||||||
|
switch ok, rule, words := Holds(mm, node, n); {
|
||||||
|
case ok:
|
||||||
|
next = delivery.Want{FP: m.FP}
|
||||||
|
case held && had && before.FP == m.FP:
|
||||||
|
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||||
|
last := lastHealthy[node]
|
||||||
|
switch {
|
||||||
|
case keepsData(m.Manifest):
|
||||||
|
next = before
|
||||||
|
next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP))
|
||||||
|
case last == "":
|
||||||
|
next = before
|
||||||
|
next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node)
|
||||||
|
default:
|
||||||
|
next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
out.Rules[node] = Holding{Rule: rule, Words: words}
|
||||||
|
if !had || before.Removed {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words}
|
||||||
|
if before.FP == m.FP {
|
||||||
|
next = delivery.Want{FP: m.FP}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !had || !delivery.SameWant(normal(before), normal(next)) {
|
||||||
|
out.Wants[node] = next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for node, before := range wants {
|
||||||
|
if !slices.Contains(m.Nodes, node) && !before.Removed {
|
||||||
|
out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// normal is a want without its version, as a decoded and a composed one are compared.
|
||||||
|
func normal(w delivery.Want) delivery.Want {
|
||||||
|
w.V = 0
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
||||||
|
func short(fp string) string {
|
||||||
|
part, err := delivery.FingerprintKeyPart(fp)
|
||||||
|
if err != nil || len(part) < 12 {
|
||||||
|
return fp
|
||||||
|
}
|
||||||
|
return part[:12]
|
||||||
|
}
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
package assign
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) }
|
||||||
|
|
||||||
|
var (
|
||||||
|
fNew = fpOf('b')
|
||||||
|
fOld = fpOf('a')
|
||||||
|
)
|
||||||
|
|
||||||
|
// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider
|
||||||
|
// store healthy on ace and the canary ace running fNew healthy.
|
||||||
|
func ready() (Module, Nodes) {
|
||||||
|
m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"},
|
||||||
|
Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}}
|
||||||
|
n := Nodes{
|
||||||
|
Online: map[string]bool{"ace": true, "g14": true},
|
||||||
|
Haves: map[string]map[string]delivery.Have{
|
||||||
|
"ace": {"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, "store": {FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
||||||
|
"g14": {"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}},
|
||||||
|
},
|
||||||
|
Providers: map[string]map[string][]string{"database": {"store": {"ace"}}},
|
||||||
|
}
|
||||||
|
return m, n
|
||||||
|
}
|
||||||
|
|
||||||
|
func wantedOld() map[string]delivery.Want {
|
||||||
|
return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so
|
||||||
|
// g14 follows at once: step 0, ADR 0528 clause 3).
|
||||||
|
func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
out := Wants(m, n, wantedOld(), nil)
|
||||||
|
for _, node := range []string{"ace", "g14"} {
|
||||||
|
if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 {
|
||||||
|
t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4, one test per rule: given every rule but N holds, no want moves to F, and the rule N is named.
|
||||||
|
func TestEachRuleHoldsTheWantBack(t *testing.T) {
|
||||||
|
for rule, breakIt := range map[int]func(*Module, *Nodes){
|
||||||
|
delivery.RuleBuildDone: func(m *Module, _ *Nodes) { m.Built = false },
|
||||||
|
delivery.RuleNotHeld: func(m *Module, _ *Nodes) { m.Held = true },
|
||||||
|
delivery.RuleCanary: func(_ *Module, n *Nodes) {
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
},
|
||||||
|
delivery.RuleProviders: func(_ *Module, n *Nodes) {
|
||||||
|
n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits}
|
||||||
|
},
|
||||||
|
delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false },
|
||||||
|
} {
|
||||||
|
m, n := ready()
|
||||||
|
breakIt(&m, &n)
|
||||||
|
out := Wants(m, n, wantedOld(), nil)
|
||||||
|
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||||
|
t.Errorf("rule %d broken: g14's want moved to F", rule)
|
||||||
|
}
|
||||||
|
if got := out.Rules["g14"].Rule; got != rule {
|
||||||
|
t.Errorf("rule %d broken: g14 is held back by rule %d (%s)", rule, got, out.Rules["g14"].Words)
|
||||||
|
}
|
||||||
|
if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) {
|
||||||
|
t.Errorf("rule %d broken: g14's want is %+v", rule, w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first.
|
||||||
|
func TestAnOfflineCanaryIsPassedOver(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
n.Online["ace"] = false
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
n.Providers["database"]["store"] = []string{"ace", "g14"}
|
||||||
|
n.Haves["g14"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}
|
||||||
|
if c := Canary(m.Nodes, n.Online); c != "g14" {
|
||||||
|
t.Fatalf("canary = %q, want g14", c)
|
||||||
|
}
|
||||||
|
out := Wants(m, n, wantedOld(), nil)
|
||||||
|
if out.Wants["g14"].FP != fNew {
|
||||||
|
t.Errorf("the canary g14's want = %+v", out.Wants["g14"])
|
||||||
|
}
|
||||||
|
if out.Rules["ace"].Rule != delivery.RuleCanary {
|
||||||
|
t.Errorf("ace, behind the canary g14, is held back by %+v", out.Rules["ace"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4: a failed health check of F holds F and sets the nodes that want it back to their last healthy fingerprint.
|
||||||
|
func TestAFailedHealthHoldsAndSetsBack(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}
|
||||||
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld})
|
||||||
|
if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" {
|
||||||
|
t.Fatalf("held = %+v", out.Held)
|
||||||
|
}
|
||||||
|
if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld {
|
||||||
|
t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w)
|
||||||
|
}
|
||||||
|
if w, ok := out.Wants["g14"]; ok && w.FP == fNew {
|
||||||
|
t.Errorf("g14 moved to a held fingerprint: %+v", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4: a module holding data is not set back; its want stays and says why.
|
||||||
|
func TestADataModuleIsNotSetBack(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}}
|
||||||
|
n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed}
|
||||||
|
out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld})
|
||||||
|
if out.Held == nil {
|
||||||
|
t.Fatal("F was not held")
|
||||||
|
}
|
||||||
|
if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") {
|
||||||
|
t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that
|
||||||
|
// is already as the rules give it is not written again.
|
||||||
|
func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) {
|
||||||
|
m, n := ready()
|
||||||
|
m.Nodes = []string{"ace"}
|
||||||
|
out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil)
|
||||||
|
if w := out.Wants["g14"]; !w.Removed {
|
||||||
|
t.Errorf("g14's want = %+v, want removed", w)
|
||||||
|
}
|
||||||
|
if _, ok := out.Wants["ace"]; ok {
|
||||||
|
t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"])
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
package manifest
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
|
)
|
||||||
|
|
||||||
|
func load(t *testing.T) catalogue.Manifest {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := os.ReadFile("module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
m, err := catalogue.ParseManifest(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err)
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the
|
||||||
|
// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps
|
||||||
|
// its three states, and holds a credential of its own.
|
||||||
|
func TestManifest(t *testing.T) {
|
||||||
|
m := load(t)
|
||||||
|
if m.Module != "mesh-delivery-assigner" {
|
||||||
|
t.Errorf("module = %q", m.Module)
|
||||||
|
}
|
||||||
|
reads := slices.Clone(m.Reads)
|
||||||
|
slices.Sort(reads)
|
||||||
|
want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants",
|
||||||
|
"mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests",
|
||||||
|
"mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}
|
||||||
|
if !reflect.DeepEqual(reads, want) {
|
||||||
|
t.Errorf("reads = %v, want exactly %v", reads, want)
|
||||||
|
}
|
||||||
|
states := map[string]int{}
|
||||||
|
for _, s := range m.State {
|
||||||
|
states[s.Name] = s.History
|
||||||
|
}
|
||||||
|
if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) {
|
||||||
|
t.Errorf("state = %v, want %v", states, want)
|
||||||
|
}
|
||||||
|
if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 {
|
||||||
|
t.Errorf("own-secrets = %v, want exactly broker (or no credential is issued)", m.OwnSecrets)
|
||||||
|
}
|
||||||
|
if m.SecretsOwner != "mesh-delivery-assigner" {
|
||||||
|
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||||
|
}
|
||||||
|
if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 {
|
||||||
|
t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,71 @@
|
|||||||
|
{
|
||||||
|
"module": "mesh-delivery-assigner",
|
||||||
|
"version": "1",
|
||||||
|
"slug": "assigner",
|
||||||
|
"reads": [
|
||||||
|
"mesh-controller.assignments",
|
||||||
|
"mesh-controller.builds",
|
||||||
|
"mesh-controller.grants",
|
||||||
|
"mesh-delivery-builder.builds",
|
||||||
|
"mesh-delivery-indexer.candidates",
|
||||||
|
"mesh-delivery-indexer.manifests",
|
||||||
|
"mesh-delivery-indexer.modules",
|
||||||
|
"mesh-delivery-watchman.presence",
|
||||||
|
"node-engine.have"
|
||||||
|
],
|
||||||
|
"state": [
|
||||||
|
{ "name": "asks", "history": 5 },
|
||||||
|
{ "name": "compositions", "history": 5 },
|
||||||
|
{ "name": "differences", "history": 64 },
|
||||||
|
{ "name": "wants", "history": 20 }
|
||||||
|
],
|
||||||
|
"own-secrets": {
|
||||||
|
"broker": "${dir:state}/broker"
|
||||||
|
},
|
||||||
|
"secrets-owner": "mesh-delivery-assigner",
|
||||||
|
"resources": [
|
||||||
|
{
|
||||||
|
"id": "account",
|
||||||
|
"type": "user",
|
||||||
|
"name": "mesh-delivery-assigner",
|
||||||
|
"shell": "/usr/bin/nologin",
|
||||||
|
"home": "/var/lib/mesh-delivery-assigner"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "state",
|
||||||
|
"type": "directory",
|
||||||
|
"mode": "0700",
|
||||||
|
"owner": "mesh-delivery-assigner",
|
||||||
|
"place": ".",
|
||||||
|
"unknown": "program"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "assigner",
|
||||||
|
"type": "process",
|
||||||
|
"name": "mesh-delivery-assigner",
|
||||||
|
"artifact": "program",
|
||||||
|
"run": [
|
||||||
|
"./mesh-delivery-assigner"
|
||||||
|
],
|
||||||
|
"env": {
|
||||||
|
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||||
|
},
|
||||||
|
"user": "mesh-delivery-assigner",
|
||||||
|
"health": {
|
||||||
|
"kind": "unit"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"build": {
|
||||||
|
"artifacts": [
|
||||||
|
{
|
||||||
|
"name": "program",
|
||||||
|
"kind": "bundle",
|
||||||
|
"language": "go",
|
||||||
|
"system": "arch",
|
||||||
|
"from": "cmd/mesh-delivery-assigner",
|
||||||
|
"binary": "mesh-delivery-assigner"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -102,43 +102,6 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the
|
|
||||||
// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every
|
|
||||||
// change after. No timer: the caller bounds the wait with its context.
|
|
||||||
type stepsWaiter struct{ js bucketOpener }
|
|
||||||
|
|
||||||
func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
|
||||||
kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module))
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
|
||||||
}
|
|
||||||
watch, err := kv.Watch(ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Step{}, err
|
|
||||||
}
|
|
||||||
defer watch.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return delivery.Step{}, ctx.Err()
|
|
||||||
case e, ok := <-watch.Updates():
|
|
||||||
if !ok {
|
|
||||||
return delivery.Step{}, errors.New("the watch of the step ended")
|
|
||||||
}
|
|
||||||
if e == nil || !opOf(e) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
step, err := delivery.DecodeStep(e.Value())
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err)
|
|
||||||
}
|
|
||||||
if step.State != delivery.StepRunning {
|
|
||||||
return step, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
||||||
type manifestReader struct{ kv jetstream.KeyValue }
|
type manifestReader struct{ kv jetstream.KeyValue }
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A done key whose blob the artifact store no longer holds is built again, by the builder whole against the bus
|
||||||
|
// (novox/hq issue 1937): on 2026-10-24 the store's 03:30 collection deleted the bare blobs of three done keys the
|
||||||
|
// controller had never taken in, and the builder kept them done.
|
||||||
|
|
||||||
|
// heldVerb answers the controller seat's verb held on the bus, as the serving controller does: {result: {output, ok,
|
||||||
|
// answer}}; a reference is gone while gone says so.
|
||||||
|
type heldVerb struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
gone func(reference string) bool
|
||||||
|
asked []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *heldVerb) serve(t *testing.T, b *bed) {
|
||||||
|
t.Helper()
|
||||||
|
sub, err := b.js.Conn().Subscribe("mesh.seat.mesh-controller.call.held.>", func(m *nats.Msg) {
|
||||||
|
var args struct {
|
||||||
|
References []string `json:"references"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(m.Data, &args)
|
||||||
|
a := delivery.HeldAnswer{Store: "store.test:5100", Held: []string{}, Gone: []string{}}
|
||||||
|
h.mu.Lock()
|
||||||
|
for _, r := range args.References {
|
||||||
|
h.asked = append(h.asked, r)
|
||||||
|
if h.gone(r) {
|
||||||
|
a.Gone = append(a.Gone, r)
|
||||||
|
} else {
|
||||||
|
a.Held = append(a.Held, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
h.mu.Unlock()
|
||||||
|
reply, _ := json.Marshal(map[string]any{"result": map[string]any{"output": "held", "ok": true, "answer": a}, "node": "ctl"})
|
||||||
|
_ = m.Respond(reply)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *heldVerb) asks() int {
|
||||||
|
h.mu.Lock()
|
||||||
|
defer h.mu.Unlock()
|
||||||
|
return len(h.asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// eventsConsumer makes EVENTS and the builder's durable consumer of artifacts-gone on it, as the controller makes them
|
||||||
|
// from the manifest's consumes.
|
||||||
|
func eventsConsumer(t *testing.T, b *bed, node string) {
|
||||||
|
t.Helper()
|
||||||
|
if _, err := b.js.CreateStream(b.ctx, jetstream.StreamConfig{Name: eventsStream, Subjects: []string{delivery.ArtifactsGoneSubject}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := b.js.CreateOrUpdateConsumer(b.ctx, eventsStream, jetstream.ConsumerConfig{Durable: node + "_mesh-delivery-builder",
|
||||||
|
AckPolicy: jetstream.AckExplicitPolicy, FilterSubjects: []string{delivery.ArtifactsGoneSubject}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// doneKey writes builds.<module>.<fp> done, naming one archive at reference, and answers its revision.
|
||||||
|
func doneKey(t *testing.T, b *bed, module, manifest, reference, digest string) uint64 {
|
||||||
|
t.Helper()
|
||||||
|
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||||
|
data, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "n2.mesh-delivery-builder",
|
||||||
|
Manifest: manifest, Step: "built mesh-build-go on n2",
|
||||||
|
Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Digest: digest, Reference: reference}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rev, err := b.builds.Put(b.ctx, key, data)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return rev
|
||||||
|
}
|
||||||
|
|
||||||
|
func revisionOf(t *testing.T, b *bed, module string) uint64 {
|
||||||
|
t.Helper()
|
||||||
|
key, _ := delivery.BuilderBuildKey(module, fp)
|
||||||
|
e, err := b.builds.Get(b.ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return e.Revision()
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1937/AC3 (integration): the builder, started whole, finds the done key of the trunk's fingerprint naming a blob held
|
||||||
|
// answers gone, builds it again as a new revision whose step says so, and leaves the rebuilt key done.
|
||||||
|
func TestADoneKeyWhoseBlobIsGoneIsBuiltAgainAtStart(t *testing.T) {
|
||||||
|
b := startBed(t)
|
||||||
|
eventsConsumer(t, b, "n1")
|
||||||
|
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||||
|
old := digestOf("collected last night")
|
||||||
|
oldRef := "artifact-store://ca-trust/files/blobs/" + old
|
||||||
|
rev := doneKey(t, b, "ca-trust", manifest, oldRef, old)
|
||||||
|
h := &heldVerb{gone: func(r string) bool { return r == oldRef }}
|
||||||
|
h.serve(t, b)
|
||||||
|
|
||||||
|
b.start(t, "n1", nil)
|
||||||
|
v := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool {
|
||||||
|
return v.State == delivery.BuildDone && len(v.Artifacts) == 1 && v.Artifacts[0].Digest == digestOf("files")
|
||||||
|
})
|
||||||
|
if !strings.Contains(v.Step, "built again: the artifact store no longer held "+oldRef) {
|
||||||
|
t.Errorf("the rebuilt key's step %q does not say why it was built again", v.Step)
|
||||||
|
}
|
||||||
|
if got := revisionOf(t, b, "ca-trust"); got <= rev {
|
||||||
|
t.Errorf("revision %d, was %d: want a new revision", got, rev)
|
||||||
|
}
|
||||||
|
if n := b.fake.count("mesh-build-go.build_go_build"); n != 1 {
|
||||||
|
t.Errorf("the archive was built %d times, want once", n)
|
||||||
|
}
|
||||||
|
if h.asks() == 0 {
|
||||||
|
t.Error("the builder never asked held")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1937/AC4 (integration): a done key whose blob held answers held at start stays done; when the controller says
|
||||||
|
// artifacts-gone for it, the builder asks again and builds it again.
|
||||||
|
func TestArtifactsGoneLeadsTheBuilderToBuildTheKeyAgain(t *testing.T) {
|
||||||
|
b := startBed(t)
|
||||||
|
eventsConsumer(t, b, "n1")
|
||||||
|
manifest := b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
||||||
|
d := digestOf("files")
|
||||||
|
ref := "artifact-store://ca-trust/files/blobs/" + d
|
||||||
|
rev := doneKey(t, b, "ca-trust", manifest, ref, d)
|
||||||
|
var mu sync.Mutex
|
||||||
|
collected := false
|
||||||
|
h := &heldVerb{gone: func(r string) bool {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
if r == ref && collected {
|
||||||
|
collected = false // gone once: the rebuild pushes it again
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}}
|
||||||
|
h.serve(t, b)
|
||||||
|
|
||||||
|
b.start(t, "n1", nil)
|
||||||
|
for deadline := time.Now().Add(20 * time.Second); h.asks() == 0 && time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
}
|
||||||
|
if h.asks() == 0 {
|
||||||
|
t.Fatal("the builder never asked held at its start")
|
||||||
|
}
|
||||||
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
if got := revisionOf(t, b, "ca-trust"); got != rev || b.fake.count("mesh-build-go.build_go_build") != 0 {
|
||||||
|
t.Fatalf("a key whose blob is held was built again (revision %d, was %d)", got, rev)
|
||||||
|
}
|
||||||
|
|
||||||
|
mu.Lock()
|
||||||
|
collected = true
|
||||||
|
mu.Unlock()
|
||||||
|
key, _ := delivery.BuilderBuildKey("ca-trust", fp)
|
||||||
|
body, err := delivery.EncodeArtifactsGone(delivery.ArtifactsGone{Module: "ca-trust", Key: key, Fingerprint: fp,
|
||||||
|
Gone: []string{ref}, At: time.Now().UTC()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := b.js.Publish(b.ctx, delivery.ArtifactsGoneSubject, body); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
v := b.await(t, "ca-trust", func(v delivery.BuilderBuild) bool {
|
||||||
|
return v.State == delivery.BuildDone && strings.Contains(v.Step, "built again")
|
||||||
|
})
|
||||||
|
if got := revisionOf(t, b, "ca-trust"); got <= rev || b.fake.count("mesh-build-go.build_go_build") != 1 {
|
||||||
|
t.Errorf("build = %+v at revision %d (was %d), %d builds: want built again once on the event", v, got, rev,
|
||||||
|
b.fake.count("mesh-build-go.build_go_build"))
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it
|
||||||
|
// (issue 1910): read once, at the take-over of a key at start, never polled.
|
||||||
|
type heldOnTheBus struct{ js jetstream.JetStream }
|
||||||
|
|
||||||
|
// Held is build.Builder's Held: a step key `<node>.<ask id>` in the build module's steps, or a message of the stream under
|
||||||
|
// the ask's subject whose Nats-Msg-Id is the ask's id.
|
||||||
|
func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) {
|
||||||
|
module, err := delivery.BuildModuleOf(a.Language)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
pattern, err := delivery.AskStepWatch(a.ID)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module))
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
||||||
|
}
|
||||||
|
keys, err := steps.ListKeysFiltered(ctx, pattern)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for range keys.Keys() {
|
||||||
|
keys.Stop()
|
||||||
|
return true, nil // a node took it
|
||||||
|
}
|
||||||
|
subject, err := delivery.AskSubject(a.Language, a.Kind)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
// The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its
|
||||||
|
// messages by sequence and nothing wider (issue 1910).
|
||||||
|
name, err := delivery.AskStream(a.Language)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
stream, err := h.js.Stream(ctx, name)
|
||||||
|
if errors.Is(err, jetstream.ErrStreamNotFound) {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
// A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence
|
||||||
|
// instead, each a direct read that consumes nothing.
|
||||||
|
info, err := stream.Info(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ {
|
||||||
|
m, err := stream.GetMsg(ctx, seq)
|
||||||
|
if errors.Is(err, jetstream.ErrMsgNotFound) {
|
||||||
|
continue // taken and acknowledged, or removed
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID {
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
@@ -21,6 +22,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -46,13 +48,12 @@ const (
|
|||||||
fp = "src2:3333333333333333333333333333333333333333333333333333333333333333"
|
fp = "src2:3333333333333333333333333333333333333333333333333333333333333333"
|
||||||
)
|
)
|
||||||
|
|
||||||
// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on
|
// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls
|
||||||
// mesh.mod.<module>.call.<tool>.<node>.<caller> answers at once with the step and its key, and the step's end is written to
|
// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that
|
||||||
// the key <node>.<id> of the module's steps bucket, some milliseconds later.
|
// answer as the real tools do, ending the step on <node>.<ask id> of its steps bucket.
|
||||||
type fakeBuildModules struct {
|
type fakeBuildModules struct {
|
||||||
js jetstream.JetStream
|
js jetstream.JetStream
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
n atomic.Int64
|
|
||||||
failBuild atomic.Bool
|
failBuild atomic.Bool
|
||||||
// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
|
// refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2).
|
||||||
refusePush atomic.Bool
|
refusePush atomic.Bool
|
||||||
@@ -60,6 +61,21 @@ type fakeBuildModules struct {
|
|||||||
calls map[string]int // by `<module>.<tool>`
|
calls map[string]int // by `<module>.<tool>`
|
||||||
fetches []map[string]any
|
fetches []map[string]any
|
||||||
builds []map[string]any // the arguments of every build call
|
builds []map[string]any // the arguments of every build call
|
||||||
|
// closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks
|
||||||
|
// waiting in their streams).
|
||||||
|
closed atomic.Bool
|
||||||
|
pullers []*buildstep.Puller
|
||||||
|
}
|
||||||
|
|
||||||
|
// open lets the nodes pull again and wakes their pullers.
|
||||||
|
func (f *fakeBuildModules) open() {
|
||||||
|
f.closed.Store(false)
|
||||||
|
f.mu.Lock()
|
||||||
|
ps := append([]*buildstep.Puller(nil), f.pullers...)
|
||||||
|
f.mu.Unlock()
|
||||||
|
for _, p := range ps {
|
||||||
|
p.Wake()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeBuildModules) count(tool string) int {
|
func (f *fakeBuildModules) count(tool string) int {
|
||||||
@@ -74,76 +90,76 @@ func digestOf(artifact string) string {
|
|||||||
return "sha256:" + hex.EncodeToString(sum[:])
|
return "sha256:" + hex.EncodeToString(sum[:])
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
|
var prefixOf = map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_", "mesh-build-image": "build_image_"}
|
||||||
steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
|
||||||
|
// step is one tool's answer, as the real module's.
|
||||||
|
func (f *fakeBuildModules) step(module, name string, args map[string]any) (any, error) {
|
||||||
|
tool := prefixOf[module] + name
|
||||||
|
f.mu.Lock()
|
||||||
|
f.calls[module+"."+tool]++
|
||||||
|
if name == "fetch" {
|
||||||
|
f.fetches = append(f.fetches, args)
|
||||||
|
} else {
|
||||||
|
f.builds = append(f.builds, args)
|
||||||
|
}
|
||||||
|
f.mu.Unlock()
|
||||||
|
switch {
|
||||||
|
case name == "fetch":
|
||||||
|
merge := ""
|
||||||
|
if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult)
|
||||||
|
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
|
||||||
|
}
|
||||||
|
return json.RawMessage(`{"workspace":"w` + buildstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil
|
||||||
|
case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs
|
||||||
|
return nil, fmt.Errorf("go build of %v: boom", args["artifact"])
|
||||||
|
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
|
||||||
|
d := digestOf(fmt.Sprint(args["artifact"]))
|
||||||
|
return json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`), nil
|
||||||
|
case tool != "build_image_build" && f.refusePush.Load():
|
||||||
|
return nil, fmt.Errorf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"])
|
||||||
|
case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1)
|
||||||
|
d := digestOf(fmt.Sprint(args["artifact"]))
|
||||||
|
return json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://` + filepath.Base(fmt.Sprint(args["module"])) + `/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`), nil
|
||||||
|
case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference
|
||||||
|
d := digestOf(fmt.Sprint(args["artifact"]))
|
||||||
|
return json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/imaged/server@` + d + `"}`), nil
|
||||||
|
default:
|
||||||
|
return json.RawMessage(`{"digest":"` + digestOf(fmt.Sprint(args["artifact"])) + `"}`), nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on nodes.
|
||||||
|
func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) {
|
||||||
|
kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) {
|
language := strings.TrimPrefix(module, "mesh-build-")
|
||||||
tokens := strings.Split(m.Subject, ".") // mesh mod <module> call <tool> <node> <caller>
|
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||||
tool, node := tokens[4], tokens[5]
|
t.Fatal(err)
|
||||||
f.mu.Lock()
|
}
|
||||||
f.calls[module+"."+tool]++
|
put := func(key string, value any) (uint64, error) {
|
||||||
f.mu.Unlock()
|
raw, _ := value.(json.RawMessage)
|
||||||
var args map[string]any
|
return kv.Put(f.ctx, key, raw)
|
||||||
_ = json.Unmarshal(m.Data, &args)
|
}
|
||||||
if strings.HasSuffix(tool, "_fetch") {
|
call := func(name string) buildstep.StepFunc {
|
||||||
f.mu.Lock()
|
return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) }
|
||||||
f.fetches = append(f.fetches, args)
|
}
|
||||||
f.mu.Unlock()
|
steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")}
|
||||||
} else {
|
for _, node := range nodes {
|
||||||
f.mu.Lock()
|
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put,
|
||||||
f.builds = append(f.builds, args)
|
Capacity: func() int {
|
||||||
f.mu.Unlock()
|
if f.closed.Load() {
|
||||||
}
|
return 0
|
||||||
id := fmt.Sprintf("s%d", f.n.Add(1))
|
|
||||||
key, _ := delivery.StepKey(node, id)
|
|
||||||
put := func(s delivery.Step) {
|
|
||||||
data, err := delivery.EncodeStep(s)
|
|
||||||
if err == nil {
|
|
||||||
_, err = steps.Put(f.ctx, key, data)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
t.Errorf("writing step %s: %v", key, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()}
|
|
||||||
put(step)
|
|
||||||
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node})
|
|
||||||
_ = m.Respond(answer)
|
|
||||||
go func() {
|
|
||||||
time.Sleep(30 * time.Millisecond)
|
|
||||||
step.Ended = time.Now().UTC()
|
|
||||||
switch {
|
|
||||||
case strings.HasSuffix(tool, "_fetch"):
|
|
||||||
merge := ""
|
|
||||||
if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult)
|
|
||||||
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
|
|
||||||
}
|
}
|
||||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`)
|
return 2
|
||||||
case f.failBuild.Load() && tool != "build_image_copy": // every node's build fails: one node builds a fingerprint
|
}, PullWait: 200 * time.Millisecond,
|
||||||
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"])
|
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||||
case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4)
|
f.mu.Lock()
|
||||||
d := digestOf(fmt.Sprint(args["artifact"]))
|
f.pullers = append(f.pullers, p)
|
||||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"artifact":"`+fmt.Sprint(args["artifact"])+`","digest":"`+d+`","reference":"registry.example/`+fmt.Sprint(args["module"])+`/`+fmt.Sprint(args["artifact"])+`@`+d+`","pushed":true}`)
|
f.mu.Unlock()
|
||||||
case tool != "build_image_build" && f.refusePush.Load():
|
go func() { _ = p.Pull(f.ctx) }()
|
||||||
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"])
|
|
||||||
case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1)
|
|
||||||
d := digestOf(fmt.Sprint(args["artifact"]))
|
|
||||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"artifact-store://`+filepath.Base(fmt.Sprint(args["module"]))+`/`+fmt.Sprint(args["artifact"])+`/blobs/`+d+`"}`)
|
|
||||||
case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference
|
|
||||||
d := digestOf(fmt.Sprint(args["artifact"]))
|
|
||||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"registry.example/imaged/server@`+d+`"}`)
|
|
||||||
default:
|
|
||||||
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+digestOf(fmt.Sprint(args["artifact"]))+`"}`)
|
|
||||||
}
|
|
||||||
put(step)
|
|
||||||
}()
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
|
||||||
}
|
}
|
||||||
|
|
||||||
type bed struct {
|
type bed struct {
|
||||||
@@ -153,8 +169,8 @@ type bed struct {
|
|||||||
candidates jetstream.KeyValue
|
candidates jetstream.KeyValue
|
||||||
builds jetstream.KeyValue
|
builds jetstream.KeyValue
|
||||||
tools map[string]jetstream.KeyValue
|
tools map[string]jetstream.KeyValue
|
||||||
presence jetstream.KeyValue
|
instances jetstream.KeyValue
|
||||||
have jetstream.KeyValue
|
stop func() // stops the builder started last
|
||||||
fake *fakeBuildModules
|
fake *fakeBuildModules
|
||||||
srvURL string
|
srvURL string
|
||||||
dir string
|
dir string
|
||||||
@@ -163,7 +179,11 @@ type bed struct {
|
|||||||
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
|
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
|
||||||
// build modules' streams and the builder's durable consumers on them, two nodes' toolchains, the indexer's manifests, and
|
// build modules' streams and the builder's durable consumers on them, two nodes' toolchains, the indexer's manifests, and
|
||||||
// the fake build modules. It runs no builder.
|
// the fake build modules. It runs no builder.
|
||||||
func startBed(t *testing.T) *bed {
|
func startBed(t *testing.T) *bed { return startBedWith(t, false) }
|
||||||
|
|
||||||
|
// startBedWith is startBed with the nodes' capacity 0 from the start when closed: a puller that reads its capacity before a
|
||||||
|
// test could close the nodes may be in a pull already and takes the first ask put (issue 1928).
|
||||||
|
func startBedWith(t *testing.T, closed bool) *bed {
|
||||||
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "b")
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "b")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -225,19 +245,12 @@ func startBed(t *testing.T) *bed {
|
|||||||
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
|
||||||
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
|
||||||
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
|
||||||
presence := mk(delivery.PresenceBucket, delivery.PresenceHistory)
|
b.instances = mk(instancesBucket, 1)
|
||||||
b.presence, b.have = presence, mk(delivery.HaveBucket, delivery.HaveHistory)
|
|
||||||
for _, node := range []string{"n1", "n2"} {
|
|
||||||
b.setOnline(t, node, true)
|
|
||||||
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
|
|
||||||
b.setHealth(t, node, module, delivery.HealthOK)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||||
b.fake.serve(t, ctl, "mesh-build-go")
|
b.fake.closed.Store(closed)
|
||||||
b.fake.serve(t, ctl, "mesh-build-node")
|
b.fake.serve(t, "mesh-build-go", "n1", "n2")
|
||||||
b.fake.serve(t, ctl, "mesh-build-image")
|
b.fake.serve(t, "mesh-build-node", "n1", "n2")
|
||||||
b.setHealth(t, "n1", "mesh-build-image", delivery.HealthOK) // the image build module runs on n1 only
|
b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only
|
||||||
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
||||||
b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1")
|
b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1")
|
||||||
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
||||||
@@ -245,28 +258,6 @@ func startBed(t *testing.T) *bed {
|
|||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
// setOnline writes the watchman's presence of a node.
|
|
||||||
func (b *bed) setOnline(t *testing.T, node string, online bool) {
|
|
||||||
t.Helper()
|
|
||||||
data, _ := delivery.EncodePresence(delivery.Presence{Online: online})
|
|
||||||
if _, err := b.presence.Put(b.ctx, node, data); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setHealth writes the node-engine's have of a build module on a node, running with this health.
|
|
||||||
func (b *bed) setHealth(t *testing.T, node, module string, health delivery.Health) {
|
|
||||||
t.Helper()
|
|
||||||
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: health})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
key, _ := delivery.HaveKey(node, module)
|
|
||||||
if _, err := b.have.Put(b.ctx, key, data); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
|
// report writes a node's toolchains key of a build module and says so on the module's stream, as the module does.
|
||||||
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
|
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -344,7 +335,9 @@ func (b *bed) start(t *testing.T, node string, env map[string]string) {
|
|||||||
runCtx, stop := context.WithCancel(b.ctx)
|
runCtx, stop := context.WithCancel(b.ctx)
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
|
go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
|
||||||
t.Cleanup(func() { stop(); <-done })
|
var once sync.Once
|
||||||
|
b.stop = func() { once.Do(func() { stop(); <-done }) }
|
||||||
|
t.Cleanup(b.stop)
|
||||||
}
|
}
|
||||||
|
|
||||||
// await reads the build key until pred holds of it, within 20 s.
|
// await reads the build key until pred holds of it, within 20 s.
|
||||||
@@ -427,20 +420,8 @@ func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed: the node-engine's have of mesh-build-go on n2 says failed, so only n1 builds.
|
// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails, so the key is
|
||||||
func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) {
|
// failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
|
||||||
b := startBed(t)
|
|
||||||
b.setHealth(t, "n2", "mesh-build-go", delivery.HealthFailed)
|
|
||||||
b.archive(t, "ca-trust", "modules/ca-trust", commit)
|
|
||||||
b.start(t, "n1", nil)
|
|
||||||
v := b.await(t, "ca-trust", isState(delivery.BuildDone))
|
|
||||||
if !strings.Contains(v.Step, "on n1") {
|
|
||||||
t.Errorf("build = %+v, want it built on n1 alone", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails on one node, so the
|
|
||||||
// key is failed naming it; the indexer writing modules.<m> again with the same fingerprint wakes the builder, which builds
|
|
||||||
// the key again as a new revision of the same key, now done.
|
// the key again as a new revision of the same key, now done.
|
||||||
func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
|
func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
@@ -713,3 +694,130 @@ func TestAManifestFallingBackToAPredefinedKeyIsBuilt(t *testing.T) {
|
|||||||
t.Errorf("build = %+v, want done: the manifest is read", v)
|
t.Errorf("build = %+v, want done: the manifest is read", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1886/AC5: a builder on n1 whose Go, Node and image build modules each answered a reference says under n1, in its bucket
|
||||||
|
// of instances, that it writes references, and takes the say away when it stops: the controller delivers from its keys
|
||||||
|
// only while it holds.
|
||||||
|
func TestABuilderWhoseBuildModulesAnsweredReferencesSaysItAndStoppedSaysNothing(t *testing.T) {
|
||||||
|
b := startBed(t)
|
||||||
|
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") // a Go and a TypeScript bundle
|
||||||
|
b.moduleCommit(t, "imaged", "catalogue-a67f0196-imaged.module.json", "modules/imaged", nil, commit) // an image, for the trunk
|
||||||
|
b.start(t, "n1", nil)
|
||||||
|
var said string
|
||||||
|
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline) && said == ""; time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err := b.instances.Get(b.ctx, "n1"); err == nil {
|
||||||
|
said = string(e.Value())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !strings.Contains(said, `"writes":["references"]`) {
|
||||||
|
t.Fatalf("the builder said %q", said)
|
||||||
|
}
|
||||||
|
b.stop()
|
||||||
|
if _, err := b.instances.Get(b.ctx, "n1"); !errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
t.Fatalf("the stopped builder still says: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/AC1, AC2 on the real bus: the builder puts the trunk fingerprint's build as asks on the build seats' delivery
|
||||||
|
// streams and calls no node's tool; the build modules that pulled them wrote the steps, and the key is done naming the
|
||||||
|
// references.
|
||||||
|
func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) {
|
||||||
|
b := startBed(t)
|
||||||
|
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
|
||||||
|
sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer sub.Close()
|
||||||
|
var calls atomic.Int64
|
||||||
|
s, err := sub.Subscribe("mesh.mod.*.call.>", func(*nats.Msg) { calls.Add(1) })
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer s.Unsubscribe()
|
||||||
|
asked := b.asksPut(t, sub)
|
||||||
|
b.start(t, "n1", nil)
|
||||||
|
v := b.await(t, "netcheck", isState(delivery.BuildDone))
|
||||||
|
for _, a := range v.Artifacts {
|
||||||
|
if a.Reference == "" {
|
||||||
|
t.Errorf("artifact %s names no reference", a.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if n := calls.Load(); n != 0 {
|
||||||
|
t.Errorf("%d tool calls reached a build module", n)
|
||||||
|
}
|
||||||
|
// 1887/AC1: one ask per build module the build needs (netcheck: a Go and a TypeScript bundle), each on its seat's
|
||||||
|
// delivery stream.
|
||||||
|
got := asked()
|
||||||
|
if len(got) != 2 {
|
||||||
|
t.Fatalf("asks put = %+v, want one Go and one Node delivery ask", got)
|
||||||
|
}
|
||||||
|
for _, a := range got {
|
||||||
|
if a.Kind != delivery.AskDelivery || a.Build == nil || a.Build.FP != fp {
|
||||||
|
t.Errorf("ask %+v, want a delivery ask of the fingerprint", a)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// asksPut hears every ask put on a build seat's accept subjects (a core subscription beside the stream) and answers the
|
||||||
|
// asks heard so far.
|
||||||
|
func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []delivery.StepAsk {
|
||||||
|
t.Helper()
|
||||||
|
var mu sync.Mutex
|
||||||
|
var asks []delivery.StepAsk
|
||||||
|
s, err := nc.Subscribe("mesh.seat.*.accept.>", func(m *nats.Msg) {
|
||||||
|
if a, err := delivery.DecodeStepAsk(m.Data); err == nil {
|
||||||
|
mu.Lock()
|
||||||
|
asks = append(asks, a)
|
||||||
|
mu.Unlock()
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { _ = s.Unsubscribe() })
|
||||||
|
return func() []delivery.StepAsk {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return append([]delivery.StepAsk(nil), asks...)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/AC3: a builder that started again while its asks waited in the streams (no node had a free slot) puts no second
|
||||||
|
// ask: it waits for the asks its building key names, and the key is done once a node took them.
|
||||||
|
func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) {
|
||||||
|
b := startBedWith(t, true) // 1928/AC1: no node can take an ask from the start, so no pull in hand takes the first one
|
||||||
|
b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
|
||||||
|
sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer sub.Close()
|
||||||
|
asked := b.asksPut(t, sub)
|
||||||
|
b.start(t, "n1", nil)
|
||||||
|
v := b.await(t, "netcheck", func(v delivery.BuilderBuild) bool { return v.State == delivery.BuildBuilding && len(v.Asks) == 2 })
|
||||||
|
for deadline := time.Now().Add(20 * time.Second); len(asked()) < 2; time.Sleep(20 * time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("asks put = %+v within 20 s, want the two the key names", asked())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.stop()
|
||||||
|
b.start(t, "n1", nil) // the same builder, started again, while both asks wait
|
||||||
|
b.fake.open()
|
||||||
|
done := b.await(t, "netcheck", isState(delivery.BuildDone))
|
||||||
|
if got := asked(); len(got) != 2 {
|
||||||
|
t.Fatalf("asks put = %d (%+v), want the two the key named and none again", len(got), got)
|
||||||
|
}
|
||||||
|
for lang, id := range v.Asks {
|
||||||
|
found := false
|
||||||
|
for _, a := range asked() {
|
||||||
|
found = found || a.ID == id
|
||||||
|
}
|
||||||
|
if !found {
|
||||||
|
t.Errorf("the %s ask %s the key named was never put", lang, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(done.Artifacts) != 2 {
|
||||||
|
t.Errorf("done = %+v", done)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -32,8 +32,9 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a
|
// defaultStepBound is how long one ask's step may run once a build node took it (a fetch downloads the Go modules, a
|
||||||
// build compiles).
|
// build compiles): from its start, never from its put, so an ask waiting in its stream for a free slot is not failed
|
||||||
|
// (buildrun.Asker.Bound; ADR 0525 §3, §6).
|
||||||
defaultStepBound = 30 * time.Minute
|
defaultStepBound = 30 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -143,43 +144,66 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
|
|
||||||
// buckets are read and written on the connection above.
|
|
||||||
account, err := process.ReadAccount(path)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer caller.Close()
|
|
||||||
|
|
||||||
bound := defaultStepBound
|
bound := defaultStepBound
|
||||||
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" {
|
||||||
if bound, err = time.ParseDuration(v); err != nil {
|
if bound, err = time.ParseDuration(v); err != nil {
|
||||||
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
|
return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_BUILDER_SETTINGS")), getenv("MESH_BUILDER_CAPACITY"))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the build capacity setting: %w", err)
|
|
||||||
}
|
|
||||||
tools := &buildrun.ToolchainSet{}
|
tools := &buildrun.ToolchainSet{}
|
||||||
presence := &buildrun.PresenceSet{}
|
// Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot
|
||||||
have := &buildrun.HaveSet{}
|
// (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity.
|
||||||
|
asks := buildrun.JetStreamAsks{JS: js}
|
||||||
|
// The builder asks the controller, as its own account, whether the artifact store still holds a done key's blobs
|
||||||
|
// (the seat verb held, novox/hq issue 1937) before it leaves the key done.
|
||||||
|
account, err := process.ReadAccount(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
calls, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer calls.Close()
|
||||||
b := &build.Builder{
|
b := &build.Builder{
|
||||||
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
|
||||||
Runner: buildrun.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound},
|
Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound},
|
||||||
|
Held: heldOnTheBus{js},
|
||||||
Toolchains: tools,
|
Toolchains: tools,
|
||||||
Presence: presence,
|
|
||||||
Health: have,
|
|
||||||
Places: &buildrun.Places{Capacity: capacity},
|
|
||||||
Manifests: manifestReader{manifests},
|
Manifests: manifestReader{manifests},
|
||||||
Logf: logger.Printf,
|
Logf: logger.Printf,
|
||||||
|
Stored: storedVia(calls.Call),
|
||||||
|
}
|
||||||
|
// **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the
|
||||||
|
// controller delivers from this builder's keys only while an instance says it writes references. A bucket not there
|
||||||
|
// yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent
|
||||||
|
// is asked since R4g, ADR 0524 §7).
|
||||||
|
switch kv, err := js.KeyValue(ctx, instancesBucket); {
|
||||||
|
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||||
|
logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket)
|
||||||
|
case err != nil:
|
||||||
|
return fmt.Errorf("the bucket %s: %w", instancesBucket, err)
|
||||||
|
default:
|
||||||
|
says := newSayer(ctx, kv, cred.Node, []string{buildrun.GoBuildModule, buildrun.NodeBuildModule, build.ImageBuildModule}, logger.Printf)
|
||||||
|
b.Answered = says.Answered
|
||||||
|
defer func() {
|
||||||
|
stopping, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if err := says.unsay(stopping); err != nil {
|
||||||
|
logger.Printf("%v", err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
// The toolchains first, so a build that starts from the first modules value finds the nodes.
|
||||||
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
watch := newToolchainWatch(ctx, js, tools, b.NodesChanged, logger)
|
||||||
|
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||||
|
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||||
|
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||||
|
watch.put = gate.Toolchain
|
||||||
|
assignErrs, err := watchAssignments(ctx, js, gate, b.NodesChanged, logger)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := watch.start(); err != nil {
|
if err := watch.start(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -200,47 +224,20 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
defer cc.Stop()
|
defer cc.Stop()
|
||||||
}
|
}
|
||||||
// The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing
|
// The controller says artifacts-gone when its take-in found a key's blob gone (issue 1937): the builder's durable
|
||||||
// builds; the builder says so and is started again with the watchman.
|
// consumer on EVENTS, made by the controller from the manifest's consumes. Where it is not there yet the builder says
|
||||||
presenceErrs := make(chan error, 1)
|
// so; it still asks held at its start and at every module value.
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
switch cons, err := js.Consumer(ctx, eventsStream, cred.Node+"_"+cred.Module); {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the builder starts again", delivery.PresenceBucket)
|
|
||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
|
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s is not heard until the builder starts again: %v",
|
||||||
|
cred.Node, cred.Module, eventsStream, delivery.ArtifactsGoneEvent, err)
|
||||||
default:
|
default:
|
||||||
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
|
cc, err := cons.Consume(func(m jetstream.Msg) { heardGone(ctx, b, m, logger.Printf) },
|
||||||
|
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
go func() {
|
defer cc.Stop()
|
||||||
presenceErrs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
|
||||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
|
||||||
b.NodesChanged(ctx) // a node came online or went: a build that waited for one may go on
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
// The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not
|
|
||||||
// chosen (ADR 0478 clause 3). Where its bucket is not there yet no build module counts as healthy, so nothing builds.
|
|
||||||
haveErrs := make(chan error, 1)
|
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); {
|
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the builder starts again", delivery.HaveBucket)
|
|
||||||
case err != nil:
|
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err)
|
|
||||||
default:
|
|
||||||
hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) })
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
haveErrs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) {
|
|
||||||
if have.Put(e.Key(), e.Value(), !opOf(e)) {
|
|
||||||
b.NodesChanged(ctx) // a build module became healthy or stopped being: a build that waited may go on
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
}
|
}
|
||||||
each := func(e jetstream.KeyValueEntry) {
|
each := func(e jetstream.KeyValueEntry) {
|
||||||
if opOf(e) {
|
if opOf(e) {
|
||||||
@@ -279,11 +276,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return nil
|
return nil
|
||||||
case err := <-errs:
|
case err := <-errs:
|
||||||
return err
|
return err
|
||||||
|
case err := <-assignErrs:
|
||||||
|
return err
|
||||||
case err := <-watch.errs:
|
case err := <-watch.errs:
|
||||||
return err
|
return err
|
||||||
case err := <-presenceErrs:
|
|
||||||
return err
|
|
||||||
case err := <-haveErrs:
|
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
// What the builder says it writes (novox/hq issue 1886, ADR 0522): each running instance keeps a key, named by its node,
|
||||||
|
// in its own bucket of instances (`mesh-delivery-builder_instances`, the manifest's state `instances`), saying that every
|
||||||
|
// done key it writes names where its artifacts are (ADR 0498 §1). The references come from the build modules' own pushes,
|
||||||
|
// so an instance says it only once a step of every build module it calls answered one since it started: a builder beside
|
||||||
|
// a build module from before R4e says nothing. The controller delivers from the builder's keys only while an instance
|
||||||
|
// says so; since R4g it asks no build-agent, and a module whose key is not yet done waits, named (ADR 0524 §7). The bucket is read by the controller alone, so its name and shape
|
||||||
|
// are not in mesh-sdk.
|
||||||
|
const (
|
||||||
|
instancesBucket = "mesh-delivery-builder_instances"
|
||||||
|
writesReferences = `{"v":1,"writes":["references"]}`
|
||||||
|
)
|
||||||
|
|
||||||
|
// sayStore is the bucket of instances as the builder writes it.
|
||||||
|
type sayStore interface {
|
||||||
|
Put(ctx context.Context, key string, value []byte) (uint64, error)
|
||||||
|
Delete(ctx context.Context, key string, opts ...jetstream.KVDeleteOpt) error
|
||||||
|
}
|
||||||
|
|
||||||
|
// sayer puts this instance's say once every build module in need answered a reference, and takes it away at the stop.
|
||||||
|
type sayer struct {
|
||||||
|
ctx context.Context
|
||||||
|
store sayStore
|
||||||
|
node string
|
||||||
|
need []string
|
||||||
|
logf func(string, ...any)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
answered map[string]bool
|
||||||
|
said bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSayer(ctx context.Context, store sayStore, node string, need []string, logf func(string, ...any)) *sayer {
|
||||||
|
return &sayer{ctx: ctx, store: store, node: node, need: need, logf: logf, answered: map[string]bool{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Answered is told the build module whose step answered a reference (build.Builder.Answered).
|
||||||
|
func (s *sayer) Answered(module string) {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
s.answered[module] = true
|
||||||
|
if s.said {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for _, m := range s.need {
|
||||||
|
if !s.answered[m] {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := s.store.Put(s.ctx, s.node, []byte(writesReferences)); err != nil {
|
||||||
|
s.logf("saying in %s that this builder writes references: %v; said again at the next reference", instancesBucket, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
s.said = true
|
||||||
|
s.logf("every build module answered a reference: said in %s that this builder writes references", instancesBucket)
|
||||||
|
}
|
||||||
|
|
||||||
|
// unsay takes the say away, if it was made: a builder replaced by one that does not write references leaves none.
|
||||||
|
func (s *sayer) unsay(ctx context.Context) error {
|
||||||
|
s.mu.Lock()
|
||||||
|
defer s.mu.Unlock()
|
||||||
|
if !s.said {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if err := s.store.Delete(ctx, s.node); err != nil {
|
||||||
|
return fmt.Errorf("taking away the say in %s: %w", instancesBucket, err)
|
||||||
|
}
|
||||||
|
s.said = false
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
)
|
||||||
|
|
||||||
|
type saysOf map[string]string
|
||||||
|
|
||||||
|
func (s saysOf) Put(_ context.Context, key string, value []byte) (uint64, error) {
|
||||||
|
s[key] = string(value)
|
||||||
|
return 1, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s saysOf) Delete(_ context.Context, key string, _ ...jetstream.KVDeleteOpt) error {
|
||||||
|
delete(s, key)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1886/AC5: a builder says, under its node, that it writes references only once every build module it calls answered
|
||||||
|
// one; stopped, it says nothing any more.
|
||||||
|
func TestTheBuilderSaysItWritesReferencesOnceEveryBuildModuleAnsweredOne(t *testing.T) {
|
||||||
|
s := saysOf{}
|
||||||
|
say := newSayer(t.Context(), s, "shanks", []string{"mesh-build-go", "mesh-build-node", "mesh-build-image"}, t.Logf)
|
||||||
|
say.Answered("mesh-build-go")
|
||||||
|
say.Answered("mesh-build-node")
|
||||||
|
if v, ok := s["shanks"]; ok {
|
||||||
|
t.Fatalf("said before the image build module answered a reference: %q", v)
|
||||||
|
}
|
||||||
|
say.Answered("mesh-build-image")
|
||||||
|
var v struct {
|
||||||
|
Writes []string `json:"writes"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal([]byte(s["shanks"]), &v); err != nil || len(v.Writes) != 1 || v.Writes[0] != "references" {
|
||||||
|
t.Fatalf("the builder's say: %q %v", s["shanks"], err)
|
||||||
|
}
|
||||||
|
if err := say.unsay(t.Context()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, ok := s["shanks"]; ok {
|
||||||
|
t.Fatalf("the stopped builder still says: %v", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
"os"
|
|
||||||
)
|
|
||||||
|
|
||||||
// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `<node>=<steps at
|
|
||||||
// once>` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a
|
|
||||||
// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch.
|
|
||||||
func capacityFrom(path, fallback string) (map[string]int, error) {
|
|
||||||
if path == "" {
|
|
||||||
return buildrun.ParseCapacity(fallback)
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the settings file: %w", err)
|
|
||||||
}
|
|
||||||
var s struct {
|
|
||||||
Capacity string `json:"capacity"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &s); err != nil {
|
|
||||||
return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
|
|
||||||
}
|
|
||||||
return buildrun.ParseCapacity(s.Capacity)
|
|
||||||
}
|
|
||||||
@@ -1,67 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func settingsFile(t *testing.T, body string) string {
|
|
||||||
t.Helper()
|
|
||||||
p := filepath.Join(t.TempDir(), "settings.json")
|
|
||||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// 1661: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere.
|
|
||||||
func TestCapacityIsReadFromTheSettingsFile(t *testing.T) {
|
|
||||||
for body, want := range map[string]map[string]int{
|
|
||||||
`{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2},
|
|
||||||
`{"capacity": ""}`: {},
|
|
||||||
`{}`: {},
|
|
||||||
} {
|
|
||||||
got, err := capacityFrom(settingsFile(t, body), "")
|
|
||||||
if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] {
|
|
||||||
t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
|
|
||||||
if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 {
|
|
||||||
t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustCapacity(t *testing.T, path string) map[string]int {
|
|
||||||
t.Helper()
|
|
||||||
c, err := capacityFrom(path, "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) {
|
|
||||||
if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil {
|
|
||||||
t.Error("shanks=-1 accepted")
|
|
||||||
}
|
|
||||||
if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 {
|
|
||||||
t.Errorf("the starting value: %v, %v", c, err)
|
|
||||||
}
|
|
||||||
if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil {
|
|
||||||
t.Error("a file that is not JSON accepted")
|
|
||||||
}
|
|
||||||
if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil {
|
|
||||||
t.Error("a settings file that is not there accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's.
|
|
||||||
func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) {
|
|
||||||
got, err := capacityFrom("", "ace=2")
|
|
||||||
if err != nil || got["ace"] != 2 {
|
|
||||||
t.Errorf("capacity %v, %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||||
|
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||||
|
// toolchain agreement this file repairs.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||||
|
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||||
|
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||||
|
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||||
|
//
|
||||||
|
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||||
|
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||||
|
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||||
|
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||||
|
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||||
|
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||||
|
type toolchainGate struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||||
|
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||||
|
logf func(format string, a ...any)
|
||||||
|
known bool // the assignments are settled
|
||||||
|
standing map[string]bool // <node>.<module> with an open assignment
|
||||||
|
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||||
|
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||||
|
}
|
||||||
|
|
||||||
|
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||||
|
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||||
|
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||||
|
for _, m := range modules {
|
||||||
|
g.modules[m] = true
|
||||||
|
}
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.raw[module] == nil {
|
||||||
|
g.raw[module] = map[string][]byte{}
|
||||||
|
}
|
||||||
|
if deleted {
|
||||||
|
delete(g.raw[module], key)
|
||||||
|
} else {
|
||||||
|
g.raw[module][key] = value
|
||||||
|
}
|
||||||
|
return g.pass(module, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||||
|
node, module, ok := strings.Cut(key, ".")
|
||||||
|
if !ok || !g.modules[module] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
open := !deleted && openAssignment(value)
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.standing[key] == open {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if open {
|
||||||
|
g.standing[key] = true
|
||||||
|
} else {
|
||||||
|
delete(g.standing, key)
|
||||||
|
}
|
||||||
|
if !g.known {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
changed := false
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||||
|
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||||
|
changed = g.pass(module, k) || changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||||
|
// passed on changed.
|
||||||
|
func (g *toolchainGate) Settled() bool {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
g.known = true
|
||||||
|
changed := false
|
||||||
|
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||||
|
changed = g.pass(module, k) || changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||||
|
// otherwise. The caller holds mu.
|
||||||
|
func (g *toolchainGate) pass(module, key string) bool {
|
||||||
|
value, held := g.raw[module][key]
|
||||||
|
if !held {
|
||||||
|
return g.forward(module, key, nil, true)
|
||||||
|
}
|
||||||
|
node, _, err := delivery.ParseToolchainKey(key)
|
||||||
|
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||||
|
delete(g.left, module+"/"+key)
|
||||||
|
return g.forward(module, key, value, false)
|
||||||
|
}
|
||||||
|
if !g.left[module+"/"+key] {
|
||||||
|
g.left[module+"/"+key] = true
|
||||||
|
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||||
|
delivery.ToolchainsBucket(module), key, module, node)
|
||||||
|
}
|
||||||
|
return g.forward(module, key, nil, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||||
|
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||||
|
func openAssignment(value []byte) bool {
|
||||||
|
var a struct {
|
||||||
|
Ended time.Time `json:"ended"`
|
||||||
|
State string `json:"state"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(value, &a) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return a.Ended.IsZero() && a.State == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||||
|
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||||
|
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||||
|
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||||
|
errs := make(chan error, 1)
|
||||||
|
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||||
|
if err != nil {
|
||||||
|
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||||
|
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
|
return errs, nil
|
||||||
|
}
|
||||||
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||||
|
w, err := settle(ctx, kv, each)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||||
|
}
|
||||||
|
if gate.Settled() {
|
||||||
|
changed(ctx)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||||
|
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||||
|
changed(ctx)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
return errs, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||||
|
t.Helper()
|
||||||
|
key, err := delivery.ToolchainKey(node, "typescript")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key, value
|
||||||
|
}
|
||||||
|
|
||||||
|
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||||
|
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||||
|
return tc.Version, none
|
||||||
|
}
|
||||||
|
|
||||||
|
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||||
|
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||||
|
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||||
|
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
var said []string
|
||||||
|
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||||
|
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||||
|
key, value := gateReport(t, node, version)
|
||||||
|
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||||
|
}
|
||||||
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||||
|
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||||
|
}
|
||||||
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
|
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||||
|
}
|
||||||
|
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||||
|
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||||
|
if !gate.Settled() {
|
||||||
|
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||||
|
}
|
||||||
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
if len(said) != 1 {
|
||||||
|
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||||
|
}
|
||||||
|
// ace reports again while unassigned (an old value seen again): still left out.
|
||||||
|
key, value := gateReport(t, "ace", "v26.12.0")
|
||||||
|
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("an unassigned node's report changed what was passed on")
|
||||||
|
}
|
||||||
|
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||||
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||||
|
t.Fatal("assigning ace again passed nothing on")
|
||||||
|
}
|
||||||
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
|
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||||
|
}
|
||||||
|
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||||
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||||
|
t.Fatal("a pending assignment is no assignment on the machine")
|
||||||
|
}
|
||||||
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||||
|
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||||
|
t.Fatalf("got %q", v)
|
||||||
|
}
|
||||||
|
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||||
|
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||||
|
t.Fatal("deleting g14's report passed nothing on")
|
||||||
|
}
|
||||||
|
if n := len(set.Entries("typescript")); n != 2 {
|
||||||
|
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||||
|
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||||
|
gate.Settled()
|
||||||
|
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||||
|
t.Fatal("an assignment with no report passed something on")
|
||||||
|
}
|
||||||
|
key, value := gateReport(t, "g14", "v26.12.1")
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("an assigned node's report was not passed on")
|
||||||
|
}
|
||||||
|
key, value = gateReport(t, "ace", "v26.12.0")
|
||||||
|
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||||
|
}
|
||||||
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-builder/internal/build"
|
||||||
|
)
|
||||||
|
|
||||||
|
// eventsStream is the stream every module's and role's events are on, where the controller makes the builder's durable
|
||||||
|
// consumer of artifacts-gone (mesh-controller internal/broker EventsStream).
|
||||||
|
const eventsStream = "EVENTS"
|
||||||
|
|
||||||
|
// toolCaller calls a tool or a seat verb as the builder's own account (mesh-sdk process.Conn.Call).
|
||||||
|
type toolCaller func(key string, body any) (json.RawMessage, error)
|
||||||
|
|
||||||
|
// storedVia is build.Builder's Stored: it asks the controller's verb held (mesh-sdk delivery.HeldVerb; novox/hq issue
|
||||||
|
// 1937) and answers the references the store no longer holds. One it could not ask about, with none gone, is an error:
|
||||||
|
// the key is then left done, and the controller's take-in still refuses a gone blob.
|
||||||
|
func storedVia(call toolCaller) func(ctx context.Context, references []string) ([]string, error) {
|
||||||
|
return func(_ context.Context, references []string) ([]string, error) {
|
||||||
|
raw, err := call(delivery.HeldVerb, map[string]any{"references": references})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("asking %s: %w", delivery.HeldVerb, err)
|
||||||
|
}
|
||||||
|
a, err := delivery.DecodeHeldAnswer(raw, references)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(a.Gone) == 0 && len(a.Unknown) > 0 {
|
||||||
|
var why []string
|
||||||
|
for r, w := range a.Unknown {
|
||||||
|
why = append(why, r+": "+w)
|
||||||
|
}
|
||||||
|
sort.Strings(why)
|
||||||
|
return nil, fmt.Errorf("the artifact store could not be asked about %s", strings.Join(why, "; "))
|
||||||
|
}
|
||||||
|
return a.Gone, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heardGone takes one artifacts-gone event in: acknowledged, then handed to the builder, which asks the store and builds
|
||||||
|
// the key again. A body that is not one is said and acknowledged, so it is not handed over again.
|
||||||
|
func heardGone(ctx context.Context, b *build.Builder, m jetstream.Msg, logf func(string, ...any)) {
|
||||||
|
e, err := delivery.DecodeArtifactsGone(m.Data())
|
||||||
|
if ackErr := m.Ack(); ackErr != nil {
|
||||||
|
logf("acknowledging %s: %v", m.Subject(), ackErr)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
logf("%s holds what is not %s, left: %v", m.Subject(), delivery.ArtifactsGoneEvent, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.ArtifactsGone(ctx, e)
|
||||||
|
}
|
||||||
@@ -23,9 +23,11 @@ var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO"
|
|||||||
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
|
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
|
||||||
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
|
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
|
||||||
type toolchainWatch struct {
|
type toolchainWatch struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
js bucketOpener
|
js bucketOpener
|
||||||
set *buildrun.ToolchainSet
|
set *buildrun.ToolchainSet
|
||||||
|
// put takes a change of a toolchains bucket in: set.Put, or the gate of the assigned build nodes before it (issue 1940).
|
||||||
|
put func(module, key string, value []byte, deleted bool) bool
|
||||||
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
|
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
|
||||||
logger *log.Logger
|
logger *log.Logger
|
||||||
errs chan error
|
errs chan error
|
||||||
@@ -34,7 +36,7 @@ type toolchainWatch struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
|
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
|
||||||
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
|
return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
|
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
|
||||||
@@ -62,14 +64,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
|||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||||
}
|
}
|
||||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) })
|
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
w.watched[module] = true
|
w.watched[module] = true
|
||||||
go func() {
|
go func() {
|
||||||
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
||||||
if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) {
|
if w.put(module, e.Key(), e.Value(), !opOf(e)) {
|
||||||
w.changed(w.ctx)
|
w.changed(w.ctx)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
{
|
|
||||||
"capacity": "shanks=3,ace=2,g14=0,novox=0"
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,7 @@ module mesh-delivery-builder
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1 h1:fzOkrz9CSp3mEjeV7NfZx8yoivI1sD1AOtIPtyyF8RI=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261024020722-34f210e5eba1/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
@@ -37,6 +38,11 @@ type Claim struct {
|
|||||||
Key string
|
Key string
|
||||||
Rev uint64
|
Rev uint64
|
||||||
Build delivery.BuilderBuild
|
Build delivery.BuilderBuild
|
||||||
|
// Gone are the references of a done key the artifact store no longer holds, for which it was claimed again
|
||||||
|
// (ClaimOptions.Stored; issue 1937): said in the log and the key's step.
|
||||||
|
Gone []string
|
||||||
|
// Unasked is why the store could not be asked about a done key taken as it stands to publish: said in the log.
|
||||||
|
Unasked error
|
||||||
}
|
}
|
||||||
|
|
||||||
// ClaimOptions say what the builder knows about the moment of a claim.
|
// ClaimOptions say what the builder knows about the moment of a claim.
|
||||||
@@ -55,6 +61,30 @@ type ClaimOptions struct {
|
|||||||
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
|
// a package's reference. A key queued waiting for a version (BuilderBuild.Waits) is taken only once it is (ADR 0500
|
||||||
// clause 6). Nil means none is.
|
// clause 6). Nil means none is.
|
||||||
Published func(ref string) bool
|
Published func(ref string) bool
|
||||||
|
// Stored answers which of the references given the artifact store no longer holds (the controller's verb held,
|
||||||
|
// mesh-sdk delivery.HeldVerb; novox/hq issue 1937). A done key of the trunk's fingerprint is asked before it is left
|
||||||
|
// done: the store's nightly collector deletes a blob nothing holds, and a key the controller never took in holds
|
||||||
|
// nothing, so a key whose blob is gone is claimed building as a new revision and built again. Nil asks nothing.
|
||||||
|
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// StoredReferences are the references of a done build's bundles and archives that name a blob in the artifact store
|
||||||
|
// (`artifact-store://<module>/<artifact>/blobs/<digest>`): what the store's collector can delete. An image or an upstream
|
||||||
|
// copy is a manifest, which the collector keeps, and a package lives in the package registry.
|
||||||
|
func StoredReferences(b delivery.BuilderBuild) []string {
|
||||||
|
var out []string
|
||||||
|
for _, a := range b.Artifacts {
|
||||||
|
if strings.HasPrefix(a.Reference, "artifact-store://") && strings.Contains(a.Reference, "/blobs/") &&
|
||||||
|
!slices.Contains(out, a.Reference) {
|
||||||
|
out = append(out, a.Reference)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoneStep is the step a key claimed again for gone blobs says (issue 1937).
|
||||||
|
func GoneStep(gone []string) string {
|
||||||
|
return "claimed again: the artifact store no longer holds " + strings.Join(gone, ", ")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has
|
// Unpublished is whether a done build holds an image built without a push or a package built without a publish: it has
|
||||||
@@ -118,6 +148,8 @@ func Unreferenced(b delivery.BuilderBuild) bool {
|
|||||||
// names it (ClaimOptions.Trunk): that is taken as it stands, with the done value, and its builder writes the new
|
// names it (ClaimOptions.Trunk): that is taken as it stands, with the done value, and its builder writes the new
|
||||||
// revision when the publishing build ends (ADR 0498 clause 6). From R4e one node builds each fingerprint (ADR 0478
|
// revision when the publishing build ends (ADR 0498 clause 6). From R4e one node builds each fingerprint (ADR 0478
|
||||||
// clause 4), so a key built on one node is never built again for a second;
|
// clause 4), so a key built on one node is never built again for a second;
|
||||||
|
// - a done key of the trunk's fingerprint whose bundle or archive blob the artifact store no longer holds
|
||||||
|
// (ClaimOptions.Stored, issue 1937) is claimed building as a new revision and built whole, its step saying which;
|
||||||
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
|
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
|
||||||
// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
|
// builder builds, for an upstream pinned by no digest or a package version taken (delivery.FailedUntilChanged: ADR 0478
|
||||||
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
// clause 2, ADR 0498 clauses 4 and 5) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
|
||||||
@@ -149,6 +181,28 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
|
|||||||
if Unreferenced(cur) {
|
if Unreferenced(cur) {
|
||||||
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
|
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
|
||||||
}
|
}
|
||||||
|
if opt.Trunk && opt.Stored != nil {
|
||||||
|
if refs := StoredReferences(cur); len(refs) > 0 {
|
||||||
|
gone, err := opt.Stored(ctx, refs)
|
||||||
|
if err != nil {
|
||||||
|
err = fmt.Errorf("%s: whether the artifact store still holds %s cannot be asked, so the key is "+
|
||||||
|
"left done (the controller's take-in refuses it if one is gone): %w", key, strings.Join(refs, ", "), err)
|
||||||
|
if len(publishable(cur, opt.ImageBuilder)) > 0 {
|
||||||
|
return &Claim{Key: key, Rev: rev, Build: cur, Unasked: err}, nil
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if len(gone) > 0 {
|
||||||
|
again := building
|
||||||
|
again.Step = GoneStep(gone)
|
||||||
|
cl, err := c.write(ctx, key, again, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
|
||||||
|
if cl != nil {
|
||||||
|
cl.Gone = gone
|
||||||
|
}
|
||||||
|
return cl, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 {
|
if opt.Trunk && len(publishable(cur, opt.ImageBuilder)) > 0 {
|
||||||
return &Claim{Key: key, Rev: rev, Build: cur}, nil
|
return &Claim{Key: key, Rev: rev, Build: cur}, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,14 @@
|
|||||||
package build
|
package build
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -25,17 +29,31 @@ func captured(t *testing.T, name string) json.RawMessage {
|
|||||||
return step.Result
|
return step.Result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The captured image steps, run as a pulled ask runs them (buildstep.RunAsk), answer what the builder reads: the trunk's
|
||||||
|
// build its digest and reference, a candidate's its digest and no reference (issue 1887 A4a).
|
||||||
func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1
|
func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1
|
||||||
var f fetchResult
|
for _, tc := range []struct {
|
||||||
if err := json.Unmarshal(captured(t, "fetch.json"), &f); err != nil || f.Workspace == "" {
|
build string
|
||||||
t.Errorf("fetch = %+v, %v, want a workspace (the field the builder reads)", f, err)
|
publish bool
|
||||||
}
|
ref bool
|
||||||
var trunk buildResult
|
}{{"build-trunk.json", true, true}, {"build-candidate.json", false, false}} {
|
||||||
if err := json.Unmarshal(captured(t, "build-trunk.json"), &trunk); err != nil || trunk.Digest == "" || trunk.Reference == "" {
|
steps := buildstep.AskSteps{
|
||||||
t.Errorf("trunk build = %+v, %v, want a digest and a reference", trunk, err)
|
Fetch: func(context.Context, map[string]any) (any, error) {
|
||||||
}
|
return json.RawMessage(captured(t, "fetch.json")), nil
|
||||||
var cand buildResult
|
},
|
||||||
if err := json.Unmarshal(captured(t, "build-candidate.json"), &cand); err != nil || cand.Digest == "" || cand.Reference != "" {
|
Build: func(context.Context, map[string]any) (any, error) { return json.RawMessage(captured(t, tc.build)), nil },
|
||||||
t.Errorf("candidate build = %+v, %v, want a digest and no reference", cand, err)
|
}
|
||||||
|
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskDelivery, Language: delivery.AskImage, Asker: delivery.BuilderModule,
|
||||||
|
At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish,
|
||||||
|
Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)},
|
||||||
|
Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}}
|
||||||
|
out, err := buildstep.RunAsk(context.Background(), steps, ask)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: %v", tc.build, err)
|
||||||
|
}
|
||||||
|
a := out.(delivery.BuildAnswer)
|
||||||
|
if len(a.Artifacts) != 1 || a.Artifacts[0].Digest == "" || (a.Artifacts[0].Reference != "") != tc.ref {
|
||||||
|
t.Errorf("%s answered %+v, want a digest and a reference only for the trunk", tc.build, a)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,204 @@
|
|||||||
|
package build
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A done key whose blob the artifact store no longer holds is built again (novox/hq issue 1937): on 2026-10-24 the
|
||||||
|
// store's 03:30 collection deleted the bare blobs of node-tools', mesh-host's and the forge bridge's done keys, which the
|
||||||
|
// controller had never taken in, and the builder kept them done.
|
||||||
|
|
||||||
|
// fakeHeld answers the controller's verb held: the references in gone are gone.
|
||||||
|
type fakeHeld struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
gone map[string]bool
|
||||||
|
err error
|
||||||
|
asked [][]string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeHeld) stored(_ context.Context, refs []string) ([]string, error) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.asked = append(f.asked, slices.Clone(refs))
|
||||||
|
if f.err != nil {
|
||||||
|
return nil, f.err
|
||||||
|
}
|
||||||
|
var out []string
|
||||||
|
for _, r := range refs {
|
||||||
|
if f.gone[r] {
|
||||||
|
out = append(out, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeHeld) asks() int {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
return len(f.asked)
|
||||||
|
}
|
||||||
|
|
||||||
|
// logged keeps what the builder says.
|
||||||
|
type logged struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
lines []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *logged) logf(format string, args ...any) {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
l.lines = append(l.lines, fmt.Sprintf(format, args...))
|
||||||
|
}
|
||||||
|
|
||||||
|
func (l *logged) has(words ...string) bool {
|
||||||
|
l.mu.Lock()
|
||||||
|
defer l.mu.Unlock()
|
||||||
|
for _, line := range l.lines {
|
||||||
|
all := true
|
||||||
|
for _, w := range words {
|
||||||
|
all = all && strings.Contains(line, w)
|
||||||
|
}
|
||||||
|
if all {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
const goneRef = "artifact-store://m/tool/blobs/" + digest
|
||||||
|
|
||||||
|
func doneKeyNaming(ref string) delivery.BuilderBuild {
|
||||||
|
return delivery.BuilderBuild{State: delivery.BuildDone, By: "shanks.builder", Manifest: manifestD, Step: "built mesh-build-go on shanks",
|
||||||
|
Artifacts: []delivery.BuiltArtifact{{Name: "tool", Kind: "bundle", Digest: digest, Reference: ref}}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1937/AC3 (unit, the claim): a done key of the trunk's fingerprint whose blob is gone is claimed building as a new
|
||||||
|
// revision, its step naming the gone reference; a candidate's done key is not asked about.
|
||||||
|
func TestADoneTrunkKeyWhoseBlobIsGoneIsClaimedAgain(t *testing.T) {
|
||||||
|
st := newMem()
|
||||||
|
c := Claimer{Store: st, By: "novox.builder"}
|
||||||
|
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||||
|
rev := putBuild(t, st, key, doneKeyNaming(goneRef))
|
||||||
|
held := &fakeHeld{gone: map[string]bool{goneRef: true}}
|
||||||
|
|
||||||
|
cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Stored: held.stored})
|
||||||
|
if err != nil || cl != nil || held.asks() != 0 {
|
||||||
|
t.Fatalf("a candidate's done key: claim %+v, %v, %d asks; want nothing and no ask", cl, err, held.asks())
|
||||||
|
}
|
||||||
|
cl, err = c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored})
|
||||||
|
if err != nil || cl == nil {
|
||||||
|
t.Fatalf("claim = %+v, %v; want the key claimed again", cl, err)
|
||||||
|
}
|
||||||
|
if cl.Rev <= rev || cl.Build.State != delivery.BuildBuilding || !slices.Equal(cl.Gone, []string{goneRef}) {
|
||||||
|
t.Errorf("claim = %+v (revision was %d); want building as a new revision naming the gone reference", cl, rev)
|
||||||
|
}
|
||||||
|
v := valueOf(t, st, key)
|
||||||
|
if v.State != delivery.BuildBuilding || !strings.Contains(v.Step, "no longer holds") || !strings.Contains(v.Step, goneRef) {
|
||||||
|
t.Errorf("the key holds %+v; want building, its step naming the gone reference", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1937/AC5 (unit, the claim): a done trunk key whose blobs are all held stays done; a store that cannot be asked leaves
|
||||||
|
// it done and says why; images and packages are not asked about.
|
||||||
|
func TestADoneTrunkKeyStaysDoneWhileTheStoreHoldsItsBlobs(t *testing.T) {
|
||||||
|
st := newMem()
|
||||||
|
c := Claimer{Store: st, By: "novox.builder"}
|
||||||
|
key, _ := delivery.BuilderBuildKey("m", fp1)
|
||||||
|
b := doneKeyNaming(goneRef)
|
||||||
|
b.Artifacts = append(b.Artifacts, delivery.BuiltArtifact{Name: "img", Kind: "image", Digest: digest, Reference: "registry.example/m/img@" + digest})
|
||||||
|
rev := putBuild(t, st, key, b)
|
||||||
|
held := &fakeHeld{gone: map[string]bool{}}
|
||||||
|
if cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored}); err != nil || cl != nil {
|
||||||
|
t.Fatalf("claim = %+v, %v; want nothing: the store holds the blob", cl, err)
|
||||||
|
}
|
||||||
|
if len(held.asked) != 1 || !slices.Equal(held.asked[0], []string{goneRef}) {
|
||||||
|
t.Errorf("asked %v; want the bundle's reference alone", held.asked)
|
||||||
|
}
|
||||||
|
held.err = errors.New("the controller did not answer")
|
||||||
|
cl, err := c.Claim(t.Context(), "m", fp1, manifestD, ClaimOptions{Trunk: true, Stored: held.stored})
|
||||||
|
if cl != nil || err == nil || !strings.Contains(err.Error(), "cannot be asked") || !strings.Contains(err.Error(), "left done") {
|
||||||
|
t.Errorf("claim = %+v, %v; want the key left done and said", cl, err)
|
||||||
|
}
|
||||||
|
if st.revs[key] != rev {
|
||||||
|
t.Errorf("the key moved to revision %d (was %d)", st.revs[key], rev)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1937/AC3 (unit, the builder): the builder's look at a module whose done key's blob is gone builds it again, says so
|
||||||
|
// in a log line and in the done key's step, and asks the store once per process while it answers held.
|
||||||
|
func TestTheBuilderBuildsAgainADoneKeyWhoseBlobIsGone(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
log := &logged{}
|
||||||
|
r.b.Logf = log.logf
|
||||||
|
held := &fakeHeld{gone: map[string]bool{goneRef: true}}
|
||||||
|
r.b.Stored = held.stored
|
||||||
|
rev := putBuild(t, r.st, r.key, doneKeyNaming(goneRef))
|
||||||
|
v := r.consider(t, "modules/m")
|
||||||
|
if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 {
|
||||||
|
t.Fatalf("build = %+v at revision %d (was %d); want built again", v, r.st.revs[r.key], rev)
|
||||||
|
}
|
||||||
|
if !strings.Contains(v.Step, "built again: the artifact store no longer held "+goneRef) {
|
||||||
|
t.Errorf("the done key's step %q does not say it was built again for the gone blob", v.Step)
|
||||||
|
}
|
||||||
|
if !log.has(r.key, "built again", goneRef) {
|
||||||
|
t.Errorf("no log line says the key was built again: %v", log.lines)
|
||||||
|
}
|
||||||
|
// The new revision's blob is held: asked once, then not again in this process.
|
||||||
|
held.mu.Lock()
|
||||||
|
held.gone = map[string]bool{}
|
||||||
|
held.mu.Unlock()
|
||||||
|
rev = r.st.revs[r.key]
|
||||||
|
asks := held.asks()
|
||||||
|
r.consider(t, "modules/m")
|
||||||
|
r.consider(t, "modules/m")
|
||||||
|
if r.st.revs[r.key] != rev {
|
||||||
|
t.Errorf("a key whose blob is held was built again")
|
||||||
|
}
|
||||||
|
if held.asks() != asks+1 {
|
||||||
|
t.Errorf("asked the store %d times for two looks at a held key; want once", held.asks()-asks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1937/AC4 (unit, the builder): artifacts-gone for the module's fingerprint makes the builder ask again, even for a key
|
||||||
|
// it found held before, and build it again; one for another fingerprint builds nothing.
|
||||||
|
func TestArtifactsGoneMakesTheBuilderAskAndBuildAgain(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
log := &logged{}
|
||||||
|
r.b.Logf = log.logf
|
||||||
|
held := &fakeHeld{gone: map[string]bool{}}
|
||||||
|
r.b.Stored = held.stored
|
||||||
|
putBuild(t, r.st, r.key, doneKeyNaming(goneRef))
|
||||||
|
if v := r.consider(t, "modules/m"); v.State != delivery.BuildDone || len(r.bm.callsOf("build_go_build")) != 0 {
|
||||||
|
t.Fatalf("a key whose blob is held was built: %+v", v)
|
||||||
|
}
|
||||||
|
rev := r.st.revs[r.key]
|
||||||
|
|
||||||
|
held.mu.Lock()
|
||||||
|
held.gone[goneRef] = true // collected after the builder's look; the controller's take-in found it
|
||||||
|
held.mu.Unlock()
|
||||||
|
r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m",
|
||||||
|
Fingerprint: "src2:" + strings.Repeat("9", 64), Gone: []string{goneRef}})
|
||||||
|
r.b.Wait()
|
||||||
|
if r.st.revs[r.key] != rev || !log.has("a fingerprint the module does not have now") {
|
||||||
|
t.Fatalf("an event for another fingerprint moved the key (revision %d, was %d): %v", r.st.revs[r.key], rev, log.lines)
|
||||||
|
}
|
||||||
|
|
||||||
|
r.b.ArtifactsGone(context.Background(), delivery.ArtifactsGone{Event: delivery.ArtifactsGoneEvent, Module: "m",
|
||||||
|
Key: r.key, Fingerprint: fp1, Gone: []string{goneRef}})
|
||||||
|
r.b.Wait()
|
||||||
|
v := valueOf(t, r.st, r.key)
|
||||||
|
if v.State != delivery.BuildDone || r.st.revs[r.key] <= rev || len(r.bm.callsOf("build_go_build")) == 0 {
|
||||||
|
t.Fatalf("build = %+v at revision %d (was %d); want built again on the event", v, r.st.revs[r.key], rev)
|
||||||
|
}
|
||||||
|
if !log.has("heard artifacts-gone") || !log.has(r.key, "built again", goneRef) {
|
||||||
|
t.Errorf("the builder did not say what it heard and did: %v", log.lines)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -111,7 +111,7 @@ func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) {
|
|||||||
func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4
|
func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4
|
||||||
up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}
|
up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}
|
||||||
r := newRig(t, up)
|
r := newRig(t, up)
|
||||||
r.b.Health = imageHave(t, "ace", "novox")
|
r.puller = "ace"
|
||||||
v := r.consider(t, "modules/m")
|
v := r.consider(t, "modules/m")
|
||||||
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
|
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
|
||||||
t.Fatalf("build = %+v, want done with the copy", v)
|
t.Fatalf("build = %+v, want done with the copy", v)
|
||||||
@@ -133,7 +133,7 @@ func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheRe
|
|||||||
|
|
||||||
func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4
|
func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4
|
||||||
r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
|
r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
|
||||||
r.b.Health = imageHave(t, "ace")
|
r.puller = "ace"
|
||||||
v := r.consider(t, "modules/m")
|
v := r.consider(t, "modules/m")
|
||||||
if ok, missing := v.Referenced(); !ok {
|
if ok, missing := v.Referenced(); !ok {
|
||||||
t.Fatalf("build = %+v, missing references %v", v, missing)
|
t.Fatalf("build = %+v, missing references %v", v, missing)
|
||||||
@@ -145,7 +145,7 @@ func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(
|
|||||||
|
|
||||||
func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2
|
func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2
|
||||||
r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
|
r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest})
|
||||||
r.b.Health = imageHave(t, "ace")
|
r.puller = "ace"
|
||||||
r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500"
|
r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500"
|
||||||
v := r.consider(t, "modules/m")
|
v := r.consider(t, "modules/m")
|
||||||
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) {
|
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) {
|
||||||
@@ -383,3 +383,21 @@ func TestACandidatesFetchNamesTheNamedPathsTheIndexerGaveIt(t *testing.T) { // A
|
|||||||
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
t.Errorf("fetch request = %+v, %v; want the candidate's base, the module's directory and its named paths", req, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1886/AC5: the builder tells who answered a reference: the build module whose step named one, and none for a step that
|
||||||
|
// answered a digest alone (a build module from before R4e).
|
||||||
|
func TestTheBuilderTellsWhichBuildModuleAnsweredAReference(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
var answered []string
|
||||||
|
r.b.Answered = func(module string) { answered = append(answered, module) }
|
||||||
|
r.bm.noRef["ace/tool"] = true
|
||||||
|
r.consider(t, "modules/m")
|
||||||
|
if len(answered) != 0 {
|
||||||
|
t.Fatalf("a step that answered no reference was told as one: %v", answered)
|
||||||
|
}
|
||||||
|
delete(r.bm.noRef, "ace/tool")
|
||||||
|
r.consider(t, "modules/m")
|
||||||
|
if len(answered) != 1 || answered[0] != "mesh-build-go" {
|
||||||
|
t.Fatalf("the build module that answered a reference: %v, want mesh-build-go", answered)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package build
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
@@ -39,26 +38,6 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1
|
|
||||||
r := newRig(t, archiveA)
|
|
||||||
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait
|
|
||||||
r.b.Health = &buildrun.HaveSet{} // no build module is healthy anywhere
|
|
||||||
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
|
|
||||||
t.Errorf("build = %+v, want failed: no node's build module is healthy", v)
|
|
||||||
}
|
|
||||||
have := &buildrun.HaveSet{}
|
|
||||||
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
|
||||||
r.b.Health = have
|
|
||||||
r.b.Reconsider(context.Background())
|
|
||||||
r.b.Wait()
|
|
||||||
v := valueOf(t, r.st, r.key)
|
|
||||||
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") {
|
|
||||||
t.Errorf("build = %+v, want done on novox, the one node that qualifies", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1
|
func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1
|
||||||
p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}})
|
p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}})
|
||||||
if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) {
|
if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) {
|
||||||
@@ -80,115 +59,6 @@ func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 4. A step never waits on a full node while another qualifying node has room.
|
|
||||||
func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4
|
|
||||||
p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}}
|
|
||||||
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks")
|
|
||||||
total := 2 + 1 + 1 + 3
|
|
||||||
var waiting int
|
|
||||||
for i := 1; i <= 4; i++ { // four modules choose before any lane has started
|
|
||||||
room := 0
|
|
||||||
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
|
|
||||||
room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n]
|
|
||||||
}
|
|
||||||
nodes, _, err := buildrun.ChooseNodes("go", entries, p, nil, 2)
|
|
||||||
if err != nil || len(nodes) != 2 {
|
|
||||||
t.Fatalf("choice %d: %v, %v", i, nodes, err)
|
|
||||||
}
|
|
||||||
if room >= 2 {
|
|
||||||
for _, n := range nodes {
|
|
||||||
if p.Reserved()[n] > p.CapacityOf(n) {
|
|
||||||
t.Errorf("choice %d: %v put a lane on full %s while %d places were free", i, nodes, n, room)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
waiting += 2 - room
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if want := 4*2 - total; waiting != want {
|
|
||||||
t.Errorf("%d lanes wait, want %d: the 8 lanes less the %d places", waiting, want, total)
|
|
||||||
}
|
|
||||||
loads := p.Reserved()
|
|
||||||
for n, c := range loads {
|
|
||||||
if c > p.CapacityOf(n) && n != "shanks" && n != "ace" && n != "g14" && n != "novox" {
|
|
||||||
t.Errorf("unknown node %s", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if loads["shanks"] != 3 || loads["ace"] != 2 || loads["g14"]+loads["novox"] != 3 {
|
|
||||||
t.Errorf("reservations = %v, want shanks 3, ace 2 and the single-place nodes taking the rest", loads)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4
|
|
||||||
p := &buildrun.Places{}
|
|
||||||
nodes, _, _ := buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
|
|
||||||
if len(nodes) != 1 || p.Reserved()["ace"] != 1 {
|
|
||||||
t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved())
|
|
||||||
}
|
|
||||||
release, err := p.Acquire(context.Background(), "ace")
|
|
||||||
if err != nil || p.Reserved()["ace"] != 0 || p.Running()["ace"] != 1 {
|
|
||||||
t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running())
|
|
||||||
}
|
|
||||||
release()
|
|
||||||
buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
|
|
||||||
p.Cancel("ace")
|
|
||||||
if len(p.Reserved()) != 0 {
|
|
||||||
t.Errorf("reserved after Cancel = %v", p.Reserved())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 5. A node whose build module is not healthy is not chosen.
|
|
||||||
func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5
|
|
||||||
var s buildrun.HaveSet
|
|
||||||
if s.Healthy("ace", buildrun.GoBuildModule) {
|
|
||||||
t.Error("a module with no have is healthy")
|
|
||||||
}
|
|
||||||
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", buildrun.GoBuildModule) {
|
|
||||||
t.Error("a running, healthy module is not")
|
|
||||||
}
|
|
||||||
if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) {
|
|
||||||
t.Error("the same state counted as a change")
|
|
||||||
}
|
|
||||||
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", buildrun.GoBuildModule) {
|
|
||||||
t.Error("a module whose health failed is healthy")
|
|
||||||
}
|
|
||||||
s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
|
||||||
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", buildrun.GoBuildModule) {
|
|
||||||
t.Error("a module whose phase failed is healthy")
|
|
||||||
}
|
|
||||||
s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
|
||||||
if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", buildrun.GoBuildModule) {
|
|
||||||
t.Error("a deleted have is healthy")
|
|
||||||
}
|
|
||||||
if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) {
|
|
||||||
t.Error("a node's key or a non-have counted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
have := &buildrun.HaveSet{}
|
|
||||||
have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's
|
|
||||||
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
|
||||||
r.b.Health = have
|
|
||||||
v := r.consider(t, "modules/m")
|
|
||||||
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") {
|
|
||||||
t.Fatalf("build = %+v, want done on novox alone", v)
|
|
||||||
}
|
|
||||||
for _, c := range r.bm.calls {
|
|
||||||
if c.node == "ace" {
|
|
||||||
t.Errorf("a step was called on ace, whose build module is unhealthy: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false)
|
|
||||||
r.st.vals = map[string][]byte{} // a fresh key
|
|
||||||
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
|
||||||
r.b.Wait()
|
|
||||||
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
|
|
||||||
t.Errorf("build = %+v, want failed: no node has a healthy build module", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// 6. A key found building is taken over, whoever holds it.
|
// 6. A key found building is taken over, whoever holds it.
|
||||||
func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6
|
func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6
|
||||||
st := newMem()
|
st := newMem()
|
||||||
|
|||||||
@@ -5,9 +5,11 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"maps"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
@@ -20,31 +22,37 @@ type ToolchainEntries interface {
|
|||||||
Entries(language string) []delivery.NodeToolchain
|
Entries(language string) []delivery.NodeToolchain
|
||||||
}
|
}
|
||||||
|
|
||||||
// toolPrefix is the start of each build module's tool names: <prefix>fetch and <prefix>build.
|
|
||||||
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}
|
|
||||||
|
|
||||||
// Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is
|
// Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is
|
||||||
// handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an
|
// handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an
|
||||||
// artifact to build claims the fingerprint's build key, builds it through the build modules on the nodes
|
// artifact to build claims the fingerprint's build key, builds it by asking the build seats (one ask per build module,
|
||||||
// the toolchains choose, and writes the key done or failed. It holds no timer; a build starts from a modules value or a
|
// pulled by whichever build node has a free slot: ADR 0525), and writes the key done or failed. It holds no timer; a build starts from a modules value or a
|
||||||
// change of the toolchains.
|
// change of the toolchains.
|
||||||
type Builder struct {
|
type Builder struct {
|
||||||
Claimer Claimer
|
Claimer Claimer
|
||||||
Runner buildrun.Runner
|
// Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq
|
||||||
Toolchains ToolchainEntries
|
// ADR 0525 §1, §3, §6; issue 1887 A4): the builder chooses no node, whichever build node pulls the ask builds it.
|
||||||
// Presence says which nodes are online (the watchman's); a node that is not is not chosen.
|
Asks interface {
|
||||||
Presence interface{ Online(node string) bool }
|
Put(ctx context.Context, a delivery.StepAsk) error
|
||||||
Manifests Manifests
|
Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error)
|
||||||
// Health says whether a node's build module is running and healthy (the node-engine's have); a node whose build module
|
|
||||||
// is not is not chosen. Nil means every node's is.
|
|
||||||
Health interface {
|
|
||||||
Healthy(node, module string) bool
|
|
||||||
}
|
}
|
||||||
// buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node.
|
// Held says whether an ask a key names is on its stream or a build node wrote a step for it (issue 1910): a key taken
|
||||||
Places *buildrun.Places
|
// over at start whose ask is neither was left by a builder that stopped between naming the ask and putting it, and
|
||||||
// Limit is how many builds run at once; four when 0.
|
// the ask is put then, under the same id. Nil takes every named ask as held.
|
||||||
|
Held interface {
|
||||||
|
Held(ctx context.Context, a delivery.StepAsk) (bool, error)
|
||||||
|
}
|
||||||
|
Toolchains ToolchainEntries
|
||||||
|
Manifests Manifests
|
||||||
|
// Limit is how many builds the builder has asked at once; their asks wait in the build seats' streams, which are the
|
||||||
|
// backlog (ADR 0525 §9), not here. 256 when 0.
|
||||||
Limit int
|
Limit int
|
||||||
Logf func(format string, args ...any)
|
Logf func(format string, args ...any)
|
||||||
|
// Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes
|
||||||
|
// rests on it. Nil tells nobody.
|
||||||
|
Answered func(buildModule string)
|
||||||
|
// Stored answers which references the artifact store no longer holds (the controller's verb held; issue 1937): a done
|
||||||
|
// key of the trunk's fingerprint is asked before it is left done (ClaimOptions.Stored). Nil asks nothing.
|
||||||
|
Stored func(ctx context.Context, references []string) (gone []string, err error)
|
||||||
|
|
||||||
once sync.Once
|
once sync.Once
|
||||||
sem chan struct{}
|
sem chan struct{}
|
||||||
@@ -61,6 +69,9 @@ type Builder struct {
|
|||||||
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
|
// published are the package versions `<name>@<version>` a done key of this bucket names as a package's reference: a key
|
||||||
// waiting for one is built once it is here (ADR 0500 clause 6).
|
// waiting for one is built once it is here (ADR 0500 clause 6).
|
||||||
published map[string]bool
|
published map[string]bool
|
||||||
|
// held are the done keys' references the store answered held, by `<key> <references>`, asked once per process: a
|
||||||
|
// blob collected later is found by the take-in, whose artifacts-gone forgets the key here (issue 1937).
|
||||||
|
held map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type planned struct {
|
type planned struct {
|
||||||
@@ -72,15 +83,12 @@ func (b *Builder) init() {
|
|||||||
b.once.Do(func() {
|
b.once.Do(func() {
|
||||||
limit := b.Limit
|
limit := b.Limit
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
limit = 4
|
limit = 256
|
||||||
}
|
}
|
||||||
b.sem = make(chan struct{}, limit)
|
b.sem = make(chan struct{}, limit)
|
||||||
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
|
||||||
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
|
||||||
b.published = map[string]bool{}
|
b.published, b.held = map[string]bool{}, map[string]bool{}
|
||||||
if b.Places == nil {
|
|
||||||
b.Places = &buildrun.Places{}
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -113,8 +121,7 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
|
|||||||
b.consider(ctx, key, m)
|
b.consider(ctx, key, m)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or
|
// NodesChanged is Reconsider after a toolchain moved, so a build that waited for the nodes to agree on one may go on.
|
||||||
// stopped being, or a toolchain moved, so a build that waited for a node may go on.
|
|
||||||
func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) }
|
func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) }
|
||||||
|
|
||||||
// Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key
|
// Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key
|
||||||
@@ -201,6 +208,55 @@ func (b *Builder) Forget(key string) {
|
|||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// storedOnce is Stored for one key, asked once per process while the store answers every reference held.
|
||||||
|
func (b *Builder) storedOnce(key string) func(ctx context.Context, references []string) ([]string, error) {
|
||||||
|
if b.Stored == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return func(ctx context.Context, references []string) ([]string, error) {
|
||||||
|
id := key + " " + strings.Join(references, " ")
|
||||||
|
b.mu.Lock()
|
||||||
|
known := b.held[id]
|
||||||
|
b.mu.Unlock()
|
||||||
|
if known {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
gone, err := b.Stored(ctx, references)
|
||||||
|
if err == nil && len(gone) == 0 {
|
||||||
|
b.mu.Lock()
|
||||||
|
b.held[id] = true
|
||||||
|
b.mu.Unlock()
|
||||||
|
}
|
||||||
|
return gone, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ArtifactsGone takes the controller's event artifacts-gone in (issue 1937): a take-in found that the store does not
|
||||||
|
// hold what the module's key names. The module is looked at again, which asks the store (ClaimOptions.Stored) and
|
||||||
|
// builds the key again; an event for a fingerprint the module no longer has is said and left. The event only wakes the
|
||||||
|
// builder: what is built rests on the store's answer, never on the event alone.
|
||||||
|
func (b *Builder) ArtifactsGone(ctx context.Context, e delivery.ArtifactsGone) {
|
||||||
|
b.init()
|
||||||
|
b.mu.Lock()
|
||||||
|
m, ok := b.known[e.Module]
|
||||||
|
if key, err := delivery.BuilderBuildKey(e.Module, e.Fingerprint); err == nil {
|
||||||
|
for id := range b.held {
|
||||||
|
if strings.HasPrefix(id, key+" ") {
|
||||||
|
delete(b.held, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.mu.Unlock()
|
||||||
|
if !ok || m.FP2 != e.Fingerprint {
|
||||||
|
b.logf("%s: heard %s for %s, a fingerprint the module does not have now: nothing to build", e.Module,
|
||||||
|
delivery.ArtifactsGoneEvent, e.Fingerprint)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
b.logf("%s: heard %s (%s): asking the artifact store and building the key again", e.Module,
|
||||||
|
delivery.ArtifactsGoneEvent, strings.Join(e.Gone, ", "))
|
||||||
|
b.consider(ctx, e.Module, m)
|
||||||
|
}
|
||||||
|
|
||||||
// Reconsider looks at every module again: the toolchains changed, so a build that failed for them may build now.
|
// Reconsider looks at every module again: the toolchains changed, so a build that failed for them may build now.
|
||||||
func (b *Builder) Reconsider(ctx context.Context) {
|
func (b *Builder) Reconsider(ctx context.Context) {
|
||||||
b.init()
|
b.init()
|
||||||
@@ -311,7 +367,8 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
|||||||
first := !b.tried[key]
|
first := !b.tried[key]
|
||||||
b.tried[key] = true
|
b.tried[key] = true
|
||||||
b.mu.Unlock()
|
b.mu.Unlock()
|
||||||
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished})
|
cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished,
|
||||||
|
Stored: b.storedOnce(key)})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
b.logf("%s: %v", module, err)
|
b.logf("%s: %v", module, err)
|
||||||
return
|
return
|
||||||
@@ -319,6 +376,13 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
|||||||
if cl == nil {
|
if cl == nil {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if cl.Unasked != nil {
|
||||||
|
b.logf("%v", cl.Unasked)
|
||||||
|
}
|
||||||
|
if len(cl.Gone) > 0 {
|
||||||
|
// Said aloud (issue 1937): the key was done, and the store's collector deleted what it names.
|
||||||
|
b.logf("%s: built again: the artifact store no longer holds %s", cl.Key, strings.Join(cl.Gone, ", "))
|
||||||
|
}
|
||||||
// A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the
|
// A done key taken again builds only what is missing: the image or the package built unpublished as a candidate's; the
|
||||||
// other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6).
|
// other artifacts stay as the done key has them (ADR 0487 clauses 5 and 8, ADR 0498 clause 6).
|
||||||
var prior *delivery.BuilderBuild
|
var prior *delivery.BuilderBuild
|
||||||
@@ -327,7 +391,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
|||||||
if cl.Build.State == delivery.BuildDone {
|
if cl.Build.State == delivery.BuildDone {
|
||||||
prior, only = &cl.Build, map[string]bool{}
|
prior, only = &cl.Build, map[string]bool{}
|
||||||
if trunk {
|
if trunk {
|
||||||
for _, bm := range publishable(cl.Build, b.imageBuilderRuns) {
|
for _, bm := range publishable(cl.Build, imageSeat) {
|
||||||
only[bm] = true
|
only[bm] = true
|
||||||
publishing = true
|
publishing = true
|
||||||
}
|
}
|
||||||
@@ -339,7 +403,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
|||||||
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
|
// any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1).
|
||||||
result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"}
|
result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"}
|
||||||
} else {
|
} else {
|
||||||
result = b.build(ctx, module, m, plan, planErr, prior, only)
|
result = b.build(ctx, module, m, plan, planErr, prior, only, b.notes(cl))
|
||||||
}
|
}
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return
|
||||||
@@ -349,6 +413,9 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
result.By, result.Manifest = b.Claimer.By, m.Manifest
|
result.By, result.Manifest = b.Claimer.By, m.Manifest
|
||||||
|
if len(cl.Gone) > 0 && result.State == delivery.BuildDone {
|
||||||
|
result.Step += " (built again: the artifact store no longer held " + strings.Join(cl.Gone, ", ") + ")"
|
||||||
|
}
|
||||||
if publishing && result.State == delivery.BuildFailed {
|
if publishing && result.State == delivery.BuildFailed {
|
||||||
// The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the
|
// The trunk's image or package was built only as a candidate's and its push or publish failed: the key says so, so the builds view, the
|
||||||
// checker and the failed-key retake see it, and a look does not build the image again for nothing.
|
// checker and the failed-key retake see it, and a look does not build the image again for nothing.
|
||||||
@@ -396,14 +463,15 @@ func failed(format string, args ...any) delivery.BuilderBuild {
|
|||||||
|
|
||||||
// lane is the build of a plan's tasks of one build module on one node.
|
// lane is the build of a plan's tasks of one build module on one node.
|
||||||
type lane struct {
|
type lane struct {
|
||||||
module, node string
|
module, node string // node is the one that pulled the lane's ask
|
||||||
tasks []Task
|
tasks []Task
|
||||||
|
toolchain *delivery.ToolchainUsed
|
||||||
digests map[string]string // artifact -> digest
|
digests map[string]string // artifact -> digest
|
||||||
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
|
references map[string]string // artifact -> the reference an image was pushed to (a trunk build only)
|
||||||
err error
|
err error
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error, prior *delivery.BuilderBuild, only map[string]bool) delivery.BuilderBuild {
|
func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error, prior *delivery.BuilderBuild, only map[string]bool, asked *askNotes) delivery.BuilderBuild {
|
||||||
switch {
|
switch {
|
||||||
case planErr != nil:
|
case planErr != nil:
|
||||||
return failed("%s", planErr)
|
return failed("%s", planErr)
|
||||||
@@ -414,76 +482,39 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
|||||||
repository := delivery.RepositoryName(m.Repository)
|
repository := delivery.RepositoryName(m.Repository)
|
||||||
var lanes []*lane
|
var lanes []*lane
|
||||||
used := map[string]delivery.ToolchainUsed{}
|
used := map[string]delivery.ToolchainUsed{}
|
||||||
var steps []string
|
|
||||||
// Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up.
|
|
||||||
giveUp := func(nodes ...string) {
|
|
||||||
for _, n := range nodes {
|
|
||||||
b.Places.Cancel(n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
giveUpLanes := func() {
|
|
||||||
for _, l := range lanes {
|
|
||||||
b.Places.Cancel(l.node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
var kept []string // build modules whose artifacts stay as the prior done key has them
|
var kept []string // build modules whose artifacts stay as the prior done key has them
|
||||||
for _, bm := range plan.Modules() {
|
for _, bm := range plan.Modules() {
|
||||||
if prior != nil && len(only) > 0 && !only[bm] {
|
if prior != nil && len(only) > 0 && !only[bm] {
|
||||||
kept = append(kept, bm)
|
kept = append(kept, bm)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
language := buildrun.LanguageOf[bm]
|
var toolchain *delivery.ToolchainUsed
|
||||||
entries := b.Toolchains.Entries(language)
|
if bm != ImageBuildModule && plan.NeedsToolchain(bm) {
|
||||||
qualifies := func(node string) bool { return b.qualifies(node, bm) }
|
// The ask names the toolchain the fingerprint was computed with; a node building with another refuses it.
|
||||||
var nodes []string
|
language := buildrun.LanguageOf[bm]
|
||||||
if bm == ImageBuildModule {
|
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
|
||||||
// An image is built on one node, chosen from the nodes whose image build module runs healthy (ADR 0487 clause 8):
|
if none != "" {
|
||||||
// the module has no toolchain language, so no toolchain entry names its nodes, and a second build would only
|
return failed("the %s toolchain is not agreed: %s", language, none)
|
||||||
// push the image again.
|
|
||||||
ns, err := buildrun.ChooseAnyNodes("image", b.imageEntries(), b.Places, qualifies, 1)
|
|
||||||
if err != nil {
|
|
||||||
giveUpLanes()
|
|
||||||
return failed("%s no node has %s, so the image cannot be built: the provision rootless-image-build is held by the module buildah; assign buildah and %s to a node (%v)", NoImageBuilderPrefix, ImageBuildModule, ImageBuildModule, err)
|
|
||||||
}
|
|
||||||
nodes = ns
|
|
||||||
} else if plan.NeedsToolchain(bm) {
|
|
||||||
ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, 1)
|
|
||||||
if err != nil {
|
|
||||||
giveUpLanes()
|
|
||||||
return failed("%s", err)
|
|
||||||
}
|
}
|
||||||
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe {
|
||||||
giveUp(ns...)
|
|
||||||
giveUpLanes()
|
|
||||||
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written
|
||||||
}
|
}
|
||||||
used[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
|
u := delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
|
||||||
nodes = ns
|
used[language], toolchain = u, &u
|
||||||
} else {
|
|
||||||
// An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies.
|
|
||||||
ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, 1)
|
|
||||||
if err != nil {
|
|
||||||
giveUpLanes()
|
|
||||||
return failed("%s", err)
|
|
||||||
}
|
|
||||||
nodes = ns
|
|
||||||
}
|
}
|
||||||
// One node builds each fingerprint (ADR 0478 clause 4, from R4e).
|
|
||||||
steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0]))
|
|
||||||
var tasks []Task
|
var tasks []Task
|
||||||
for _, t := range plan.Tasks {
|
for _, t := range plan.Tasks {
|
||||||
if t.Module == bm {
|
if t.Module == bm {
|
||||||
tasks = append(tasks, t)
|
tasks = append(tasks, t)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, n := range nodes {
|
// One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3).
|
||||||
lanes = append(lanes, &lane{module: bm, node: n, tasks: tasks, digests: map[string]string{}, references: map[string]string{}})
|
lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}})
|
||||||
}
|
|
||||||
}
|
}
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
for _, l := range lanes {
|
for _, l := range lanes {
|
||||||
wg.Add(1)
|
wg.Add(1)
|
||||||
go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module) }()
|
go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module, asked) }()
|
||||||
}
|
}
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
var errs []string
|
var errs []string
|
||||||
@@ -545,6 +576,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
|
|||||||
}
|
}
|
||||||
artifacts = append(artifacts, built)
|
artifacts = append(artifacts, built)
|
||||||
}
|
}
|
||||||
|
var steps []string
|
||||||
|
for _, l := range lanes {
|
||||||
|
steps = append(steps, fmt.Sprintf("%s on %s", l.module, l.node))
|
||||||
|
}
|
||||||
for _, bm := range kept {
|
for _, bm := range kept {
|
||||||
steps = append(steps, bm+" kept")
|
steps = append(steps, bm+" kept")
|
||||||
for lang, u := range prior.Toolchains {
|
for lang, u := range prior.Toolchains {
|
||||||
@@ -614,73 +649,56 @@ func keyError(errs []string) string {
|
|||||||
return strings.Join(errs, "; ")
|
return strings.Join(errs, "; ")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) }
|
// imageSeat says an image build is asked of the image build seat, which takes it when a node pulls it: a key failed for
|
||||||
|
// want of an image builder before R4g is taken again (ADR 0525 §1).
|
||||||
|
func imageSeat() bool { return true }
|
||||||
|
|
||||||
// qualifies is whether the node can build with the build module now: it is online and the build module on it is healthy.
|
// askLanguage is the build seat language of each build module (delivery.AskSubject).
|
||||||
func (b *Builder) qualifies(node, bm string) bool {
|
var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode, ImageBuildModule: delivery.AskImage}
|
||||||
return b.Places.CapacityOf(node) > 0 && b.online(node) && (b.Health == nil || b.Health.Healthy(node, bm))
|
|
||||||
|
// askNotes writes a claimed key's asks into it, building, before each is put (ADR 0525 §1; issue 1887 A4b), and holds
|
||||||
|
// the asks a key taken over at start already names, so they are waited for and never put again.
|
||||||
|
type askNotes struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
b *Builder
|
||||||
|
cl *Claim
|
||||||
|
resumed map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// imageEntries are the nodes an image may be built on, as toolchain entries for the chooser: every node the builder has a
|
func (b *Builder) notes(cl *Claim) *askNotes {
|
||||||
// capacity for or a build module toolchain from. Whether the image build module runs there is qualifies', from the have.
|
n := &askNotes{b: b, cl: cl}
|
||||||
func (b *Builder) imageEntries() []delivery.NodeToolchain {
|
if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 {
|
||||||
seen := map[string]bool{}
|
n.resumed = maps.Clone(cl.Build.Asks)
|
||||||
var entries []delivery.NodeToolchain
|
|
||||||
add := func(n string) {
|
|
||||||
if !seen[n] {
|
|
||||||
seen[n] = true
|
|
||||||
entries = append(entries, delivery.NodeToolchain{Node: n})
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
for n := range b.Places.Capacity {
|
return n
|
||||||
add(n)
|
}
|
||||||
|
|
||||||
|
// resume is the id of the ask a taken-over key names for language, or "".
|
||||||
|
func (n *askNotes) resume(language string) string {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
return n.resumed[language]
|
||||||
|
}
|
||||||
|
|
||||||
|
// note writes the ask's id into the key, building, by compare-and-set at the claim's revision.
|
||||||
|
func (n *askNotes) note(ctx context.Context, language, id, step string) error {
|
||||||
|
n.mu.Lock()
|
||||||
|
defer n.mu.Unlock()
|
||||||
|
v := n.cl.Build
|
||||||
|
v.State, v.By, v.Step, v.Error, v.Artifacts = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil
|
||||||
|
v.Asks = maps.Clone(v.Asks)
|
||||||
|
if v.Asks == nil {
|
||||||
|
v.Asks = map[string]string{}
|
||||||
}
|
}
|
||||||
for _, language := range []string{"go", "typescript"} {
|
v.Asks[language] = id
|
||||||
for _, e := range b.Toolchains.Entries(language) {
|
return n.b.Claimer.Finish(ctx, n.cl, v)
|
||||||
add(e.Node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
sort.Slice(entries, func(i, j int) bool { return entries[i].Node < entries[j].Node })
|
|
||||||
return entries
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// imageBuilderRuns is whether a node can build an image now.
|
// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the
|
||||||
func (b *Builder) imageBuilderRuns() bool {
|
// check stream for a candidate's) and reads what the node that pulled it built (ADR 0525 §1, §3, §6). An upstream artifact
|
||||||
for _, e := range b.imageEntries() {
|
// is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches nothing. A lane of a
|
||||||
if b.qualifies(e.Node, ImageBuildModule) {
|
// key taken over at start waits for the ask the key names and puts none.
|
||||||
return true
|
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string, asked *askNotes) {
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// step runs one step of a build module; the lane that runs it holds its node's place.
|
|
||||||
func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) {
|
|
||||||
return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args)
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchResult and buildResult are the results of a build module's fetch and build steps as the builder reads them.
|
|
||||||
type fetchResult struct {
|
|
||||||
Workspace string `json:"workspace"`
|
|
||||||
Merge string `json:"merge"`
|
|
||||||
Conflicts []string `json:"conflicts"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type buildResult struct {
|
|
||||||
Digest string `json:"digest"`
|
|
||||||
Reference string `json:"reference"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time.
|
|
||||||
// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches
|
|
||||||
// nothing.
|
|
||||||
func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) {
|
|
||||||
release, err := b.Places.Acquire(ctx, l.node)
|
|
||||||
if err != nil {
|
|
||||||
l.err = err
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit}
|
req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit}
|
||||||
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
|
||||||
b.mu.Lock()
|
b.mu.Lock()
|
||||||
@@ -699,55 +717,86 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
|
|||||||
}
|
}
|
||||||
sort.Strings(req.Paths)
|
sort.Strings(req.Paths)
|
||||||
}
|
}
|
||||||
fetchArgs := req.Args()
|
|
||||||
candidate := req.Base != ""
|
candidate := req.Base != ""
|
||||||
var fetched fetchResult
|
language := askLanguage[l.module]
|
||||||
|
kind := delivery.AskDelivery
|
||||||
|
if candidate {
|
||||||
|
kind = delivery.AskCheck
|
||||||
|
}
|
||||||
|
job := &delivery.BuildJob{Module: module, Path: m.Path, FP: m.FP2, Toolchain: l.toolchain,
|
||||||
|
// Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12,
|
||||||
|
// ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3).
|
||||||
|
Publish: !candidate}
|
||||||
|
for _, t := range l.tasks {
|
||||||
|
a := delivery.AskArtifact{Artifact: t.Artifact}
|
||||||
|
if t.Kind == catalogue.ArtifactUpstream {
|
||||||
|
a.From = t.From // the copy runs no code of a change: the module's name, the artifact and its pin
|
||||||
|
}
|
||||||
|
job.Artifacts = append(job.Artifacts, a)
|
||||||
|
}
|
||||||
if needsFetch(l.tasks) {
|
if needsFetch(l.tasks) {
|
||||||
raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs)
|
job.Fetch = req.Args()
|
||||||
if err != nil {
|
}
|
||||||
|
ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job}
|
||||||
|
if id := asked.resume(language); id != "" {
|
||||||
|
ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it
|
||||||
|
if b.Held != nil {
|
||||||
|
held, err := b.Held.Held(ctx, ask)
|
||||||
|
if err != nil {
|
||||||
|
// Said, and the key is not failed: the ask may well be on its stream and its build finish (issue 1910).
|
||||||
|
b.logf("whether the ask %s of %s the key names is on its stream cannot be read, so it is waited for as put: %v", ask.ID, l.module, err)
|
||||||
|
held = true
|
||||||
|
}
|
||||||
|
if !held {
|
||||||
|
// The builder that named it stopped before it put it (issue 1910): put it now, under the same id.
|
||||||
|
b.logf("the ask %s of %s that the key names was never put: it is put now", ask.ID, l.module)
|
||||||
|
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||||
|
l.err = err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil {
|
||||||
|
l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := b.Asks.Put(ctx, ask); err != nil {
|
||||||
l.err = err
|
l.err = err
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
|
}
|
||||||
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
|
step, err := b.Asks.Wait(ctx, ask)
|
||||||
return
|
if err != nil {
|
||||||
}
|
l.err = fmt.Errorf("%s: %w", l.module, err)
|
||||||
// A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at
|
return
|
||||||
// the trunk, and building that would store the trunk's build under the candidate's fingerprint.
|
}
|
||||||
if candidate {
|
l.node = step.Node
|
||||||
if len(fetched.Conflicts) > 0 {
|
if step.State != delivery.StepPassed {
|
||||||
l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", "))
|
l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if fetched.Merge == "" {
|
var answer delivery.BuildAnswer
|
||||||
l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw)
|
if err := json.Unmarshal(step.Result, &answer); err != nil {
|
||||||
return
|
l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result)
|
||||||
}
|
return
|
||||||
}
|
}
|
||||||
|
built := map[string]delivery.AskBuilt{}
|
||||||
|
for _, a := range answer.Artifacts {
|
||||||
|
built[a.Artifact] = a
|
||||||
}
|
}
|
||||||
for _, t := range l.tasks {
|
for _, t := range l.tasks {
|
||||||
tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact}
|
a, ok := built[t.Artifact]
|
||||||
if t.Kind == catalogue.ArtifactUpstream {
|
if !ok || a.Digest == "" {
|
||||||
// The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin.
|
l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, step.Node, step.Result)
|
||||||
tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From}
|
|
||||||
} else if !candidate {
|
|
||||||
// Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12,
|
|
||||||
// ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3).
|
|
||||||
args["publish"] = true
|
|
||||||
}
|
|
||||||
raw, err := b.step(ctx, l.module, l.node, tool, args)
|
|
||||||
if err != nil {
|
|
||||||
l.err = err
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var built buildResult
|
l.digests[t.Artifact] = a.Digest
|
||||||
if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" {
|
if a.Reference != "" && (stored(t.Kind) || !candidate) {
|
||||||
l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw)
|
l.references[t.Artifact] = a.Reference
|
||||||
return
|
|
||||||
}
|
}
|
||||||
l.digests[t.Artifact] = built.Digest
|
if a.Reference != "" && b.Answered != nil {
|
||||||
if built.Reference != "" && (stored(t.Kind) || !candidate) {
|
b.Answered(l.module)
|
||||||
l.references[t.Artifact] = built.Reference
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package build
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -10,6 +11,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -124,22 +126,56 @@ type rig struct {
|
|||||||
st *memStore
|
st *memStore
|
||||||
bm *buildModules
|
bm *buildModules
|
||||||
tools *buildrun.ToolchainSet
|
tools *buildrun.ToolchainSet
|
||||||
pres *buildrun.PresenceSet
|
|
||||||
mf *fakeManifests
|
mf *fakeManifests
|
||||||
key string
|
key string
|
||||||
|
// asks are the build seats' streams; puller is the node that pulls every ask ("ace" when empty).
|
||||||
|
asks *buildrun.FakeAsks
|
||||||
|
puller string
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
|
||||||
|
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"}
|
||||||
|
|
||||||
|
// answer runs an ask as the build module that pulls it does (buildstep.RunAsk), with the fake build modules' tools on the
|
||||||
|
// rig's pulling node, and answers the step that node writes.
|
||||||
|
func (r *rig) answer(a delivery.StepAsk) delivery.Step {
|
||||||
|
node := r.puller
|
||||||
|
if node == "" {
|
||||||
|
node = "ace"
|
||||||
|
}
|
||||||
|
module, _ := delivery.BuildModuleOf(a.Language)
|
||||||
|
runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}
|
||||||
|
tool := func(name string) buildstep.StepFunc {
|
||||||
|
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||||
|
raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
steps := buildstep.AskSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")}
|
||||||
|
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||||
|
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed}
|
||||||
|
if err != nil {
|
||||||
|
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
step.Result, _ = json.Marshal(out)
|
||||||
|
return step
|
||||||
}
|
}
|
||||||
|
|
||||||
func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, pres: &buildrun.PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
|
r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
|
||||||
for _, n := range []string{"ace", "novox"} {
|
for _, n := range []string{"ace", "novox"} {
|
||||||
r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
|
r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
|
||||||
r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
|
r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
|
||||||
r.pres.Put(n, presenceValue(t, true), false)
|
|
||||||
}
|
}
|
||||||
|
r.asks = &buildrun.FakeAsks{Answer: r.answer}
|
||||||
r.b = &Builder{
|
r.b = &Builder{
|
||||||
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second},
|
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second},
|
||||||
Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf,
|
Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf,
|
||||||
}
|
}
|
||||||
var err error
|
var err error
|
||||||
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil {
|
||||||
@@ -220,15 +256,53 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOneQualifyingNodeBuildsOnce(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
|
// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key
|
||||||
|
// before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference.
|
||||||
|
func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) {
|
||||||
r := newRig(t, goBundle)
|
r := newRig(t, goBundle)
|
||||||
r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true)
|
r.asks.Answer = nil // no node has pulled it yet
|
||||||
v := r.consider(t, "modules/m")
|
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||||
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") {
|
var asks []delivery.StepAsk
|
||||||
t.Errorf("build = %+v, want done with its step saying novox", v)
|
for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("no ask was put")
|
||||||
|
}
|
||||||
|
asks = r.asks.Asks()
|
||||||
}
|
}
|
||||||
if n := len(r.bm.callsOf("build_go_build")); n != 1 {
|
a := asks[0]
|
||||||
t.Errorf("%d builds, want one", n)
|
if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 {
|
||||||
|
t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks)
|
||||||
|
}
|
||||||
|
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID {
|
||||||
|
t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID)
|
||||||
|
}
|
||||||
|
if n := len(r.bm.callsOf("build_go_build")) + len(r.bm.callsOf("build_go_fetch")); n != 0 {
|
||||||
|
t.Fatalf("the builder called a node %d times", n)
|
||||||
|
}
|
||||||
|
r.asks.End(r.answer(a)) // a node pulled it and ran it
|
||||||
|
r.b.Wait()
|
||||||
|
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || !strings.Contains(v.Step, "on ace") || v.Artifacts[0].Reference == "" || len(v.Asks) != 0 {
|
||||||
|
t.Fatalf("the key = %+v, want done on ace naming the reference", v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/AC3: a builder that starts again while its ask waits puts no second ask: it waits for the step of the ask the
|
||||||
|
// building key names.
|
||||||
|
func TestARestartedBuilderWaitsForTheAskTheKeyNames(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
const id = "mvl0frv18d1957"
|
||||||
|
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||||
|
Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}})
|
||||||
|
r.st.vals[r.key] = data
|
||||||
|
r.asks.Answer = nil
|
||||||
|
r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed,
|
||||||
|
Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)})
|
||||||
|
v := r.consider(t, "modules/m")
|
||||||
|
if n := len(r.asks.Asks()); n != 0 {
|
||||||
|
t.Fatalf("%d asks put again", n)
|
||||||
|
}
|
||||||
|
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on shanks") {
|
||||||
|
t.Fatalf("the key = %+v, want done from the ask's step on shanks", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -247,20 +321,9 @@ func TestAFailingStepFailsTheBuildWithItsReasonAndNodeAndIsTriedAgainAsANewRevis
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// imageHave marks mesh-build-image running and healthy on these nodes.
|
|
||||||
func imageHave(t *testing.T, nodes ...string) *buildrun.HaveSet {
|
|
||||||
t.Helper()
|
|
||||||
have := &buildrun.HaveSet{}
|
|
||||||
for _, n := range nodes {
|
|
||||||
have.Put(n+".mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
|
||||||
have.Put(n+"."+ImageBuildModule, haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
|
|
||||||
}
|
|
||||||
return have
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { // 1705a, ADR 0487 clause 5
|
func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { // 1705a, ADR 0487 clause 5
|
||||||
r := newRig(t, imageArt)
|
r := newRig(t, imageArt)
|
||||||
r.b.Health = imageHave(t, "ace", "novox") // two nodes qualify; an image still builds on one
|
r.puller = "ace" // two nodes qualify; an image still builds on one
|
||||||
v := r.consider(t, "modules/m")
|
v := r.consider(t, "modules/m")
|
||||||
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest {
|
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest {
|
||||||
t.Fatalf("build = %+v, want done with one image artifact and its digest", v)
|
t.Fatalf("build = %+v, want done with one image artifact and its digest", v)
|
||||||
@@ -278,19 +341,9 @@ func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { /
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnImageIsBuiltOnlyWhereMeshBuildImageIsHealthy(t *testing.T) { // 1705a
|
|
||||||
r := newRig(t, imageArt)
|
|
||||||
r.b.Health = imageHave(t, "novox")
|
|
||||||
r.consider(t, "modules/m")
|
|
||||||
c := r.bm.callsOf("build_image_build")
|
|
||||||
if len(c) != 1 || c[0].node != "novox" {
|
|
||||||
t.Errorf("calls = %+v, want one on novox", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
|
func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
|
||||||
r := newRig(t, goBundle, imageArt)
|
r := newRig(t, goBundle, imageArt)
|
||||||
r.b.Health = imageHave(t, "ace", "novox")
|
r.puller = "ace"
|
||||||
v := r.consider(t, "modules/m")
|
v := r.consider(t, "modules/m")
|
||||||
if v.State != delivery.BuildDone || len(v.Artifacts) != 2 {
|
if v.State != delivery.BuildDone || len(v.Artifacts) != 2 {
|
||||||
t.Fatalf("build = %+v", v)
|
t.Fatalf("build = %+v", v)
|
||||||
@@ -303,33 +356,9 @@ func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWithNoImageNodeTheKeyFailsNamingTheProvisionAndIsNotRetakenUntilANodeReportsTheModule(t *testing.T) { // 1705b, lead's decision
|
|
||||||
r := newRig(t, imageArt)
|
|
||||||
r.b.Health = imageHave(t) // mesh-build-image runs nowhere
|
|
||||||
v := r.consider(t, "modules/m")
|
|
||||||
if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, NoImageBuilderPrefix) || !strings.Contains(v.Error, "rootless-image-build") || !strings.Contains(v.Error, "buildah") || !strings.Contains(v.Error, ImageBuildModule) {
|
|
||||||
t.Fatalf("build = %+v, want failed naming rootless-image-build, buildah and mesh-build-image", v)
|
|
||||||
}
|
|
||||||
if len(r.bm.calls) != 0 {
|
|
||||||
t.Errorf("%d calls, want none", len(r.bm.calls))
|
|
||||||
}
|
|
||||||
writes := r.st.writes
|
|
||||||
r.consider(t, "modules/m")
|
|
||||||
r.consider(t, "modules/m")
|
|
||||||
if r.st.writes != writes {
|
|
||||||
t.Errorf("the failed key was written again (%d writes, was %d)", r.st.writes, writes)
|
|
||||||
}
|
|
||||||
// A node reports the module: the key is taken again.
|
|
||||||
r.b.Health = imageHave(t, "ace")
|
|
||||||
v = r.consider(t, "modules/m")
|
|
||||||
if v.State != delivery.BuildDone {
|
|
||||||
t.Errorf("build = %+v, want done once a node reports mesh-build-image", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision
|
func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision
|
||||||
r := newRig(t, imageArt)
|
r := newRig(t, imageArt)
|
||||||
r.b.Health = imageHave(t, "ace")
|
r.puller = "ace"
|
||||||
trunkModule(t, r)
|
trunkModule(t, r)
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||||
r.b.Wait()
|
r.b.Wait()
|
||||||
@@ -348,7 +377,7 @@ func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T)
|
|||||||
|
|
||||||
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b
|
func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b
|
||||||
r := newRig(t, imageArt)
|
r := newRig(t, imageArt)
|
||||||
r.b.Health = imageHave(t, "ace")
|
r.puller = "ace"
|
||||||
r.consider(t, "modules/m")
|
r.consider(t, "modules/m")
|
||||||
calls := r.bm.callsOf("build_image_build")
|
calls := r.bm.callsOf("build_image_build")
|
||||||
if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true {
|
if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true {
|
||||||
@@ -439,7 +468,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t *
|
|||||||
// whose compare-and-set comes first, and the other's write is refused, never overwriting it.
|
// whose compare-and-set comes first, and the other's write is refused, never overwriting it.
|
||||||
func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4
|
func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4
|
||||||
r := newRig(t, goBundle)
|
r := newRig(t, goBundle)
|
||||||
other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf}
|
other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Asks: r.b.Asks, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf}
|
||||||
value := moduleValue(t, "modules/m")
|
value := moduleValue(t, "modules/m")
|
||||||
var wg sync.WaitGroup
|
var wg sync.WaitGroup
|
||||||
for _, b := range []*Builder{r.b, other} {
|
for _, b := range []*Builder{r.b, other} {
|
||||||
@@ -522,36 +551,9 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3
|
|
||||||
r := newRig(t, goBundle)
|
|
||||||
r.tools.Put(buildrun.GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps
|
|
||||||
v := r.consider(t, "modules/m")
|
|
||||||
if v.State != delivery.BuildDone {
|
|
||||||
t.Fatalf("build = %+v", v)
|
|
||||||
}
|
|
||||||
for _, c := range r.bm.calls {
|
|
||||||
if c.node == "g14" {
|
|
||||||
t.Errorf("a call went to g14, which is not online: %+v", c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// With no node online the key fails naming why; a node coming online builds it as a new revision.
|
|
||||||
r2 := newRig(t, goBundle)
|
|
||||||
r2.pres.Put("ace", nil, true)
|
|
||||||
r2.pres.Put("novox", nil, true)
|
|
||||||
if v := r2.consider(t, "modules/m"); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "online") {
|
|
||||||
t.Fatalf("build = %+v, want failed because no node is online", v)
|
|
||||||
}
|
|
||||||
r2.pres.Put("ace", presenceValue(t, true), false)
|
|
||||||
r2.b.Reconsider(context.Background())
|
|
||||||
r2.b.Wait()
|
|
||||||
if v := valueOf(t, r2.st, r2.key); v.State != delivery.BuildDone {
|
|
||||||
t.Errorf("build = %+v, want done once a node is online", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5
|
func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5
|
||||||
r := newRig(t, imageArt)
|
r := newRig(t, imageArt)
|
||||||
r.b.Health = imageHave(t, "ace")
|
r.puller = "ace"
|
||||||
trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's
|
trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||||
r.b.Wait()
|
r.b.Wait()
|
||||||
@@ -590,7 +592,7 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi
|
|||||||
|
|
||||||
func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5
|
func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5
|
||||||
r := newRig(t, imageArt)
|
r := newRig(t, imageArt)
|
||||||
r.b.Health = imageHave(t, "ace")
|
r.puller = "ace"
|
||||||
trunkModule(t, r)
|
trunkModule(t, r)
|
||||||
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
|
||||||
r.b.Wait()
|
r.b.Wait()
|
||||||
@@ -612,3 +614,86 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re
|
|||||||
t.Errorf("build = %+v, want done with a reference", v)
|
t.Errorf("build = %+v, want done with a reference", v)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked
|
||||||
|
// again; the key is done when a node takes it.
|
||||||
|
func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
r.asks.Answer = nil
|
||||||
|
r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond}
|
||||||
|
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("no ask was put")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding {
|
||||||
|
t.Fatalf("the key = %+v while its ask waits, want building", v)
|
||||||
|
}
|
||||||
|
r.asks.End(r.answer(r.asks.Asks()[0]))
|
||||||
|
r.b.Wait()
|
||||||
|
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 {
|
||||||
|
t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// heldAsks says an ask is held when its id is among held.
|
||||||
|
type heldAsks map[string]bool
|
||||||
|
|
||||||
|
func (h heldAsks) Held(_ context.Context, a delivery.StepAsk) (bool, error) { return h[a.ID], nil }
|
||||||
|
|
||||||
|
// 1910: a builder that stopped between naming the ask in its key and putting it left a key naming an ask neither the
|
||||||
|
// stream holds nor any node ran: started again, the builder puts that ask, under the same id, once, and the key is
|
||||||
|
// built from its step; an ask the stream holds is not put again.
|
||||||
|
func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) {
|
||||||
|
for _, held := range []bool{false, true} {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
const id = "mvl0frv18d1958"
|
||||||
|
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||||
|
Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}})
|
||||||
|
r.st.vals[r.key] = data
|
||||||
|
r.b.Held = heldAsks{id: held}
|
||||||
|
if held {
|
||||||
|
r.asks.Answer = nil
|
||||||
|
r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed,
|
||||||
|
Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)})
|
||||||
|
}
|
||||||
|
v := r.consider(t, "modules/m")
|
||||||
|
asks := r.asks.Asks()
|
||||||
|
switch {
|
||||||
|
case held && len(asks) != 0:
|
||||||
|
t.Errorf("an ask the stream holds was put again: %+v", asks)
|
||||||
|
case !held && (len(asks) != 1 || asks[0].ID != id):
|
||||||
|
t.Errorf("the ask nobody put: asks %+v, want one under %s", asks, id)
|
||||||
|
}
|
||||||
|
if v.State != delivery.BuildDone {
|
||||||
|
t.Errorf("held %v: the key = %+v, want done", held, v)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// unreadableHeld cannot tell whether an ask is held.
|
||||||
|
type unreadableHeld struct{}
|
||||||
|
|
||||||
|
func (unreadableHeld) Held(context.Context, delivery.StepAsk) (bool, error) {
|
||||||
|
return false, errors.New("permissions violation for publish to $JS.API.STREAM.INFO.SEAT_NODE_BUILD_GO")
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1910: a take-over that cannot read whether the ask its key names is on its stream says so and waits for the ask's step:
|
||||||
|
// the key is not failed, and the build finishes when a node ran the ask.
|
||||||
|
func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) {
|
||||||
|
r := newRig(t, goBundle)
|
||||||
|
const id = "mvl0frv18d1959"
|
||||||
|
data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD,
|
||||||
|
Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}})
|
||||||
|
r.st.vals[r.key] = data
|
||||||
|
r.b.Held = unreadableHeld{}
|
||||||
|
r.asks.Answer = nil
|
||||||
|
r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed,
|
||||||
|
Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)})
|
||||||
|
v := r.consider(t, "modules/m")
|
||||||
|
if v.State != delivery.BuildDone || len(r.asks.Asks()) != 0 {
|
||||||
|
t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -38,6 +38,7 @@ type manifest struct {
|
|||||||
Consumes []string `json:"consumes"`
|
Consumes []string `json:"consumes"`
|
||||||
State []state `json:"state"`
|
State []state `json:"state"`
|
||||||
Reads []string `json:"reads"`
|
Reads []string `json:"reads"`
|
||||||
|
Uses []string `json:"uses"`
|
||||||
Invokes []string `json:"invokes"`
|
Invokes []string `json:"invokes"`
|
||||||
Config map[string]struct {
|
Config map[string]struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
@@ -83,18 +84,26 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
|||||||
if m.Module != "mesh-delivery-builder" {
|
if m.Module != "mesh-delivery-builder" {
|
||||||
t.Errorf("module = %q", m.Module)
|
t.Errorf("module = %q", m.Module)
|
||||||
}
|
}
|
||||||
if want := []state{{Name: "builds", History: 5}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory
|
||||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||||
}
|
}
|
||||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
// 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and
|
||||||
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
// holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5).
|
||||||
|
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||||
|
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||||
}
|
}
|
||||||
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||||
t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes)
|
t.Errorf("uses = %v, want the three build seats", m.Uses)
|
||||||
}
|
}
|
||||||
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
|
// 1937/AC3: the builder asks the controller whether the artifact store still holds a done key's blobs (held), and no
|
||||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
// other tool; it holds no capacity.
|
||||||
|
if want := []string{"seat:mesh-controller.held"}; !reflect.DeepEqual(m.Invokes, want) || len(m.Config) != 0 {
|
||||||
|
t.Errorf("invokes = %v, config = %v, want %v and no config", m.Invokes, m.Config, want)
|
||||||
|
}
|
||||||
|
// The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it
|
||||||
|
// starts; and by the controller's artifacts-gone, a take-in refused for a gone blob (1937/AC4).
|
||||||
|
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported", "mesh-controller.artifacts-gone"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||||
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
t.Errorf("consumes = %v, want %v", m.Consumes, want)
|
||||||
}
|
}
|
||||||
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
|
||||||
@@ -103,7 +112,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
|
|||||||
if m.SecretsOwner != "mesh-delivery-builder" {
|
if m.SecretsOwner != "mesh-delivery-builder" {
|
||||||
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
t.Errorf("secrets-owner = %q", m.SecretsOwner)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} {
|
for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "config"} {
|
||||||
if _, ok := top[k]; ok {
|
if _, ok := top[k]; ok {
|
||||||
t.Errorf("the manifest declares %q: the builder has none", k)
|
t.Errorf("the manifest declares %q: the builder has none", k)
|
||||||
}
|
}
|
||||||
@@ -129,35 +138,19 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1
|
|||||||
t.Errorf("process = %v", p)
|
t.Errorf("process = %v", p)
|
||||||
}
|
}
|
||||||
env, _ := p["env"].(map[string]any)
|
env, _ := p["env"].(map[string]any)
|
||||||
// The build capacity of each node is the module's setting `capacity` (issue 1661, ADR 0478 clause 3), laid into the
|
if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
|
||||||
// settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity.
|
t.Errorf("env = %v, want the broker file alone", p["env"])
|
||||||
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_SETTINGS"] != "${dir:state}/settings.json" {
|
|
||||||
t.Errorf("env = %v", p["env"])
|
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) {
|
|
||||||
t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"])
|
|
||||||
}
|
|
||||||
c, ok := m.Config["capacity"]
|
|
||||||
if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "shanks=3,ace=2,g14=0,novox=0" || c.Description == "" {
|
|
||||||
t.Errorf("config = %+v, want one string setting, capacity, shanks=3,ace=2,g14=0,novox=0 to start", m.Config)
|
|
||||||
}
|
|
||||||
var settings map[string]any
|
|
||||||
for _, r := range m.Resources {
|
for _, r := range m.Resources {
|
||||||
if r["id"] == "settings" {
|
if r["id"] == "settings" {
|
||||||
settings = r
|
t.Errorf("a settings resource %v: the builder holds no setting since the capacity is the build modules' (ADR 0525 §5)", r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" {
|
|
||||||
t.Errorf("settings resource = %v", settings)
|
|
||||||
}
|
|
||||||
if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": "shanks=3,ace=2,g14=0,novox=0"`) {
|
|
||||||
t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err)
|
|
||||||
}
|
|
||||||
if _, bus := p["bus"]; bus {
|
if _, bus := p["bus"]; bus {
|
||||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||||
}
|
}
|
||||||
if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "files" || m.Build.Artifacts[1].Kind != "archive" || m.Build.Artifacts[1].From != "files" {
|
if len(m.Build.Artifacts) != 1 {
|
||||||
t.Fatalf("artifacts = %+v, want the program and the files", m.Build.Artifacts)
|
t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts)
|
||||||
}
|
}
|
||||||
a := m.Build.Artifacts[0]
|
a := m.Build.Artifacts[0]
|
||||||
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
|
||||||
|
|||||||
@@ -4,10 +4,15 @@
|
|||||||
"slug": "builder",
|
"slug": "builder",
|
||||||
"consumes": [
|
"consumes": [
|
||||||
"mesh-build-go.toolchain.reported",
|
"mesh-build-go.toolchain.reported",
|
||||||
"mesh-build-node.toolchain.reported"
|
"mesh-build-node.toolchain.reported",
|
||||||
|
"mesh-controller.artifacts-gone"
|
||||||
|
],
|
||||||
|
"invokes": [
|
||||||
|
"seat:mesh-controller.held"
|
||||||
],
|
],
|
||||||
"state": [
|
"state": [
|
||||||
{ "name": "builds", "history": 5 }
|
{ "name": "builds", "history": 5 },
|
||||||
|
{ "name": "instances", "history": 1 }
|
||||||
],
|
],
|
||||||
"reads": [
|
"reads": [
|
||||||
"mesh-build-go.steps",
|
"mesh-build-go.steps",
|
||||||
@@ -15,29 +20,17 @@
|
|||||||
"mesh-build-image.steps",
|
"mesh-build-image.steps",
|
||||||
"mesh-build-node.steps",
|
"mesh-build-node.steps",
|
||||||
"mesh-build-node.toolchains",
|
"mesh-build-node.toolchains",
|
||||||
|
"mesh-controller.assignments",
|
||||||
"mesh-delivery-indexer.candidates",
|
"mesh-delivery-indexer.candidates",
|
||||||
"mesh-delivery-indexer.manifests",
|
"mesh-delivery-indexer.manifests",
|
||||||
"mesh-delivery-indexer.modules",
|
"mesh-delivery-indexer.modules"
|
||||||
"mesh-delivery-watchman.presence",
|
|
||||||
"node-engine.have"
|
|
||||||
],
|
],
|
||||||
"invokes": [
|
"uses": [
|
||||||
"mesh-build-go.build_go_build",
|
"node-build-go",
|
||||||
"mesh-build-go.build_go_fetch",
|
"node-build-image",
|
||||||
"mesh-build-image.build_image_build",
|
"node-build-node"
|
||||||
"mesh-build-image.build_image_copy",
|
|
||||||
"mesh-build-image.build_image_fetch",
|
|
||||||
"mesh-build-node.build_node_build",
|
|
||||||
"mesh-build-node.build_node_fetch"
|
|
||||||
],
|
],
|
||||||
"config": {
|
|
||||||
"capacity": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "How many build steps the builder runs at once on each node, as node=steps pairs separated by commas, such as shanks=3,ace=2 (ADR 0478 clause 3); a node set to 0 never builds, a node not named runs one at a time, and empty is one everywhere. The builder builds on shanks and ace only (the operator, 2026-10-21).",
|
|
||||||
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
|
|
||||||
"default": "shanks=3,ace=2,g14=0,novox=0"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"own-secrets": {
|
"own-secrets": {
|
||||||
"broker": "${dir:state}/broker"
|
"broker": "${dir:state}/broker"
|
||||||
},
|
},
|
||||||
@@ -58,17 +51,6 @@
|
|||||||
"place": ".",
|
"place": ".",
|
||||||
"unknown": "program"
|
"unknown": "program"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "settings",
|
|
||||||
"trusted": false,
|
|
||||||
"type": "file",
|
|
||||||
"path": "${dir:state}/settings.json",
|
|
||||||
"mode": "0600",
|
|
||||||
"owner": "mesh-delivery-builder",
|
|
||||||
"takes-config": ["capacity"],
|
|
||||||
"merge": "json",
|
|
||||||
"template": "files/settings.json"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "builder",
|
"id": "builder",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -78,12 +60,8 @@
|
|||||||
"./mesh-delivery-builder"
|
"./mesh-delivery-builder"
|
||||||
],
|
],
|
||||||
"env": {
|
"env": {
|
||||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||||
"MESH_BUILDER_SETTINGS": "${dir:state}/settings.json"
|
|
||||||
},
|
},
|
||||||
"restart-on": [
|
|
||||||
"settings"
|
|
||||||
],
|
|
||||||
"user": "mesh-delivery-builder",
|
"user": "mesh-delivery-builder",
|
||||||
"health": {
|
"health": {
|
||||||
"kind": "unit"
|
"kind": "unit"
|
||||||
@@ -99,11 +77,6 @@
|
|||||||
"system": "arch",
|
"system": "arch",
|
||||||
"from": "cmd/mesh-delivery-builder",
|
"from": "cmd/mesh-delivery-builder",
|
||||||
"binary": "mesh-delivery-builder"
|
"binary": "mesh-delivery-builder"
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "files",
|
|
||||||
"kind": "archive",
|
|
||||||
"from": "files"
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func TryTools(tr Trying) []stdio.Tool {
|
|||||||
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " +
|
||||||
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
"its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " +
|
||||||
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
"where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " +
|
||||||
"the checker checks no level of (hq, one whose trunk the gate does not name) is refused.",
|
"the checker checks no level of (one whose trunk the gate does not name) is refused.",
|
||||||
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{
|
||||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||||
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
"head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"},
|
||||||
@@ -104,8 +104,7 @@ func Tools(r check.Reader) []stdio.Tool {
|
|||||||
return map[string]any{"verdicts": vs}, err
|
return map[string]any{"verdicts": vs}, err
|
||||||
}},
|
}},
|
||||||
{Name: check.PullReadTool,
|
{Name: check.PullReadTool,
|
||||||
Description: "The verdict that judges one pull request and the full report of its status mesh/check; for an hq pull request, that hq is judged by " +
|
Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.",
|
||||||
"merge-check.sh until issue 1296. Read-only.",
|
|
||||||
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
|
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
|
||||||
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
|
||||||
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
|
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
|
||||||
@@ -143,50 +142,47 @@ func Tools(r check.Reader) []stdio.Tool {
|
|||||||
return map[string]any{"previews": ps, "unreadable": bad}, err
|
return map[string]any{"previews": ps, "unreadable": bad}, err
|
||||||
}},
|
}},
|
||||||
{Name: check.StatusReadTool,
|
{Name: check.StatusReadTool,
|
||||||
Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " +
|
Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " +
|
||||||
"delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " +
|
"and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " +
|
||||||
"candidates without a verdict for their heads. The steps the checker holds waiting for a place are in its process, not in a bucket, and the " +
|
"build module left running (started past the step bound, or its build module not running on its node) is listed apart under " +
|
||||||
"answer says so. Read-only.",
|
"left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " +
|
||||||
|
"build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.",
|
||||||
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
Input: map[string]any{"type": "object", "properties": map[string]any{}},
|
||||||
Run: func(map[string]any) (any, error) { return r.Status() }},
|
Run: func(map[string]any) (any, error) { return r.Status() }},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func capacity() (string, error) {
|
// stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a
|
||||||
path := os.Getenv("MESH_CHECKER_SETTINGS")
|
// duration.
|
||||||
if path == "" {
|
func stepBound() time.Duration {
|
||||||
return os.Getenv("MESH_CHECKER_CAPACITY"), nil
|
if d, err := time.ParseDuration(os.Getenv("MESH_CHECKER_STEP_BOUND")); err == nil && d > 0 {
|
||||||
|
return d
|
||||||
}
|
}
|
||||||
raw, err := os.ReadFile(path)
|
return check.DefaultStepBound
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
var s struct {
|
|
||||||
Capacity string `json:"capacity"`
|
|
||||||
}
|
|
||||||
return s.Capacity, json.Unmarshal(raw, &s)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")),
|
||||||
PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")),
|
PreviewState: read(stdio.State("previews")),
|
||||||
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))},
|
||||||
Capacity: capacity}
|
HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()}
|
||||||
tries := stdio.State(check.TriesState)
|
tries := stdio.State(check.TriesState)
|
||||||
source := stdio.State("mesh-delivery-gate.source")
|
source := stdio.State("mesh-delivery-gate.source")
|
||||||
|
trunks := func() (map[string]string, error) {
|
||||||
|
e, err := source.Get(delivery.SourceMainKey)
|
||||||
|
if err != nil || e == nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
s, err := delivery.DecodeSource(e.Value)
|
||||||
|
return s.Heads, err
|
||||||
|
}
|
||||||
|
r.Trunks = trunks
|
||||||
trying := Trying{
|
trying := Trying{
|
||||||
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
Tries: stateStore{StateRead: read(tries), create: func(key string, value []byte) error {
|
||||||
_, err := tries.Create(key, json.RawMessage(value))
|
_, err := tries.Create(key, json.RawMessage(value))
|
||||||
return err
|
return err
|
||||||
}},
|
}},
|
||||||
Trunks: func() (map[string]string, error) {
|
Trunks: trunks,
|
||||||
e, err := source.Get(delivery.SourceMainKey)
|
|
||||||
if err != nil || e == nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
s, err := delivery.DecodeSource(e.Value)
|
|
||||||
return s.Heads, err
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
if err := stdio.Serve("", append(Tools(r), TryTools(trying)...)); err != nil {
|
||||||
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
|
||||||
|
|||||||
@@ -105,43 +105,6 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the
|
|
||||||
// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every
|
|
||||||
// change after. No timer: the caller bounds the wait with its context.
|
|
||||||
type stepsWaiter struct{ js bucketOpener }
|
|
||||||
|
|
||||||
func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
|
|
||||||
kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module))
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
|
|
||||||
}
|
|
||||||
watch, err := kv.Watch(ctx, key)
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Step{}, err
|
|
||||||
}
|
|
||||||
defer watch.Stop()
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return delivery.Step{}, ctx.Err()
|
|
||||||
case e, ok := <-watch.Updates():
|
|
||||||
if !ok {
|
|
||||||
return delivery.Step{}, errors.New("the watch of the step ended")
|
|
||||||
}
|
|
||||||
if e == nil || !opOf(e) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
step, err := delivery.DecodeStep(e.Value())
|
|
||||||
if err != nil {
|
|
||||||
return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err)
|
|
||||||
}
|
|
||||||
if step.State != delivery.StepRunning {
|
|
||||||
return step, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
|
||||||
type manifestReader struct{ kv jetstream.KeyValue }
|
type manifestReader struct{ kv jetstream.KeyValue }
|
||||||
|
|
||||||
|
|||||||
+63
@@ -0,0 +1,63 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// assignerWrote raises the assigner's compositions bucket, the assigner online in the watchman's presence, and its verdict
|
||||||
|
// under the check's key, as the assigner leaves them on the bus.
|
||||||
|
func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerdict) {
|
||||||
|
t.Helper()
|
||||||
|
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, pkey, on); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
data, err := delivery.EncodeCompositionVerdict(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(b.ctx, key, data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not
|
||||||
|
// "not yet run": a pass names its machine, a fail fails the check with the assigner's words.
|
||||||
|
func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) {
|
||||||
|
for _, c := range []struct {
|
||||||
|
state, owed, words string
|
||||||
|
}{
|
||||||
|
{delivery.CompositionPass, "success", "composition passed"},
|
||||||
|
{delivery.CompositionFail, "failure", "g14 runs web, which the change removes from its source"},
|
||||||
|
} {
|
||||||
|
b := startBed(t)
|
||||||
|
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||||
|
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
|
||||||
|
v := delivery.CompositionVerdict{Group: group, Heads: heads, State: c.state, SDK: "v0.1.14-x", Machines: []string{"g14"}}
|
||||||
|
if c.state == delivery.CompositionFail {
|
||||||
|
v.Machine, v.Words = "g14", c.words
|
||||||
|
}
|
||||||
|
b.assignerWrote(t, key, v)
|
||||||
|
b.start(t, nil)
|
||||||
|
got := b.verdict(t, key, ended)
|
||||||
|
if got.Composition == nil || got.Composition.State != c.state {
|
||||||
|
t.Fatalf("%s: the verdict's composition is %+v", c.state, got.Composition)
|
||||||
|
}
|
||||||
|
o := b.owed(t, 7, head1, c.owed)
|
||||||
|
if !strings.Contains(o.Text, c.words) || strings.Contains(o.Text, "not yet run") {
|
||||||
|
t.Errorf("%s: the line says %q", c.state, o.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-checker/internal/check"
|
||||||
|
)
|
||||||
|
|
||||||
|
// compositionWatch reads the assigner's composition of a check (ADR 0528 clause 5) by watching its key and the
|
||||||
|
// assigner's presence: no poll and no timer. With no assigner online it answers check.ErrAssignerNotRunning at once.
|
||||||
|
type compositionWatch struct {
|
||||||
|
kv jetstream.KeyValue
|
||||||
|
// presence is the watchman's presence bucket; nil (not there) counts as no assigner online.
|
||||||
|
presence jetstream.KeyValue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Await answers the composition under key once the assigner wrote one that does not wait (check.AwaitComposition).
|
||||||
|
func (c compositionWatch) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||||
|
ctx, cancel := context.WithCancel(ctx)
|
||||||
|
defer cancel()
|
||||||
|
comps := make(chan check.CompositionEvent)
|
||||||
|
presence := make(chan check.PresenceEvent)
|
||||||
|
cw, err := c.kv.Watch(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
return delivery.CompositionVerdict{}, err
|
||||||
|
}
|
||||||
|
defer cw.Stop()
|
||||||
|
go func() {
|
||||||
|
for e := range cw.Updates() {
|
||||||
|
ev := check.CompositionEvent{Settled: e == nil}
|
||||||
|
if e != nil && e.Operation() == jetstream.KeyValuePut {
|
||||||
|
ev.Value = e.Value()
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case comps <- ev:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
if c.presence == nil {
|
||||||
|
go func() {
|
||||||
|
select {
|
||||||
|
case presence <- check.PresenceEvent{Settled: true}:
|
||||||
|
case <-ctx.Done():
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
} else {
|
||||||
|
pw, err := c.presence.Watch(ctx, "part."+delivery.AssignerModule+".*")
|
||||||
|
if err != nil {
|
||||||
|
return delivery.CompositionVerdict{}, err
|
||||||
|
}
|
||||||
|
defer pw.Stop()
|
||||||
|
go func() {
|
||||||
|
for e := range pw.Updates() {
|
||||||
|
ev := check.PresenceEvent{Settled: e == nil}
|
||||||
|
if e != nil {
|
||||||
|
ev.Key = e.Key()
|
||||||
|
if e.Operation() == jetstream.KeyValuePut {
|
||||||
|
if p, err := delivery.DecodePresence(e.Value()); err == nil {
|
||||||
|
ev.Online = p.Online
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case presence <- ev:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
return check.AwaitComposition(ctx, presence, comps)
|
||||||
|
}
|
||||||
|
|
||||||
|
// followCompositions hands the checker each composition the assigner writes from now on, so a verdict that said its
|
||||||
|
// composition was not judged takes it in when the assigner starts (event-driven).
|
||||||
|
func followCompositions(ctx context.Context, kv jetstream.KeyValue, ch *check.Checker) error {
|
||||||
|
w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly())
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
defer w.Stop()
|
||||||
|
for e := range w.Updates() {
|
||||||
|
if e == nil || e.Operation() != jetstream.KeyValuePut {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if comp, err := delivery.DecodeCompositionVerdict(e.Value()); err == nil {
|
||||||
|
ch.Composed(ctx, comp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// lazyCompositions opens the assigner's compositions when a check first asks for one and the bucket is there: until it
|
||||||
|
// is, no assigner ever ran, so the check says at once that its composition is not judged (check.ErrAssignerNotRunning),
|
||||||
|
// and the next check looks again. No timer: each look is a check asking.
|
||||||
|
type lazyCompositions struct {
|
||||||
|
open func(ctx context.Context) (check.CompositionReader, error)
|
||||||
|
|
||||||
|
mu sync.Mutex
|
||||||
|
reader check.CompositionReader
|
||||||
|
}
|
||||||
|
|
||||||
|
// Await is the opened reader's Await, opening it first when it is not yet.
|
||||||
|
func (l *lazyCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||||
|
l.mu.Lock()
|
||||||
|
r := l.reader
|
||||||
|
if r == nil {
|
||||||
|
opened, err := l.open(ctx)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||||
|
l.mu.Unlock()
|
||||||
|
return delivery.CompositionVerdict{}, check.ErrAssignerNotRunning
|
||||||
|
case err != nil:
|
||||||
|
l.mu.Unlock()
|
||||||
|
return delivery.CompositionVerdict{}, err
|
||||||
|
}
|
||||||
|
l.reader, r = opened, opened
|
||||||
|
}
|
||||||
|
l.mu.Unlock()
|
||||||
|
return r.Await(ctx, key)
|
||||||
|
}
|
||||||
+106
@@ -0,0 +1,106 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats-server/v2/server"
|
||||||
|
"github.com/nats-io/nats.go"
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-checker/internal/check"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is
|
||||||
|
// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key.
|
||||||
|
func TestTheCheckerAwaitsTheAssignersComposition(t *testing.T) {
|
||||||
|
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { os.RemoveAll(dir) })
|
||||||
|
srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
go srv.Start()
|
||||||
|
if !srv.ReadyForConnections(10 * time.Second) {
|
||||||
|
t.Fatal("the bus did not come up")
|
||||||
|
}
|
||||||
|
t.Cleanup(srv.Shutdown)
|
||||||
|
nc, err := nats.Connect(srv.ClientURL())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(nc.Close)
|
||||||
|
js, _ := jetstream.New(nc)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pkv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PresenceBucket, History: 5})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox")
|
||||||
|
on, _ := delivery.EncodePresence(delivery.Presence{Online: true})
|
||||||
|
if _, err := pkv.Put(ctx, pkey, on); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// With no assigner online the wait answers at once (a second key, before the assigner of this test is online).
|
||||||
|
off, _ := delivery.EncodePresence(delivery.Presence{Online: false})
|
||||||
|
if _, err := pkv.Put(ctx, pkey, off); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
other, _ := delivery.CompositionKey("feat/y", "0123456789ab")
|
||||||
|
if _, err := (compositionWatch{kv: kv, presence: pkv}).Await(ctx, other); !errors.Is(err, check.ErrAssignerNotRunning) {
|
||||||
|
t.Fatalf("with no assigner online the wait answered %v", err)
|
||||||
|
}
|
||||||
|
if _, err := pkv.Put(ctx, pkey, on); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key, _ := delivery.CompositionKey("feat/x", "0123456789ab")
|
||||||
|
got := make(chan delivery.CompositionVerdict, 1)
|
||||||
|
go func() {
|
||||||
|
v, err := compositionWatch{kv: kv, presence: pkv}.Await(ctx, key)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
got <- v
|
||||||
|
}()
|
||||||
|
put := func(v delivery.CompositionVerdict) {
|
||||||
|
data, err := delivery.EncodeCompositionVerdict(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, key, data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(200 * time.Millisecond)
|
||||||
|
put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "the indexer holds no candidates yet"})
|
||||||
|
select {
|
||||||
|
case v := <-got:
|
||||||
|
t.Fatalf("a waiting composition was answered: %+v", v)
|
||||||
|
case <-time.After(300 * time.Millisecond):
|
||||||
|
}
|
||||||
|
put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}})
|
||||||
|
select {
|
||||||
|
case v := <-got:
|
||||||
|
if v.State != delivery.CompositionPass || v.Machines[0] != "g14" {
|
||||||
|
t.Errorf("answered %+v", v)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("the composition written was not answered")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
|
"mesh-delivery-checker/internal/check"
|
||||||
|
)
|
||||||
|
|
||||||
|
type answering struct{ v delivery.CompositionVerdict }
|
||||||
|
|
||||||
|
func (a answering) Await(context.Context, string) (delivery.CompositionVerdict, error) {
|
||||||
|
return a.v, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297 review item 3: the compositions bucket missing when the checker starts is looked up again at the next check: while
|
||||||
|
// it is missing a check says at once the assigner is not running; once it is there the check reads its composition, and
|
||||||
|
// it is opened once.
|
||||||
|
func TestTheCompositionsBucketIsLookedUpAtEachCheck(t *testing.T) {
|
||||||
|
opens, there := 0, false
|
||||||
|
l := &lazyCompositions{open: func(context.Context) (check.CompositionReader, error) {
|
||||||
|
opens++
|
||||||
|
if !there {
|
||||||
|
return nil, jetstream.ErrBucketNotFound
|
||||||
|
}
|
||||||
|
return answering{delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x"}}, nil
|
||||||
|
}}
|
||||||
|
if _, err := l.Await(context.Background(), "k"); !errors.Is(err, check.ErrAssignerNotRunning) {
|
||||||
|
t.Fatalf("without the bucket: %v", err)
|
||||||
|
}
|
||||||
|
there = true
|
||||||
|
for range 2 {
|
||||||
|
v, err := l.Await(context.Background(), "k")
|
||||||
|
if err != nil || v.State != delivery.CompositionPass {
|
||||||
|
t.Fatalf("with the bucket: %+v, %v", v, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if opens != 2 {
|
||||||
|
t.Errorf("opened %d times, want twice (missing, then once)", opens)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
|
||||||
)
|
|
||||||
|
|
||||||
// heardCaller is the build runner's caller with the calls that reached nobody marked: no responders, or none answering within
|
|
||||||
// the call's bound. The step never began there, so the check may give it to another node (issue 1820). A call that was
|
|
||||||
// answered, with a failure or not, is left as it is.
|
|
||||||
type heardCaller struct {
|
|
||||||
call func(node, key string, body any) (json.RawMessage, error)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (h heardCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
|
|
||||||
raw, err := h.call(node, key, body)
|
|
||||||
if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) {
|
|
||||||
return nil, &check.Unheard{Err: err}
|
|
||||||
}
|
|
||||||
return raw, err
|
|
||||||
}
|
|
||||||
@@ -1,30 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestACallThatReachedNobodyIsMarkedAndAnAnsweredFailureIsNot(t *testing.T) { // 1820/AC6
|
|
||||||
for _, err := range []error{nats.ErrNoResponders, nats.ErrTimeout, context.DeadlineExceeded} {
|
|
||||||
h := heardCaller{call: func(string, string, any) (json.RawMessage, error) { return nil, fmt.Errorf("call: %w", err) }}
|
|
||||||
_, got := h.CallOn("g14", "m.t", nil)
|
|
||||||
var u *check.Unheard
|
|
||||||
if !errors.As(got, &u) {
|
|
||||||
t.Errorf("%v: %v is not marked unheard", err, got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
h := heardCaller{call: func(string, string, any) (json.RawMessage, error) { return nil, errors.New("the module refused") }}
|
|
||||||
_, got := h.CallOn("g14", "m.t", nil)
|
|
||||||
var u *check.Unheard
|
|
||||||
if errors.As(got, &u) {
|
|
||||||
t.Errorf("an answered failure %v is marked unheard", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,101 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"log"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go/jetstream"
|
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
|
||||||
)
|
|
||||||
|
|
||||||
// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): whether one does is the list of the keys of
|
|
||||||
// the controller's bucket of the delivery plans in their build phase, read each time a check is about to start. Its keys age
|
|
||||||
// out (a time to live, and no delete marker), which a watch never hears, so a watch cannot decide it (mesh-controller
|
|
||||||
// internal/link/holds.go lists the keys too); the watch is only the wake-up, for a plan that ended by a delete. The bucket
|
|
||||||
// is read as empty where it is absent (issue 1652 makes it) or cannot be read, saying so once. It may come into being after
|
|
||||||
// the checker started and the checker holds no timer (ADR 0437 §8), so each time a check is about to start and the bucket is
|
|
||||||
// not known yet, the checker looks for it.
|
|
||||||
type holdWatch struct {
|
|
||||||
ctx context.Context
|
|
||||||
js bucketOpener
|
|
||||||
keys func(ctx context.Context) ([]string, error)
|
|
||||||
changed func(ctx context.Context) // a plan key changed: the checks that waited may start
|
|
||||||
logger *log.Logger
|
|
||||||
errs chan error
|
|
||||||
mu sync.Mutex
|
|
||||||
watched bool
|
|
||||||
said bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func newHoldWatch(ctx context.Context, js bucketOpener, changed func(context.Context), logger *log.Logger) *holdWatch {
|
|
||||||
return &holdWatch{ctx: ctx, js: js, changed: changed, logger: logger, errs: make(chan error, 1)}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Held says whether a delivery builds now.
|
|
||||||
func (w *holdWatch) Held() bool {
|
|
||||||
w.ensure()
|
|
||||||
w.mu.Lock()
|
|
||||||
keys := w.keys
|
|
||||||
w.mu.Unlock()
|
|
||||||
if keys == nil {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
list, err := keys(w.ctx)
|
|
||||||
if err != nil {
|
|
||||||
w.say("the keys of %s cannot be listed, so it is read as empty: %v", check.PlansBuildingBucket, err)
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return len(list) > 0
|
|
||||||
}
|
|
||||||
|
|
||||||
func (w *holdWatch) say(format string, args ...any) {
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
if !w.said {
|
|
||||||
w.said = true
|
|
||||||
w.logger.Printf(format, args...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensure finds the bucket if it is not known and exists, and starts the wake-up watch.
|
|
||||||
func (w *holdWatch) ensure() {
|
|
||||||
w.mu.Lock()
|
|
||||||
if w.watched || w.js == nil {
|
|
||||||
w.mu.Unlock()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.mu.Unlock()
|
|
||||||
kv, err := w.js.KeyValue(w.ctx, check.PlansBuildingBucket)
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
w.say("the bucket %s does not exist yet: it is read as empty, so no check is held back", check.PlansBuildingBucket)
|
|
||||||
return
|
|
||||||
case err != nil:
|
|
||||||
w.say("the bucket %s cannot be read, so it is read as empty: %v", check.PlansBuildingBucket, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.mu.Lock()
|
|
||||||
defer w.mu.Unlock()
|
|
||||||
if w.watched {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.watched, w.said = true, false
|
|
||||||
w.keys = func(ctx context.Context) ([]string, error) {
|
|
||||||
list, err := kv.Keys(ctx)
|
|
||||||
if errors.Is(err, jetstream.ErrNoKeysFound) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
return list, err
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
wake, err := kv.WatchAll(w.ctx)
|
|
||||||
if err != nil {
|
|
||||||
w.errs <- err
|
|
||||||
return
|
|
||||||
}
|
|
||||||
w.errs <- follow(w.ctx, wake, func(jetstream.KeyValueEntry) { w.changed(w.ctx) })
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
@@ -1,35 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"log"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The controller's plans-building keys age out of the bucket (a time to live, no delete marker), which a watch never hears:
|
|
||||||
// whether a delivery builds is the list of the keys now, and the watch is only a wake-up (review D3).
|
|
||||||
func TestADeliveryHoldsWhileItsKeyIsListedAndNotWhenItAgedOutWithNoEvent(t *testing.T) {
|
|
||||||
listed := []string{"plan-1"}
|
|
||||||
w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return listed, nil }}
|
|
||||||
if !w.Held() {
|
|
||||||
t.Fatal("a plan key is listed and nothing holds")
|
|
||||||
}
|
|
||||||
listed = nil // the key aged out: no delete reached the watch
|
|
||||||
if w.Held() {
|
|
||||||
t.Error("the key is gone from the bucket and the checker is still held")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestABucketThatCannotBeListedIsReadAsEmpty(t *testing.T) {
|
|
||||||
w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return []string{"plan-1"}, errors.New("the bus is down") }}
|
|
||||||
if w.Held() {
|
|
||||||
t.Error("an unreadable bucket held the checker")
|
|
||||||
}
|
|
||||||
var absent holdWatch
|
|
||||||
absent.logger = log.New(io.Discard, "", 0)
|
|
||||||
if absent.Held() {
|
|
||||||
t.Error("a watch with no bucket held the checker")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,10 +5,12 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -20,6 +22,7 @@ import (
|
|||||||
"github.com/nats-io/nats.go/jetstream"
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/announce"
|
"git.novox.be/novox/mesh-sdk/go/announce"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
|
||||||
"mesh-delivery-checker/internal/check"
|
"mesh-delivery-checker/internal/check"
|
||||||
@@ -75,11 +78,13 @@ func readCaptured(t *testing.T, name string) map[string]any {
|
|||||||
type fakeBuildModules struct {
|
type fakeBuildModules struct {
|
||||||
js jetstream.JetStream
|
js jetstream.JetStream
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
n atomic.Int64
|
|
||||||
conflict atomic.Bool
|
conflict atomic.Bool
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
calls map[string]int // by `<module>.<tool>`
|
calls map[string]int // by `<module>.<tool>`
|
||||||
testTool *testing.T
|
testTool *testing.T
|
||||||
|
// beside is, by workspace, the siblings the fetch placed in it as `<owner/repo> <commit>`, sorted: a digest echoes them
|
||||||
|
// (1929/AC2; mesh-sdk buildstep.besideOf), or the level runs whole and keeps nothing.
|
||||||
|
beside map[string][]string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeBuildModules) count(tool string) int {
|
func (f *fakeBuildModules) count(tool string) int {
|
||||||
@@ -99,10 +104,30 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
|||||||
}
|
}
|
||||||
fetched := readCaptured(t, name)
|
fetched := readCaptured(t, name)
|
||||||
fetched["commit"], fetched["base"] = args["commit"], args["base"]
|
fetched["commit"], fetched["base"] = args["commit"], args["base"]
|
||||||
|
var refs []struct {
|
||||||
|
Repository string `json:"repository"`
|
||||||
|
Commit string `json:"commit"`
|
||||||
|
}
|
||||||
|
raw, _ := json.Marshal(args["beside"])
|
||||||
|
_ = json.Unmarshal(raw, &refs)
|
||||||
|
var lines []string
|
||||||
|
for _, r := range refs {
|
||||||
|
lines = append(lines, r.Repository+" "+r.Commit)
|
||||||
|
}
|
||||||
|
sort.Strings(lines)
|
||||||
|
f.mu.Lock()
|
||||||
|
if f.beside == nil {
|
||||||
|
f.beside = map[string][]string{}
|
||||||
|
}
|
||||||
|
f.beside[fmt.Sprint(fetched["workspace"])] = lines
|
||||||
|
f.mu.Unlock()
|
||||||
v = fetched
|
v = fetched
|
||||||
case strings.HasSuffix(tool, "_digest"):
|
case strings.HasSuffix(tool, "_digest"):
|
||||||
d := map[string]string{"unit": digU, "integration": digI}[level]
|
d := map[string]string{"unit": digU, "integration": digI}[level]
|
||||||
v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}}
|
f.mu.Lock()
|
||||||
|
beside := f.beside[fmt.Sprint(args["workspace"])]
|
||||||
|
f.mu.Unlock()
|
||||||
|
v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}, "beside": beside}
|
||||||
case strings.HasSuffix(tool, "_test"):
|
case strings.HasSuffix(tool, "_test"):
|
||||||
r := readCaptured(t, "go-test-slow")
|
r := readCaptured(t, "go-test-slow")
|
||||||
r["level"] = level
|
r["level"] = level
|
||||||
@@ -112,45 +137,55 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any
|
|||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
|
// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on n1 as the real module
|
||||||
steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
// does since R4g (ADR 0525; issue 1887 A5): each ask's job runs with step bodies answering the captured results, and a
|
||||||
|
// check takes the kept verdicts of the checker's kept bucket by their digests.
|
||||||
|
func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.KeyValue) {
|
||||||
|
kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) {
|
language := strings.TrimPrefix(module, "mesh-build-")
|
||||||
tokens := strings.Split(m.Subject, ".") // mesh mod <module> call <tool> <node> <caller>
|
if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil {
|
||||||
tool, node := tokens[4], tokens[5]
|
t.Fatal(err)
|
||||||
f.mu.Lock()
|
}
|
||||||
f.calls[module+"."+tool]++
|
prefix := map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_"}[module]
|
||||||
f.mu.Unlock()
|
call := func(name string) buildstep.StepFunc {
|
||||||
var args map[string]any
|
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||||
_ = json.Unmarshal(m.Data, &args)
|
tool := prefix + name
|
||||||
id := fmt.Sprintf("s%d", f.n.Add(1))
|
f.mu.Lock()
|
||||||
key, _ := delivery.StepKey(node, id)
|
f.calls[module+"."+tool]++
|
||||||
put := func(s delivery.Step) {
|
f.mu.Unlock()
|
||||||
data, err := delivery.EncodeStep(s)
|
return f.result(t, tool, args), nil
|
||||||
if err == nil {
|
}
|
||||||
_, err = steps.Put(f.ctx, key, data)
|
}
|
||||||
|
steps := buildstep.AskSteps{Fetch: call("fetch"), Test: call("test"),
|
||||||
|
Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
|
||||||
|
key, err := delivery.KeptKey(level, digest)
|
||||||
|
if err != nil {
|
||||||
|
return 0, false, err
|
||||||
|
}
|
||||||
|
e, err := kept.Get(ctx, key)
|
||||||
|
if errors.Is(err, jetstream.ErrKeyNotFound) {
|
||||||
|
return 0, false, nil
|
||||||
}
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Errorf("writing step %s: %v", key, err)
|
return 0, false, err
|
||||||
}
|
}
|
||||||
}
|
v, err := delivery.DecodeKept(e.Value())
|
||||||
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()}
|
return v.Tests, err == nil, err
|
||||||
put(step)
|
}}
|
||||||
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node})
|
if module == "mesh-build-go" {
|
||||||
_ = m.Respond(answer)
|
steps.Digest = call("digest")
|
||||||
go func() {
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
step.Ended = time.Now().UTC()
|
|
||||||
step.State, step.Result = delivery.StepPassed, f.result(t, tool, args)
|
|
||||||
put(step)
|
|
||||||
}()
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
}
|
||||||
t.Cleanup(func() { _ = sub.Unsubscribe() })
|
put := func(key string, value any) (uint64, error) {
|
||||||
|
raw, _ := value.(json.RawMessage)
|
||||||
|
return kv.Put(f.ctx, key, raw)
|
||||||
|
}
|
||||||
|
p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put,
|
||||||
|
Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond,
|
||||||
|
Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }}
|
||||||
|
go func() { _ = p.Pull(f.ctx) }()
|
||||||
}
|
}
|
||||||
|
|
||||||
// serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured):
|
// serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured):
|
||||||
@@ -277,14 +312,23 @@ func startBed(t *testing.T) *bed {
|
|||||||
}
|
}
|
||||||
b.buckets[name] = kv
|
b.buckets[name] = kv
|
||||||
}
|
}
|
||||||
|
// 1929/AC1: the indexer's value of the repository's module, with no TypeScript manifest, so the Go build module alone
|
||||||
|
// judges the repository (an unknown repository is judged by both, issue 1887 A12).
|
||||||
|
known, err := delivery.EncodeModule(delivery.Module{Repository: repo, Commit: trunk, Path: "modules/m", None2: "the fingerprint is computed with its chain"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := b.buckets[delivery.ModulesBucket].Put(ctx, "m", known); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
b.setOnline(t, "n1", true)
|
b.setOnline(t, "n1", true)
|
||||||
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
|
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
|
||||||
b.setHealth(t, "n1", module)
|
b.setHealth(t, "n1", module)
|
||||||
}
|
}
|
||||||
b.serveForge(t, ctl)
|
b.serveForge(t, ctl)
|
||||||
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
|
||||||
b.fake.serve(t, ctl, "mesh-build-go")
|
b.fake.serve(t, "mesh-build-go", b.buckets[delivery.CheckerKeptBucket])
|
||||||
b.fake.serve(t, ctl, "mesh-build-node")
|
b.fake.serve(t, "mesh-build-node", b.buckets[delivery.CheckerKeptBucket])
|
||||||
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
|
||||||
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
|
||||||
b.setTrunk(t, map[string]string{repo: trunk})
|
b.setTrunk(t, map[string]string{repo: trunk})
|
||||||
@@ -425,7 +469,7 @@ func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
|
|||||||
// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
|
// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
|
||||||
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
|
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
|
||||||
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
|
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
|
||||||
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1
|
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1, 1929/AC1
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||||
b.start(t, nil)
|
b.start(t, nil)
|
||||||
@@ -465,7 +509,7 @@ func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300
|
|||||||
|
|
||||||
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
|
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
|
||||||
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
|
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
|
||||||
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6
|
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6, 1929/AC2
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||||
b.start(t, nil)
|
b.start(t, nil)
|
||||||
@@ -520,14 +564,25 @@ func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { /
|
|||||||
|
|
||||||
// TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in
|
// TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in
|
||||||
// the same run is.
|
// the same run is.
|
||||||
func TestAnHqPullRequestGetsNoVerdictAndAnotherDoes(t *testing.T) { // ADR 0484 clause 8
|
// 1887/A10, ADR 0527 §2: a pull request into hq gets a verdict and a mesh/check from the checker, as any repository's.
|
||||||
|
func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) {
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
|
b.setTrunk(t, map[string]string{repo: trunk, "novox/hq": trunk})
|
||||||
hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1))
|
hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1))
|
||||||
other := b.candidate(t, "feat/x", pull(repo, 7, head2))
|
|
||||||
b.start(t, nil)
|
b.start(t, nil)
|
||||||
b.verdict(t, other, ended)
|
if v := b.verdict(t, hq, ended); v.State != delivery.VerdictPass {
|
||||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, hq); err == nil {
|
t.Fatalf("hq's verdict = %+v", v)
|
||||||
t.Error("hq's pull request has a verdict")
|
}
|
||||||
|
key, _ := delivery.CheckerOwedKey("novox/hq", 3, head1)
|
||||||
|
for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key); err == nil {
|
||||||
|
if o, err := delivery.DecodeCheckerOwed(e.Value()); err == nil && o.State == "success" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatalf("hq#3 owes no success within 20 s")
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -550,37 +605,9 @@ func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds: the controller's bucket of the plans in their build phase
|
|
||||||
// holds a plan's key; the candidate waits, no step is called; the key deleted, the check starts and passes (ADR 0484 clause 12).
|
|
||||||
func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1
|
|
||||||
b := startBed(t)
|
|
||||||
plans, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_plans-building", History: 1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := plans.Put(b.ctx, "plan-1", []byte(`{"plan":"plan-1","since":"2026-10-21T19:00:00Z","seen":1}`)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
|
||||||
b.start(t, nil)
|
|
||||||
time.Sleep(1500 * time.Millisecond) // a wait for nothing to happen: there is no event to wait on
|
|
||||||
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key); err == nil {
|
|
||||||
t.Fatal("a check started while a delivery builds")
|
|
||||||
}
|
|
||||||
if n := b.fake.count("mesh-build-go.build_go_fetch"); n != 0 {
|
|
||||||
t.Fatalf("%d fetches while a delivery builds", n)
|
|
||||||
}
|
|
||||||
if err := plans.Delete(b.ctx, "plan-1"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
|
|
||||||
t.Errorf("verdict = %+v", v)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not
|
// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not
|
||||||
// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9).
|
// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9).
|
||||||
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2
|
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2, 1929/AC3
|
||||||
b := startBed(t)
|
b := startBed(t)
|
||||||
b.stalePin.Store(true)
|
b.stalePin.Store(true)
|
||||||
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
|
||||||
@@ -589,9 +616,7 @@ func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T
|
|||||||
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, repo+"#7") {
|
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, repo+"#7") {
|
||||||
t.Fatalf("verdict = %+v", v)
|
t.Fatalf("verdict = %+v", v)
|
||||||
}
|
}
|
||||||
if n := b.fake.count("mesh-build-go.build_go_test"); n != 0 {
|
// 1929/AC3: no assertion that no test ran: one ask runs fetch, digest and tests (ADR 0525 §3).
|
||||||
t.Errorf("%d test steps ran after a stale pin", n)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// triesStore is the tries bucket as the tools write it.
|
// triesStore is the tries bucket as the tools write it.
|
||||||
@@ -677,7 +702,7 @@ func TestTheCheckerSaysOwedReadableOnTheBus(t *testing.T) { // R2-1
|
|||||||
break
|
break
|
||||||
}
|
}
|
||||||
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
if said.Module != "mesh-delivery-checker" || said.Node != "n1" || !said.Keeps("owed") || !said.Readable("owed") ||
|
||||||
!said.Readable("previews") || !said.Readable("verdicts") || said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
|
!said.Readable("previews") || !said.Readable("verdicts") || !said.Readable("kept") || said.Readable("tries") || len(said.States) != 5 {
|
||||||
t.Fatalf("the checker said %+v, want its five states with owed, previews and verdicts readable", said)
|
t.Fatalf("the checker said %+v, want its five states with kept, owed, previews and verdicts readable (1929/AC4; 1887 A5b)", said)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -15,11 +15,8 @@ import (
|
|||||||
|
|
||||||
// judgeDeps are the parts the judge of a check is made of.
|
// judgeDeps are the parts the judge of a check is made of.
|
||||||
type judgeDeps struct {
|
type judgeDeps struct {
|
||||||
runner buildrun.Runner
|
asks check.Asks
|
||||||
tools *buildrun.ToolchainSet
|
tools *buildrun.ToolchainSet
|
||||||
presence *buildrun.PresenceSet
|
|
||||||
have *buildrun.HaveSet
|
|
||||||
places *buildrun.Places
|
|
||||||
trunks *check.Trunks
|
trunks *check.Trunks
|
||||||
kept check.KV
|
kept check.KV
|
||||||
previews check.KV
|
previews check.KV
|
||||||
@@ -27,11 +24,13 @@ type judgeDeps struct {
|
|||||||
forge check.Forge
|
forge check.Forge
|
||||||
announce check.Announcer
|
announce check.Announcer
|
||||||
builds check.BuildStates
|
builds check.BuildStates
|
||||||
logf func(format string, args ...any)
|
// compositions is the assigner's bucket of compositions; nil leaves every composition not run.
|
||||||
|
compositions check.CompositionReader
|
||||||
|
logf func(format string, args ...any)
|
||||||
}
|
}
|
||||||
|
|
||||||
func newJudge(d judgeDeps) check.Judge {
|
func newJudge(d judgeDeps) check.Judge {
|
||||||
return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places,
|
return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools,
|
||||||
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
|
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -142,17 +141,28 @@ func (r *repoModules) PackagePaths(repository string) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildModules are the build modules that judge a repository, sorted.
|
// BuildModules are the build modules that judge a repository, sorted: the Go build module always, and the Node build
|
||||||
|
// module when a module of the repository builds TypeScript. A repository of which the mesh knows no module (mesh-lab, hq)
|
||||||
|
// is judged by both, each finding its own directories in the tree (go.mod files; package.json files), so a repository
|
||||||
|
// with tests in either language and no manifest is judged whole (issue 1887 A12).
|
||||||
func (r *repoModules) BuildModules(repository string) []string {
|
func (r *repoModules) BuildModules(repository string) []string {
|
||||||
out := []string{buildrun.GoBuildModule}
|
out := []string{buildrun.GoBuildModule}
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
defer r.mu.Unlock()
|
||||||
|
known := false
|
||||||
for module, repo := range r.repoOf {
|
for module, repo := range r.repoOf {
|
||||||
if repo == repository && r.langs[r.typed[module]]["typescript"] {
|
if repo != repository {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
known = true
|
||||||
|
if r.langs[r.typed[module]]["typescript"] {
|
||||||
out = append(out, buildrun.NodeBuildModule)
|
out = append(out, buildrun.NodeBuildModule)
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !known {
|
||||||
|
out = append(out, buildrun.NodeBuildModule)
|
||||||
|
}
|
||||||
sort.Strings(out)
|
sort.Strings(out)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,8 +69,10 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test
|
|||||||
if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" {
|
if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" {
|
||||||
t.Errorf("the commit the mesh runs = %q", got)
|
t.Errorf("the commit the mesh runs = %q", got)
|
||||||
}
|
}
|
||||||
if got := r.BuildModules("novox/mesh-sdk"); strings.Join(got, ",") != buildrun.GoBuildModule {
|
// 1887/A12: a repository with no module the mesh knows (mesh-lab: nested go.mod files and a root package.json) is
|
||||||
t.Errorf("a repository with no module: %v, want the Go build module", got)
|
// judged by both build modules.
|
||||||
|
if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule {
|
||||||
|
t.Errorf("a repository with no module: %v, want both build modules", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -37,7 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { //
|
|||||||
if !equal(manifest, announced) {
|
if !equal(manifest, announced) {
|
||||||
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
t.Errorf("announced %v, the manifest keeps %v", announced, manifest)
|
||||||
}
|
}
|
||||||
if want := []string{"owed", "previews", "verdicts"}; !equal(readable, want) {
|
// kept is read by the Go build module for a check ask (issue 1887 design limit 1).
|
||||||
|
if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) {
|
||||||
t.Errorf("readable %v, want %v", readable, want)
|
t.Errorf("readable %v, want %v", readable, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,9 +40,6 @@ func eventSubject(event string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
// defaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a
|
|
||||||
// test level runs a repository's tests).
|
|
||||||
defaultStepBound = 30 * time.Minute
|
|
||||||
// defaultLink is the base of the address a status points at; the verdict key follows.
|
// defaultLink is the base of the address a status points at; the verdict key follows.
|
||||||
defaultLink = "mesh-delivery-view/checks"
|
defaultLink = "mesh-delivery-view/checks"
|
||||||
)
|
)
|
||||||
@@ -117,7 +114,9 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti
|
|||||||
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
// keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module,
|
||||||
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
// which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate.
|
||||||
var keptStates = []announce.State{
|
var keptStates = []announce.State{
|
||||||
{Name: "kept"},
|
// The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's
|
||||||
|
// design limit 1); the checker alone writes them (ADR 0444 clause 5).
|
||||||
|
{Name: "kept", Readable: true},
|
||||||
{Name: "owed", Readable: true},
|
{Name: "owed", Readable: true},
|
||||||
{Name: "previews", Readable: true},
|
{Name: "previews", Readable: true},
|
||||||
{Name: "tries"},
|
{Name: "tries"},
|
||||||
@@ -184,7 +183,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
logger.Printf("what the checker keeps could not be said on the bus, so readers of owed, previews and verdicts are not granted until it is: %v", err)
|
logger.Printf("what the checker keeps could not be said on the bus, so readers of owed, previews and verdicts are not granted until it is: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", defaultStepBound)
|
stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", check.DefaultStepBound)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -192,20 +191,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_CHECKER_SETTINGS")), getenv("MESH_CHECKER_CAPACITY"))
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the capacity setting: %w", err)
|
|
||||||
}
|
|
||||||
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
|
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
|
||||||
if link == "" {
|
if link == "" {
|
||||||
link = defaultLink
|
link = defaultLink
|
||||||
}
|
}
|
||||||
tools := &buildrun.ToolchainSet{}
|
tools := &buildrun.ToolchainSet{}
|
||||||
presence := &buildrun.PresenceSet{}
|
presence := &buildrun.PresenceSet{}
|
||||||
have := &buildrun.HaveSet{}
|
|
||||||
trunks := &check.Trunks{}
|
trunks := &check.Trunks{}
|
||||||
places := &buildrun.Places{Capacity: capacity}
|
// A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a
|
||||||
runner := buildrun.Runner{Caller: heardCaller{caller.CallOn}, Steps: stepsWaiter{js}, Bound: stepBound}
|
// free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step.
|
||||||
|
asks := buildrun.JetStreamAsks{JS: js}
|
||||||
ch := &check.Checker{
|
ch := &check.Checker{
|
||||||
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
|
||||||
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
|
||||||
@@ -225,11 +220,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
default:
|
default:
|
||||||
builds = buildReader{kvStore{kv}}
|
builds = buildReader{kvStore{kv}}
|
||||||
}
|
}
|
||||||
|
// The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is
|
||||||
|
// there, so an assigner delivered after the checker started is read at the next check, and followed from then on.
|
||||||
|
compositions := &lazyCompositions{open: func(at context.Context) (check.CompositionReader, error) {
|
||||||
|
kv, err := js.KeyValue(at, delivery.AssignerCompositionsBucket)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
cw := compositionWatch{kv: kv}
|
||||||
|
if pkv, err := js.KeyValue(at, delivery.PresenceBucket); err == nil {
|
||||||
|
cw.presence = pkv
|
||||||
|
}
|
||||||
|
if err := followCompositions(ctx, kv, ch); err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s cannot be watched: %w", delivery.AssignerCompositionsBucket, err)
|
||||||
|
}
|
||||||
|
logger.Printf("the assigner's compositions are read from %s", delivery.AssignerCompositionsBucket)
|
||||||
|
return cw, nil
|
||||||
|
}}
|
||||||
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
// The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries.
|
||||||
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf}
|
||||||
holds := newHoldWatch(ctx, js, func(ctx context.Context) { ch.Reconsider(ctx); tryer.Reconsider(ctx) }, logger)
|
|
||||||
ch.Hold = holds.Held
|
|
||||||
tryer.Hold = holds.Held
|
|
||||||
announce := func(ctx context.Context, event, key string) {
|
announce := func(ctx context.Context, event, key string) {
|
||||||
body, _ := json.Marshal(map[string]string{"key": key})
|
body, _ := json.Marshal(map[string]string{"key": key})
|
||||||
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
|
||||||
@@ -238,16 +247,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
|
||||||
ch.Judge = newJudge(judgeDeps{
|
ch.Judge = newJudge(judgeDeps{
|
||||||
runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks,
|
asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks,
|
||||||
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
|
||||||
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
|
||||||
announce: announce,
|
announce: announce, compositions: compositions,
|
||||||
})
|
})
|
||||||
ch.Announce = announce
|
ch.Announce = announce
|
||||||
tryer.Judge = ch.Judge
|
tryer.Judge = ch.Judge
|
||||||
|
|
||||||
// The toolchains first, so a check that starts from the first candidate finds the nodes.
|
// The toolchains first, so a check that starts from the first candidate finds the nodes.
|
||||||
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
|
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
|
||||||
|
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||||
|
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||||
|
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, tools.Put, logger.Printf)
|
||||||
|
watch.put = gate.Toolchain
|
||||||
|
assignErrs, err := watchAssignments(ctx, js, gate, ch.NodesChanged, logger)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if err := watch.start(); err != nil {
|
if err := watch.start(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -269,8 +286,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
defer cc.Stop()
|
defer cc.Stop()
|
||||||
}
|
}
|
||||||
errs := make(chan error, 8)
|
errs := make(chan error, 8)
|
||||||
// The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing
|
// The watchman's presence: a node coming back online asks again a check whose forge was not heard (UnheardWords), and
|
||||||
// is checked; the checker says so and is started again with the watchman.
|
// the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no
|
||||||
|
// longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6,
|
||||||
|
// issue 1912), so the node-engine's have is not watched here.
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
case errors.Is(err, jetstream.ErrBucketNotFound):
|
||||||
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
|
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
|
||||||
@@ -284,27 +303,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
go func() {
|
go func() {
|
||||||
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
|
||||||
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
|
||||||
ch.NodesChanged(ctx) // a node came online or went: a check that waited for one may go on
|
ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard
|
||||||
}
|
|
||||||
})
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
// The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not
|
|
||||||
// chosen. Where its bucket is not there yet no build module counts as healthy, so nothing is checked.
|
|
||||||
switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); {
|
|
||||||
case errors.Is(err, jetstream.ErrBucketNotFound):
|
|
||||||
logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the checker starts again", delivery.HaveBucket)
|
|
||||||
case err != nil:
|
|
||||||
return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err)
|
|
||||||
default:
|
|
||||||
hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) })
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
errs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) {
|
|
||||||
if have.Put(e.Key(), e.Value(), !opOf(e)) {
|
|
||||||
ch.NodesChanged(ctx)
|
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}()
|
}()
|
||||||
@@ -378,9 +377,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return nil
|
return nil
|
||||||
case err := <-errs:
|
case err := <-errs:
|
||||||
return err
|
return err
|
||||||
case err := <-watch.errs:
|
case err := <-assignErrs:
|
||||||
return err
|
return err
|
||||||
case err := <-holds.errs:
|
case err := <-watch.errs:
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1912: with checks as asks on the build seats' streams, the stream redelivers a lost node's ask (ADR 0525 §6), so the
|
||||||
|
// checker program opens no node-engine have to start again a check that failed for want of a node.
|
||||||
|
func TestTheCheckerWatchesNoHave(t *testing.T) {
|
||||||
|
raw, err := os.ReadFile("main.go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, s := range []string{"delivery.HaveBucket", "buildrun.HaveSet"} {
|
||||||
|
if strings.Contains(string(raw), s) {
|
||||||
|
t.Errorf("the checker program still reads %s", s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
)
|
|
||||||
|
|
||||||
// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `<node>=<steps at
|
|
||||||
// once>` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a
|
|
||||||
// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch.
|
|
||||||
func capacityFrom(path, fallback string) (map[string]int, error) {
|
|
||||||
if path == "" {
|
|
||||||
return buildrun.ParseCapacity(fallback)
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(path)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("the settings file: %w", err)
|
|
||||||
}
|
|
||||||
var s struct {
|
|
||||||
Capacity string `json:"capacity"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &s); err != nil {
|
|
||||||
return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
|
|
||||||
}
|
|
||||||
return buildrun.ParseCapacity(s.Capacity)
|
|
||||||
}
|
|
||||||
@@ -1,68 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
)
|
|
||||||
|
|
||||||
func settingsFile(t *testing.T, body string) string {
|
|
||||||
t.Helper()
|
|
||||||
p := filepath.Join(t.TempDir(), "settings.json")
|
|
||||||
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
// ADR 0484 clause 12: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere.
|
|
||||||
func TestCapacityIsReadFromTheSettingsFile(t *testing.T) {
|
|
||||||
for body, want := range map[string]map[string]int{
|
|
||||||
`{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2},
|
|
||||||
`{"capacity": ""}`: {},
|
|
||||||
`{}`: {},
|
|
||||||
} {
|
|
||||||
got, err := capacityFrom(settingsFile(t, body), "")
|
|
||||||
if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] {
|
|
||||||
t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
|
|
||||||
if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 {
|
|
||||||
t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace"))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func mustCapacity(t *testing.T, path string) map[string]int {
|
|
||||||
t.Helper()
|
|
||||||
c, err := capacityFrom(path, "")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return c
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) {
|
|
||||||
if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil {
|
|
||||||
t.Error("shanks=-1 accepted")
|
|
||||||
}
|
|
||||||
if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 {
|
|
||||||
t.Errorf("the starting value: %v, %v", c, err)
|
|
||||||
}
|
|
||||||
if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil {
|
|
||||||
t.Error("a file that is not JSON accepted")
|
|
||||||
}
|
|
||||||
if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil {
|
|
||||||
t.Error("a settings file that is not there accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's.
|
|
||||||
func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) {
|
|
||||||
got, err := capacityFrom("", "ace=2")
|
|
||||||
if err != nil || got["ace"] != 2 {
|
|
||||||
t.Errorf("capacity %v, %v", got, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// This file is the same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker (issue 1940): it moves
|
||||||
|
// to mesh-sdk go/delivery/buildrun once mesh-sdk can be delivered again, since a change of mesh-sdk waits on the very
|
||||||
|
// toolchain agreement this file repairs.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"maps"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/nats-io/nats.go/jetstream"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/provider"
|
||||||
|
)
|
||||||
|
|
||||||
|
// A build module writes <node>.<language> of its toolchains state on every node it runs on, and nothing deletes the key
|
||||||
|
// when the module is unassigned from a node: the controller writes no module's state, and the module is gone. A report of
|
||||||
|
// a node the build module no longer runs on is therefore not a builder (issue 1940: mesh-build-node unassigned from ace
|
||||||
|
// on 2026-10-23 left ace.typescript at v26.12.0, and every Node fingerprint said "builders disagree").
|
||||||
|
//
|
||||||
|
// toolchainGate stands between the toolchains buckets and what takes their entries in (the indexer's PutToolchain, a
|
||||||
|
// buildrun.ToolchainSet): an entry is passed on only while the controller's assignments (mesh-controller_assignments,
|
||||||
|
// key <node>.<module>) hold an open assignment of its build module on its node; it is passed on as deleted otherwise,
|
||||||
|
// and passed on again when the module is assigned there again. Until the assignments are settled (or where they cannot
|
||||||
|
// be read) every entry is passed on, as before. Builders that run and report different versions still disagree: that
|
||||||
|
// refusal is the guard of novox/hq ADR 0465, and it stays.
|
||||||
|
type toolchainGate struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
modules map[string]bool // the build modules whose toolchains pass the gate
|
||||||
|
forward func(module, key string, value []byte, deleted bool) bool // answers whether it changed what it holds
|
||||||
|
logf func(format string, a ...any)
|
||||||
|
known bool // the assignments are settled
|
||||||
|
standing map[string]bool // <node>.<module> with an open assignment
|
||||||
|
raw map[string]map[string][]byte // module -> toolchains key -> value, as the bucket holds it
|
||||||
|
left map[string]bool // <module>/<key> said as left out, so it is said once
|
||||||
|
}
|
||||||
|
|
||||||
|
func newToolchainGate(modules []string, forward func(module, key string, value []byte, deleted bool) bool, logf func(string, ...any)) *toolchainGate {
|
||||||
|
g := &toolchainGate{modules: map[string]bool{}, forward: forward, logf: logf, standing: map[string]bool{},
|
||||||
|
raw: map[string]map[string][]byte{}, left: map[string]bool{}}
|
||||||
|
for _, m := range modules {
|
||||||
|
g.modules[m] = true
|
||||||
|
}
|
||||||
|
return g
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toolchain takes one change of a build module's toolchains bucket in and answers whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Toolchain(module, key string, value []byte, deleted bool) bool {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.raw[module] == nil {
|
||||||
|
g.raw[module] = map[string][]byte{}
|
||||||
|
}
|
||||||
|
if deleted {
|
||||||
|
delete(g.raw[module], key)
|
||||||
|
} else {
|
||||||
|
g.raw[module][key] = value
|
||||||
|
}
|
||||||
|
return g.pass(module, key)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Assignment takes one change of the controller's assignments in and answers whether what it passed on changed.
|
||||||
|
func (g *toolchainGate) Assignment(key string, value []byte, deleted bool) bool {
|
||||||
|
node, module, ok := strings.Cut(key, ".")
|
||||||
|
if !ok || !g.modules[module] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
open := !deleted && openAssignment(value)
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
if g.standing[key] == open {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if open {
|
||||||
|
g.standing[key] = true
|
||||||
|
} else {
|
||||||
|
delete(g.standing, key)
|
||||||
|
}
|
||||||
|
if !g.known {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
changed := false
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||||
|
if n, _, err := delivery.ParseToolchainKey(k); err == nil && n == node {
|
||||||
|
changed = g.pass(module, k) || changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Settled says the assignments are read: from now on only the reports of assigned nodes pass. It answers whether what it
|
||||||
|
// passed on changed.
|
||||||
|
func (g *toolchainGate) Settled() bool {
|
||||||
|
g.mu.Lock()
|
||||||
|
defer g.mu.Unlock()
|
||||||
|
g.known = true
|
||||||
|
changed := false
|
||||||
|
for _, module := range slices.Sorted(maps.Keys(g.raw)) {
|
||||||
|
for _, k := range slices.Sorted(maps.Keys(g.raw[module])) {
|
||||||
|
changed = g.pass(module, k) || changed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return changed
|
||||||
|
}
|
||||||
|
|
||||||
|
// pass hands one key on as it stands: its value when its node is assigned (or the assignments are not known), deleted
|
||||||
|
// otherwise. The caller holds mu.
|
||||||
|
func (g *toolchainGate) pass(module, key string) bool {
|
||||||
|
value, held := g.raw[module][key]
|
||||||
|
if !held {
|
||||||
|
return g.forward(module, key, nil, true)
|
||||||
|
}
|
||||||
|
node, _, err := delivery.ParseToolchainKey(key)
|
||||||
|
if err != nil || !g.known || g.standing[node+"."+module] {
|
||||||
|
delete(g.left, module+"/"+key)
|
||||||
|
return g.forward(module, key, value, false)
|
||||||
|
}
|
||||||
|
if !g.left[module+"/"+key] {
|
||||||
|
g.left[module+"/"+key] = true
|
||||||
|
g.logf("%s.%s is left out of the agreed toolchain: %s has no assignment on %s, so it is no builder (issue 1940)",
|
||||||
|
delivery.ToolchainsBucket(module), key, module, node)
|
||||||
|
}
|
||||||
|
return g.forward(module, key, nil, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
// openAssignment is whether a value of the controller's assignments holds an assignment that stands: one that has not
|
||||||
|
// ended and has no state (a pending one is not on the machine yet; mesh-controller internal/chain Assignment).
|
||||||
|
func openAssignment(value []byte) bool {
|
||||||
|
var a struct {
|
||||||
|
Ended time.Time `json:"ended"`
|
||||||
|
State string `json:"state"`
|
||||||
|
}
|
||||||
|
if json.Unmarshal(value, &a) != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return a.Ended.IsZero() && a.State == ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// watchAssignments settles the controller's assignments into the gate, says them settled, and follows them; changed is
|
||||||
|
// told whenever what the gate passes on changed. Where the bucket cannot be read (not there, or this module may not read
|
||||||
|
// it) it says so and the gate passes every report on, as before; an error of the watch after that ends on the channel.
|
||||||
|
func watchAssignments(ctx context.Context, js bucketOpener, gate *toolchainGate, changed func(context.Context), logger *log.Logger) (<-chan error, error) {
|
||||||
|
errs := make(chan error, 1)
|
||||||
|
kv, err := js.KeyValue(ctx, provider.AssignmentsBucket)
|
||||||
|
if err != nil {
|
||||||
|
// Not fatal: an indexer, builder or checker that stops for it stops every delivery (a grant comes with a delivery).
|
||||||
|
logger.Printf("the bucket %s cannot be read, so every node's toolchain report counts, an unassigned build node's too (issue 1940): %v", provider.AssignmentsBucket, err)
|
||||||
|
return errs, nil
|
||||||
|
}
|
||||||
|
each := func(e jetstream.KeyValueEntry) { gate.Assignment(e.Key(), e.Value(), !opOf(e)) }
|
||||||
|
w, err := settle(ctx, kv, each)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("the bucket %s: %w", provider.AssignmentsBucket, err)
|
||||||
|
}
|
||||||
|
if gate.Settled() {
|
||||||
|
changed(ctx)
|
||||||
|
}
|
||||||
|
go func() {
|
||||||
|
errs <- follow(ctx, w, func(e jetstream.KeyValueEntry) {
|
||||||
|
if gate.Assignment(e.Key(), e.Value(), !opOf(e)) {
|
||||||
|
changed(ctx)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}()
|
||||||
|
return errs, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
// The same in mesh-delivery-indexer, mesh-delivery-builder and mesh-delivery-checker, as standing.go is (issue 1940).
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
|
)
|
||||||
|
|
||||||
|
func gateReport(t *testing.T, node, version string) (string, []byte) {
|
||||||
|
t.Helper()
|
||||||
|
key, err := delivery.ToolchainKey(node, "typescript")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "typescript", Version: version, Recipe: "f59571cf4be75adf", At: time.Unix(0, 0).UTC()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return key, value
|
||||||
|
}
|
||||||
|
|
||||||
|
func agreedOf(set *buildrun.ToolchainSet) (string, string) {
|
||||||
|
tc, none := delivery.AgreedToolchain(set.Entries("typescript"))
|
||||||
|
return tc.Version, none
|
||||||
|
}
|
||||||
|
|
||||||
|
// The live case of issue 1940: mesh-build-node was unassigned from ace, whose report stayed at v26.12.0 while the
|
||||||
|
// three nodes it runs on report v26.12.1. Without the assignments every report counts (and they disagree); with them
|
||||||
|
// ace's is left out and the three agree; assigned to ace again, its report counts again and the guard refuses again.
|
||||||
|
func TestAnUnassignedBuildNodesReportIsNoBuilder(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
var said []string
|
||||||
|
gate := newToolchainGate([]string{buildrun.GoBuildModule, buildrun.NodeBuildModule}, set.Put, func(f string, a ...any) { said = append(said, f) })
|
||||||
|
for node, version := range map[string]string{"ace": "v26.12.0", "g14": "v26.12.1", "novox": "v26.12.1", "shanks": "v26.12.1"} {
|
||||||
|
key, value := gateReport(t, node, version)
|
||||||
|
gate.Toolchain(buildrun.NodeBuildModule, key, value, false)
|
||||||
|
}
|
||||||
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree") {
|
||||||
|
t.Fatalf("before the assignments are read every report counts, so they disagree; got %q", none)
|
||||||
|
}
|
||||||
|
for _, node := range []string{"g14", "novox", "shanks"} {
|
||||||
|
gate.Assignment(node+"."+buildrun.NodeBuildModule, []byte(`{"node":"`+node+`","module":"mesh-build-node"}`), false)
|
||||||
|
}
|
||||||
|
gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","ended":"2026-10-23T08:00:00Z"}`), false)
|
||||||
|
gate.Assignment("ace.mesh-build-go", []byte(`{"node":"ace","module":"mesh-build-go"}`), false) // another module's: no matter
|
||||||
|
if !gate.Settled() {
|
||||||
|
t.Fatal("settling the assignments left ace's report out, so what was passed on changed")
|
||||||
|
}
|
||||||
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("the assigned nodes agree on v26.12.1; got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
if len(said) != 1 {
|
||||||
|
t.Fatalf("ace's report is said left out once; said %v", said)
|
||||||
|
}
|
||||||
|
// ace reports again while unassigned (an old value seen again): still left out.
|
||||||
|
key, value := gateReport(t, "ace", "v26.12.0")
|
||||||
|
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("an unassigned node's report changed what was passed on")
|
||||||
|
}
|
||||||
|
// Assigned to ace again: its report counts, and the builders really disagree: the guard stays.
|
||||||
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node"}`), false) {
|
||||||
|
t.Fatal("assigning ace again passed nothing on")
|
||||||
|
}
|
||||||
|
if _, none := agreedOf(set); !strings.HasPrefix(none, "builders disagree: v26.12.0 recipe f59571cf4be75adf on ace") {
|
||||||
|
t.Fatalf("builders that run different versions still disagree; got %q", none)
|
||||||
|
}
|
||||||
|
// The assignment's key deleted, or a pending one written: no assignment stands, and ace's report goes again.
|
||||||
|
if !gate.Assignment("ace.mesh-build-node", []byte(`{"node":"ace","module":"mesh-build-node","state":"pending"}`), false) {
|
||||||
|
t.Fatal("a pending assignment is no assignment on the machine")
|
||||||
|
}
|
||||||
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
gate.Assignment("ace.mesh-build-node", nil, true)
|
||||||
|
if v, _ := agreedOf(set); v != "v26.12.1" {
|
||||||
|
t.Fatalf("got %q", v)
|
||||||
|
}
|
||||||
|
// A report deleted from the bucket goes whether its node is assigned or not.
|
||||||
|
key, _ = gateReport(t, "g14", "v26.12.1")
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, nil, true) {
|
||||||
|
t.Fatal("deleting g14's report passed nothing on")
|
||||||
|
}
|
||||||
|
if n := len(set.Entries("typescript")); n != 2 {
|
||||||
|
t.Fatalf("novox and shanks are left; got %d entries", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A newly assigned build node that has not reported yet changes nothing; its report counts as it comes.
|
||||||
|
func TestANewlyAssignedNodeCountsFromItsReport(t *testing.T) {
|
||||||
|
set := &buildrun.ToolchainSet{}
|
||||||
|
gate := newToolchainGate([]string{buildrun.NodeBuildModule}, set.Put, func(string, ...any) {})
|
||||||
|
gate.Settled()
|
||||||
|
if gate.Assignment("g14.mesh-build-node", []byte(`{}`), false) {
|
||||||
|
t.Fatal("an assignment with no report passed something on")
|
||||||
|
}
|
||||||
|
key, value := gateReport(t, "g14", "v26.12.1")
|
||||||
|
if !gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("an assigned node's report was not passed on")
|
||||||
|
}
|
||||||
|
key, value = gateReport(t, "ace", "v26.12.0")
|
||||||
|
if gate.Toolchain(buildrun.NodeBuildModule, key, value, false) {
|
||||||
|
t.Fatal("a report of a node with no assignment key was passed on after the assignments settled")
|
||||||
|
}
|
||||||
|
if v, none := agreedOf(set); v != "v26.12.1" || none != "" {
|
||||||
|
t.Fatalf("got %q, %q", v, none)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -24,9 +24,11 @@ var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO"
|
|||||||
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
|
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
|
||||||
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
|
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
|
||||||
type toolchainWatch struct {
|
type toolchainWatch struct {
|
||||||
ctx context.Context
|
ctx context.Context
|
||||||
js bucketOpener
|
js bucketOpener
|
||||||
set *buildrun.ToolchainSet
|
set *buildrun.ToolchainSet
|
||||||
|
// put takes a change of a toolchains bucket in: set.Put, or the gate of the assigned build nodes before it (issue 1940).
|
||||||
|
put func(module, key string, value []byte, deleted bool) bool
|
||||||
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
|
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
|
||||||
logger *log.Logger
|
logger *log.Logger
|
||||||
errs chan error
|
errs chan error
|
||||||
@@ -35,7 +37,7 @@ type toolchainWatch struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
|
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
|
||||||
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
|
return &toolchainWatch{ctx: ctx, js: js, set: set, put: set.Put, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
|
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
|
||||||
@@ -63,14 +65,14 @@ func (w *toolchainWatch) ensure(module string) error {
|
|||||||
case err != nil:
|
case err != nil:
|
||||||
return fmt.Errorf("the bucket %s: %w", name, err)
|
return fmt.Errorf("the bucket %s: %w", name, err)
|
||||||
}
|
}
|
||||||
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) })
|
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.put(module, e.Key(), e.Value(), !opOf(e)) })
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
w.watched[module] = true
|
w.watched[module] = true
|
||||||
go func() {
|
go func() {
|
||||||
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
|
||||||
if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) {
|
if w.put(module, e.Key(), e.Value(), !opOf(e)) {
|
||||||
w.changed(w.ctx)
|
w.changed(w.ctx)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -1,3 +0,0 @@
|
|||||||
{
|
|
||||||
"capacity": ""
|
|
||||||
}
|
|
||||||
@@ -3,7 +3,7 @@ module mesh-delivery-checker
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93 h1:FAiFY2KBKEoZSH49OZTt7/zRBIQxyl5g/KhYrpggbN4=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023213200-403a5d72ab93/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -0,0 +1,141 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrAssignerNotRunning is what a wait for a composition answers when no instance of the assigner is online: nobody is
|
||||||
|
// making it, so the check does not wait for it (issue 1297). The verdict says the composition is not judged, and when the
|
||||||
|
// assigner starts and writes it, the checker takes it in then (Checker.Composed).
|
||||||
|
var ErrAssignerNotRunning = errors.New(delivery.AssignerNotRunning)
|
||||||
|
|
||||||
|
// PresenceEvent is a change of the watchman's presence bucket, or Settled once what stands was handed.
|
||||||
|
type PresenceEvent struct {
|
||||||
|
Key string
|
||||||
|
Online bool
|
||||||
|
Settled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompositionEvent is a change of the composition's key (its value, nil for a delete), or Settled once what stands was
|
||||||
|
// handed.
|
||||||
|
type CompositionEvent struct {
|
||||||
|
Value []byte
|
||||||
|
Settled bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// AwaitComposition answers the assigner's composition once one is written that does not wait, while an instance of the
|
||||||
|
// assigner is online by the watchman's presence (`part.mesh-delivery-assigner.<instance>`). Once both watches handed
|
||||||
|
// what stands, no instance online is ErrAssignerNotRunning at once, and so is the last instance going offline during the
|
||||||
|
// wait. No timer: every answer follows an event; ctx ending is ctx's error.
|
||||||
|
func AwaitComposition(ctx context.Context, presence <-chan PresenceEvent, comps <-chan CompositionEvent) (delivery.CompositionVerdict, error) {
|
||||||
|
online := map[string]bool{}
|
||||||
|
presenceSettled, compSettled := false, false
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return delivery.CompositionVerdict{}, ctx.Err()
|
||||||
|
case e, ok := <-presence:
|
||||||
|
if !ok {
|
||||||
|
return delivery.CompositionVerdict{}, errors.New("the watch of the assigner's presence ended")
|
||||||
|
}
|
||||||
|
if e.Settled {
|
||||||
|
presenceSettled = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if _, part, instance, err := delivery.ParsePresenceKey(e.Key); err == nil && part == delivery.AssignerModule {
|
||||||
|
if e.Online {
|
||||||
|
online[instance] = true
|
||||||
|
} else {
|
||||||
|
delete(online, instance)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case e, ok := <-comps:
|
||||||
|
if !ok {
|
||||||
|
return delivery.CompositionVerdict{}, errors.New("the watch of the composition ended")
|
||||||
|
}
|
||||||
|
if e.Settled {
|
||||||
|
compSettled = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if e.Value == nil {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
v, err := delivery.DecodeCompositionVerdict(e.Value)
|
||||||
|
if err != nil {
|
||||||
|
return delivery.CompositionVerdict{}, fmt.Errorf("the composition cannot be read: %w", err)
|
||||||
|
}
|
||||||
|
if v.State != delivery.CompositionWaits {
|
||||||
|
return v, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if presenceSettled && compSettled && len(online) == 0 {
|
||||||
|
return delivery.CompositionVerdict{}, ErrAssignerNotRunning
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// withComposition is v with the assigner's composition taken in: a failed one fails the check, naming its machine.
|
||||||
|
func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) {
|
||||||
|
v.Composition = comp.Composition()
|
||||||
|
v.Took.Composition = comp.Took
|
||||||
|
if comp.State == delivery.CompositionFail {
|
||||||
|
machine := comp.Machine
|
||||||
|
if machine == "" {
|
||||||
|
machine = "the composition"
|
||||||
|
}
|
||||||
|
v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: machine, Words: comp.Words}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// notJudged says whether a verdict's composition was not judged because the assigner was not running.
|
||||||
|
func notJudged(v delivery.Verdict) bool {
|
||||||
|
return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning
|
||||||
|
}
|
||||||
|
|
||||||
|
// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the
|
||||||
|
// assigner was not running (the assigner started since): the verdict and its status are written again with it. A
|
||||||
|
// composition of heads no candidate holds now, or of a verdict that judged its composition, is left.
|
||||||
|
func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) {
|
||||||
|
ch.init()
|
||||||
|
if comp.State == delivery.CompositionWaits {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ckey, err := delivery.CandidatesKey(comp.Group)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ch.mu.Lock()
|
||||||
|
c, ok := ch.known[ckey]
|
||||||
|
ch.mu.Unlock()
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cs := CaseOf(c, ch.heads())
|
||||||
|
if cs == nil || cs.HeadsHash != comp.Heads {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cur, rev, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||||
|
if err != nil || !found {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
v, err := delivery.DecodeVerdict(cur)
|
||||||
|
if err != nil || v.State != delivery.VerdictPass || !notJudged(v) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
withComposition(&v, comp)
|
||||||
|
data, err := delivery.EncodeVerdict(v)
|
||||||
|
if err != nil {
|
||||||
|
ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, rev); err != nil {
|
||||||
|
ch.logf("%s: the verdict with its composition could not be written: %v", cs.Key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
ch.writeOwed(ctx, cs, v, Outcome{Verdict: v})
|
||||||
|
ch.logf("%s: the composition came after the verdict: %s", cs.Key, comp.State)
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
func assignerOn(instance string, online bool) PresenceEvent {
|
||||||
|
key, _ := delivery.PresencePartKey(delivery.AssignerModule, instance)
|
||||||
|
return PresenceEvent{Key: key, Online: online}
|
||||||
|
}
|
||||||
|
|
||||||
|
func compEvent(t *testing.T, v delivery.CompositionVerdict) CompositionEvent {
|
||||||
|
t.Helper()
|
||||||
|
data, err := delivery.EncodeCompositionVerdict(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return CompositionEvent{Value: data}
|
||||||
|
}
|
||||||
|
|
||||||
|
var passed = delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}}
|
||||||
|
|
||||||
|
// 1297/B3d: with no assigner online, the wait answers ErrAssignerNotRunning as soon as both watches handed what stands.
|
||||||
|
func TestNoAssignerAnswersAtOnce(t *testing.T) {
|
||||||
|
presence, comps := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4)
|
||||||
|
presence <- assignerOn("novox", false)
|
||||||
|
presence <- PresenceEvent{Settled: true}
|
||||||
|
comps <- CompositionEvent{Settled: true}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||||
|
defer cancel()
|
||||||
|
if _, err := AwaitComposition(ctx, presence, comps); !errors.Is(err, ErrAssignerNotRunning) {
|
||||||
|
t.Fatalf("answered %v, want the assigner not running", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3d: with the assigner online it waits for the key, past a composition that waits, and answers the verdict.
|
||||||
|
func TestAnOnlineAssignerIsWaitedFor(t *testing.T) {
|
||||||
|
presence, comps := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4)
|
||||||
|
presence <- assignerOn("novox", true)
|
||||||
|
presence <- PresenceEvent{Settled: true}
|
||||||
|
comps <- CompositionEvent{Settled: true}
|
||||||
|
done := make(chan error, 1)
|
||||||
|
var got delivery.CompositionVerdict
|
||||||
|
go func() {
|
||||||
|
var err error
|
||||||
|
got, err = AwaitComposition(context.Background(), presence, comps)
|
||||||
|
done <- err
|
||||||
|
}()
|
||||||
|
comps <- compEvent(t, delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "w"})
|
||||||
|
select {
|
||||||
|
case err := <-done:
|
||||||
|
t.Fatalf("answered before the composition was written: %v", err)
|
||||||
|
case <-time.After(50 * time.Millisecond):
|
||||||
|
}
|
||||||
|
comps <- compEvent(t, passed)
|
||||||
|
if err := <-done; err != nil || got.State != delivery.CompositionPass {
|
||||||
|
t.Fatalf("answered %+v, %v", got, err)
|
||||||
|
}
|
||||||
|
// And the last instance going offline during a wait ends it as not running.
|
||||||
|
presence2, comps2 := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4)
|
||||||
|
presence2 <- assignerOn("novox", true)
|
||||||
|
presence2 <- PresenceEvent{Settled: true}
|
||||||
|
comps2 <- CompositionEvent{Settled: true}
|
||||||
|
presence2 <- assignerOn("novox", false)
|
||||||
|
if _, err := AwaitComposition(context.Background(), presence2, comps2); !errors.Is(err, ErrAssignerNotRunning) {
|
||||||
|
t.Errorf("the assigner going offline answered %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3d: a passing check with no assigner says at once, loud, that its composition is not judged; never a pass of it.
|
||||||
|
func TestAPassingCheckWithNoAssignerSaysTheCompositionIsNotJudged(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.ex.Compositions = &fakeCompositions{err: ErrAssignerNotRunning}
|
||||||
|
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || v.Composition.Words != delivery.AssignerNotRunning {
|
||||||
|
t.Fatalf("composition = %+v", v.Composition)
|
||||||
|
}
|
||||||
|
if words := compositionWords(v.Composition, 0); words != "composition not judged: the assigner is not running" {
|
||||||
|
t.Errorf("the line says %q", words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3d: the assigner starting writes the composition later, and the checker takes it into the verdict that said it
|
||||||
|
// was not judged, writing the status again; a failed one fails the check.
|
||||||
|
func TestTheAssignerStartingBringsTheComposition(t *testing.T) {
|
||||||
|
for _, comp := range []delivery.CompositionVerdict{passed,
|
||||||
|
{Group: "feat/x", Heads: "", State: delivery.CompositionFail, SDK: "x", Machine: "g14", Words: "g14 runs web, which the change removes"}} {
|
||||||
|
ch, cs := checkerWithNotJudged(t)
|
||||||
|
comp.Heads = cs.HeadsHash
|
||||||
|
ch.Composed(context.Background(), comp)
|
||||||
|
raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key)
|
||||||
|
v, err := delivery.DecodeVerdict(raw)
|
||||||
|
if err != nil || v.Composition == nil || v.Composition.State != comp.State {
|
||||||
|
t.Fatalf("after the composition came, the verdict is %+v, %v", v, err)
|
||||||
|
}
|
||||||
|
if comp.State == delivery.CompositionFail && (v.State != delivery.VerdictFail || v.Failed.Machine != "g14") {
|
||||||
|
t.Errorf("a failed composition left the verdict %+v", v)
|
||||||
|
}
|
||||||
|
owed := ch.Owed.(*memKV)
|
||||||
|
keys, _ := owed.Keys(context.Background())
|
||||||
|
if len(keys) == 0 {
|
||||||
|
t.Fatal("no status was written again")
|
||||||
|
}
|
||||||
|
data, _, _, _ := owed.Get(context.Background(), keys[0])
|
||||||
|
o, _ := delivery.DecodeCheckerOwed(data)
|
||||||
|
if !strings.Contains(o.Text, "composition passed") && !strings.Contains(o.Text, "g14 runs web") {
|
||||||
|
t.Errorf("the status's line says %q", o.Text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged.
|
||||||
|
func checkerWithNotJudged(t *testing.T) (*Checker, *Case) {
|
||||||
|
t.Helper()
|
||||||
|
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
raw := candidateOf(t, cs.Group, pullOf(repoA, 7, head1))
|
||||||
|
ch := &Checker{Verdicts: newMemKV(), Owed: newMemKV(), Trunks: fixedTrunks{cs.Trunk}}
|
||||||
|
ckey, _ := delivery.CandidatesKey(cs.Group)
|
||||||
|
ch.init()
|
||||||
|
c, err := delivery.DecodeCandidate(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
ch.known[ckey] = c
|
||||||
|
if got := CaseOf(c, ch.heads()); got == nil || got.Key != cs.Key {
|
||||||
|
t.Fatalf("the case is %+v, want %s", got, cs.Key)
|
||||||
|
}
|
||||||
|
v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, State: delivery.VerdictPass,
|
||||||
|
Levels: map[string]delivery.LevelResult{delivery.LevelUnit: {Run: 1}, delivery.LevelIntegration: {Run: 1}},
|
||||||
|
Composition: &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}}
|
||||||
|
data, err := delivery.EncodeVerdict(v)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := ch.Verdicts.Create(context.Background(), cs.Key, data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return ch, cs
|
||||||
|
}
|
||||||
|
|
||||||
|
type fixedTrunks struct{ heads map[string]string }
|
||||||
|
|
||||||
|
func (f fixedTrunks) Heads() map[string]string { return f.heads }
|
||||||
@@ -26,7 +26,7 @@ const (
|
|||||||
// checks/deps). Which commit of each is besideCommit's rule.
|
// checks/deps). Which commit of each is besideCommit's rule.
|
||||||
var besideRules = map[string][]string{
|
var besideRules = map[string][]string{
|
||||||
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
|
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
|
||||||
nodeEngineRepo: {catalogueRepository, SDKRepository},
|
nodeEngineRepo: {catalogueRepository, SDKRepository, controllerRepo},
|
||||||
SDKRepository: {catalogueRepository, nodeEngineRepo},
|
SDKRepository: {catalogueRepository, nodeEngineRepo},
|
||||||
catalogueRepository: {SDKRepository, controllerRepo},
|
catalogueRepository: {SDKRepository, controllerRepo},
|
||||||
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
|
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
|
||||||
@@ -35,13 +35,18 @@ var besideRules = map[string][]string{
|
|||||||
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the catalogue at its trunk, always, even in a
|
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the catalogue at its trunk, always, even in a
|
||||||
// group holding a catalogue pull request; the node-engine at the commit the mesh runs (the commit of the indexer's module
|
// group holding a catalogue pull request; the node-engine at the commit the mesh runs (the commit of the indexer's module
|
||||||
// mesh-host, its trunk head when none is known); mesh-sdk, for the controller's tests, at the commit the controller's go.mod
|
// mesh-host, its trunk head when none is known); mesh-sdk, for the controller's tests, at the commit the controller's go.mod
|
||||||
// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk.
|
// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk. And
|
||||||
|
// first, the change's own repository at the trunk it is merged onto, so a test comparing a shrink-only list with the
|
||||||
|
// trunk's copy reads it at $MESH_CHECK_BESIDE/<name> (issue 1887 A12, ADR 0529 §2).
|
||||||
func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef, error) {
|
func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef, error) {
|
||||||
var heads map[string]string
|
var heads map[string]string
|
||||||
if j.Trunks != nil {
|
if j.Trunks != nil {
|
||||||
heads = j.Trunks.Heads()
|
heads = j.Trunks.Heads()
|
||||||
}
|
}
|
||||||
var out []BesideRef
|
var out []BesideRef
|
||||||
|
if p.Trunk != "" {
|
||||||
|
out = append(out, BesideRef{Repository: p.Repository, Commit: p.Trunk})
|
||||||
|
}
|
||||||
for _, repo := range besideRules[p.Repository] {
|
for _, repo := range besideRules[p.Repository] {
|
||||||
switch {
|
switch {
|
||||||
case repo == catalogueRepository:
|
case repo == catalogueRepository:
|
||||||
|
|||||||
@@ -11,13 +11,6 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// HQRepository is the repository the dark checker writes no status for (ADR 0484 clause 8): hq builds no module and its
|
|
||||||
// check is a script no build module can run, so it is judged by its merge-check.sh until issue 1296 settles how.
|
|
||||||
const HQRepository = "novox/hq"
|
|
||||||
|
|
||||||
// HQNote is what the read tools say of an hq pull request (ADR 0484 clause 8).
|
|
||||||
const HQNote = "hq: judged by merge-check.sh until issue 1296"
|
|
||||||
|
|
||||||
// Pull is one pull request the checker judges: its head, the trunk head of its repository it is merged onto and the
|
// Pull is one pull request the checker judges: its head, the trunk head of its repository it is merged onto and the
|
||||||
// modules the indexer says it touches, with the fingerprints the indexer wrote for them.
|
// modules the indexer says it touches, with the fingerprints the indexer wrote for them.
|
||||||
type Pull struct {
|
type Pull struct {
|
||||||
@@ -29,8 +22,8 @@ type Pull struct {
|
|||||||
Fps map[string]delivery.CandidateModule
|
Fps map[string]delivery.CandidateModule
|
||||||
}
|
}
|
||||||
|
|
||||||
// Case is one check: a candidate's pull requests that are judged (those of hq and of a repository whose trunk is not known
|
// Case is one check: a candidate's pull requests that are judged (those of a repository whose trunk is not known are left
|
||||||
// are left out), the trunk head of each repository and the key of the verdict.
|
// out; hq is judged as any repository since R4g, its checks Go tests: ADR 0527), the trunk head of each repository and the key of the verdict.
|
||||||
type Case struct {
|
type Case struct {
|
||||||
// Key is the verdict key, `<group>.<heads hash>`.
|
// Key is the verdict key, `<group>.<heads hash>`.
|
||||||
Key string
|
Key string
|
||||||
@@ -41,7 +34,7 @@ type Case struct {
|
|||||||
Pulls []Pull
|
Pulls []Pull
|
||||||
// Trunk is the trunk head of each repository of Pulls.
|
// Trunk is the trunk head of each repository of Pulls.
|
||||||
Trunk map[string]string
|
Trunk map[string]string
|
||||||
// Left says why pull requests of the candidate are not judged (hq's, a repository with no known trunk); empty when all are.
|
// Left says why pull requests of the candidate are not judged (a repository with no known trunk); empty when all are.
|
||||||
Left []string
|
Left []string
|
||||||
// Queued is when the checker took the case up, for the time of its queue.
|
// Queued is when the checker took the case up, for the time of its queue.
|
||||||
Queued time.Time
|
Queued time.Time
|
||||||
@@ -49,6 +42,9 @@ type Case struct {
|
|||||||
Try *TryOptions
|
Try *TryOptions
|
||||||
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
|
||||||
Progress *Progress
|
Progress *Progress
|
||||||
|
// Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build
|
||||||
|
// seats' recheck stream, every other check on check (ADR 0536 §4).
|
||||||
|
Kind string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Progress is the step a check is at, written by the check and read when its bound passes.
|
// Progress is the step a check is at, written by the check and read when its bound passes.
|
||||||
@@ -83,8 +79,6 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
|
|||||||
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}}
|
||||||
for _, p := range c.Pulls {
|
for _, p := range c.Pulls {
|
||||||
switch head, known := trunk[p.Repository]; {
|
switch head, known := trunk[p.Repository]; {
|
||||||
case p.Repository == HQRepository:
|
|
||||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, HQNote))
|
|
||||||
case !known || head == "":
|
case !known || head == "":
|
||||||
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
|
||||||
default:
|
default:
|
||||||
|
|||||||
@@ -11,12 +11,11 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678).
|
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678):
|
||||||
const DefaultBound = 30 * time.Minute
|
// none since R4g. A check's asks wait in the build seats' streams as long as the backlog is (ADR 0525 §9), and each ask's
|
||||||
|
// step is bounded from its start on the node that took it (buildrun.Asker.Bound, DefaultStepBound); a bound set by
|
||||||
// NodesWords is in the Words of a failure that is the build nodes' (none qualifies, the toolchains disagree): such a verdict
|
// MESH_CHECKER_BOUND still ends a check that ran past it.
|
||||||
// is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8).
|
const DefaultBound time.Duration = 0
|
||||||
const NodesWords = "build node qualifies"
|
|
||||||
|
|
||||||
// UnheardWords end the Words of a failure because a part of the mesh the check reads was not heard (the forge, issue 1624):
|
// UnheardWords end the Words of a failure because a part of the mesh the check reads was not heard (the forge, issue 1624):
|
||||||
// like a failure of the nodes, it is not the change's, and is judged again when the nodes change.
|
// like a failure of the nodes, it is not the change's, and is judged again when the nodes change.
|
||||||
@@ -40,6 +39,9 @@ type Outcome struct {
|
|||||||
Builds []BuildNote
|
Builds []BuildNote
|
||||||
// Slowest names the slowest tests of a level that went over its bound ("unit pkg.Test 1.2s").
|
// Slowest names the slowest tests of a level that went over its bound ("unit pkg.Test 1.2s").
|
||||||
Slowest []string
|
Slowest []string
|
||||||
|
// Judged says, by pull request (pullKey), which program judged a change to a build module (ADR 0535 §5); a pull request
|
||||||
|
// that changes none is not named.
|
||||||
|
Judged map[string]string
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
|
||||||
@@ -63,12 +65,8 @@ type Checker struct {
|
|||||||
Owed KV
|
Owed KV
|
||||||
Judge Judge
|
Judge Judge
|
||||||
Trunks interface{ Heads() map[string]string }
|
Trunks interface{ Heads() map[string]string }
|
||||||
// Bound is how long one check may run; DefaultBound when 0.
|
// Bound is how long one check may run; none when 0 (DefaultBound).
|
||||||
Bound time.Duration
|
Bound time.Duration
|
||||||
// Limit is how many checks run at once; two when 0.
|
|
||||||
Limit int
|
|
||||||
// Hold says whether a delivery builds, so no new check starts (ADR 0484 clause 12); nil never holds.
|
|
||||||
Hold func() bool
|
|
||||||
// Link is the base of the address a status points at; the verdict key follows.
|
// Link is the base of the address a status points at; the verdict key follows.
|
||||||
Link string
|
Link string
|
||||||
// Announce says a status was written (EventOwedWritten); nil says nothing.
|
// Announce says a status was written (EventOwedWritten); nil says nothing.
|
||||||
@@ -77,7 +75,6 @@ type Checker struct {
|
|||||||
Logf func(format string, args ...any)
|
Logf func(format string, args ...any)
|
||||||
|
|
||||||
once sync.Once
|
once sync.Once
|
||||||
sem chan struct{}
|
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
known map[string]delivery.Candidate // by candidates key
|
known map[string]delivery.Candidate // by candidates key
|
||||||
@@ -90,11 +87,6 @@ type Checker struct {
|
|||||||
|
|
||||||
func (ch *Checker) init() {
|
func (ch *Checker) init() {
|
||||||
ch.once.Do(func() {
|
ch.once.Do(func() {
|
||||||
limit := ch.Limit
|
|
||||||
if limit <= 0 {
|
|
||||||
limit = 2
|
|
||||||
}
|
|
||||||
ch.sem = make(chan struct{}, limit)
|
|
||||||
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
|
||||||
ch.nodesFail = map[string]uint64{}
|
ch.nodesFail = map[string]uint64{}
|
||||||
})
|
})
|
||||||
@@ -181,11 +173,7 @@ func (ch *Checker) heads() map[string]string {
|
|||||||
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
|
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
|
||||||
cs := CaseOf(c, ch.heads())
|
cs := CaseOf(c, ch.heads())
|
||||||
if cs == nil {
|
if cs == nil {
|
||||||
return // nothing to judge: hq's pull requests, or no trunk known
|
return // nothing to judge: no trunk known
|
||||||
}
|
|
||||||
if ch.Hold != nil && ch.Hold() {
|
|
||||||
ch.logf("%s: not started, a delivery builds", cs.Key)
|
|
||||||
return // started when the plan bucket empties (Reconsider)
|
|
||||||
}
|
}
|
||||||
cs.Queued = ch.now()
|
cs.Queued = ch.now()
|
||||||
ch.mu.Lock()
|
ch.mu.Lock()
|
||||||
@@ -199,9 +187,11 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
|||||||
ch.wg.Add(1)
|
ch.wg.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer ch.wg.Done()
|
defer ch.wg.Done()
|
||||||
ch.sem <- struct{}{}
|
|
||||||
defer func() { <-ch.sem }()
|
|
||||||
for {
|
for {
|
||||||
|
// Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a
|
||||||
|
// failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9;
|
||||||
|
// ADR 0536 §4). The checker holds no place and no capacity of its own.
|
||||||
|
cs.Kind = ch.kindOf(ctx, cs)
|
||||||
ch.try(ctx, key, cs)
|
ch.try(ctx, key, cs)
|
||||||
ch.mu.Lock()
|
ch.mu.Lock()
|
||||||
more := ch.again[cs.Key]
|
more := ch.again[cs.Key]
|
||||||
@@ -215,10 +205,37 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat
|
|||||||
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
|
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
|
||||||
cs = next
|
cs = next
|
||||||
}
|
}
|
||||||
|
cs.Queued = ch.now()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on
|
||||||
|
// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after.
|
||||||
|
const (
|
||||||
|
KindNew = "new"
|
||||||
|
KindNodes = "nodes"
|
||||||
|
KindTrunk = "trunk"
|
||||||
|
)
|
||||||
|
|
||||||
|
// kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a
|
||||||
|
// failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk
|
||||||
|
// move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do.
|
||||||
|
func (ch *Checker) kindOf(ctx context.Context, cs *Case) string {
|
||||||
|
value, _, found, err := ch.Verdicts.Get(ctx, cs.Key)
|
||||||
|
if err != nil || !found {
|
||||||
|
return KindNew
|
||||||
|
}
|
||||||
|
v, err := delivery.DecodeVerdict(value)
|
||||||
|
switch {
|
||||||
|
case err != nil || v.State == delivery.VerdictRunning:
|
||||||
|
return KindNew
|
||||||
|
case IsNodesFailure(v):
|
||||||
|
return KindNodes
|
||||||
|
}
|
||||||
|
return KindTrunk
|
||||||
|
}
|
||||||
|
|
||||||
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
|
||||||
ch.mu.Lock()
|
ch.mu.Lock()
|
||||||
defer ch.mu.Unlock()
|
defer ch.mu.Unlock()
|
||||||
@@ -289,9 +306,11 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64
|
|||||||
return &claimed{rev}, nil
|
return &claimed{rev}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// IsNodesFailure says whether a verdict failed because no build node could take the check.
|
// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords):
|
||||||
|
// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build
|
||||||
|
// seats' streams (ADR 0525 §6, issue 1912).
|
||||||
func IsNodesFailure(v delivery.Verdict) bool {
|
func IsNodesFailure(v delivery.Verdict) bool {
|
||||||
return v.State == delivery.VerdictFail && (strings.Contains(v.Failed.Words, NodesWords) || strings.Contains(v.Failed.Words, UnheardWords))
|
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords)
|
||||||
}
|
}
|
||||||
|
|
||||||
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
|
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
|
||||||
@@ -311,7 +330,7 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
began := ch.now()
|
began := ch.now()
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
|
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
out := ch.judge(ctx, cs)
|
out := ch.judge(ctx, cs)
|
||||||
if ctx.Err() != nil {
|
if ctx.Err() != nil {
|
||||||
return
|
return
|
||||||
@@ -342,11 +361,11 @@ func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
|
|||||||
if ch.movedSince(cs) {
|
if ch.movedSince(cs) {
|
||||||
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
// Judged on a trunk that moved while it ran: the verdict is kept as judged, and the status stays pending until the
|
||||||
// check of the new trunk, which Reconsider starts, ends.
|
// check of the new trunk, which Reconsider starts, ends.
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
|
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
|
ch.logf("%s: %s on a trunk that moved while it ran; the status stays pending", cs.Key, v.State)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
ch.writeOwed(ctx, cs, v, out.Builds, out.Slowest)
|
ch.writeOwed(ctx, cs, v, out)
|
||||||
ch.logf("%s: %s", cs.Key, v.State)
|
ch.logf("%s: %s", cs.Key, v.State)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -366,7 +385,7 @@ func failure(machine, words string) delivery.Verdict {
|
|||||||
// writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success
|
// writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success
|
||||||
// or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge
|
// or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge
|
||||||
// performs one act for each change; an older head's key is never touched.
|
// performs one act for each change; an older head's key is never touched.
|
||||||
func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, builds []BuildNote, slowest []string) {
|
func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, out Outcome) {
|
||||||
for _, p := range cs.Pulls {
|
for _, p := range cs.Pulls {
|
||||||
key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head)
|
key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -375,7 +394,7 @@ func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict,
|
|||||||
}
|
}
|
||||||
state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State]
|
state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State]
|
||||||
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state,
|
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state,
|
||||||
Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, builds, slowest)}
|
Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, out.Builds, out.Slowest, out.Judged)}
|
||||||
data, err := delivery.EncodeCheckerOwed(o)
|
data, err := delivery.EncodeCheckerOwed(o)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err)
|
ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err)
|
||||||
@@ -486,7 +505,7 @@ func (ch *Checker) TrunkMoved(ctx context.Context) {
|
|||||||
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
|
if v, err := delivery.DecodeVerdict(value); err == nil && sameTrunk(v.Trunk, cs.Trunk) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
|
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, Outcome{})
|
||||||
}
|
}
|
||||||
ch.Reconsider(ctx)
|
ch.Reconsider(ctx)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,6 +11,8 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const hqRepo = "novox/hq"
|
||||||
|
|
||||||
const (
|
const (
|
||||||
repoA = "novox/mesh-delivery"
|
repoA = "novox/mesh-delivery"
|
||||||
repoSDK = "novox/mesh-sdk"
|
repoSDK = "novox/mesh-sdk"
|
||||||
@@ -206,11 +208,13 @@ func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNoQualifyingNodeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4
|
// A check whose forge was not heard is judged again when the nodes change (issue 1912: no check fails for want of a build
|
||||||
|
// node any more; its asks wait in the streams).
|
||||||
|
func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
|
j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
|
||||||
if n == 1 {
|
if n == 1 {
|
||||||
return Outcome{Verdict: failure("shanks", "no go "+NodesWords+": none is online")}, nil
|
return Outcome{Verdict: failure("the forge", "novox/mesh-delivery#7: go.mod cannot be read: no responders (the forge was "+UnheardWords+")")}, nil
|
||||||
}
|
}
|
||||||
return Outcome{Verdict: passing()}, nil
|
return Outcome{Verdict: passing()}, nil
|
||||||
}}
|
}}
|
||||||
@@ -234,10 +238,9 @@ func TestNoQualifyingNodeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestHqAndARepositoryWithNoTrunkGetNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clauses 8 and 11
|
func TestARepositoryWithNoTrunkGetsNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clause 11
|
||||||
ctx, j := context.Background(), passJudge()
|
ctx, j := context.Background(), passJudge()
|
||||||
r := newRig(t, j)
|
r := newRig(t, j)
|
||||||
r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(HQRepository, 3, head1)))
|
|
||||||
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
|
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
|
||||||
r.ch.Wait()
|
r.ch.Wait()
|
||||||
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
|
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
|
||||||
@@ -245,17 +248,24 @@ func TestHqAndARepositoryWithNoTrunkGetNoVerdict(t *testing.T) { // N6/AC6, ADR
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAGroupWithHqJudgesItsOtherMembersAndWritesNoStatusForHq(t *testing.T) { // ADR 0484 clause 8
|
// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a
|
||||||
|
// mesh/check; in a group, hq's member is judged with the others.
|
||||||
|
func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) {
|
||||||
ctx, j := context.Background(), passJudge()
|
ctx, j := context.Background(), passJudge()
|
||||||
r := newRig(t, j)
|
r := newRig(t, j)
|
||||||
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(HQRepository, 3, head1), pullOf(repoA, 7, head2)))
|
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA2})
|
||||||
|
r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(hqRepo, 3, head1)))
|
||||||
r.ch.Wait()
|
r.ch.Wait()
|
||||||
if j.count() != 1 {
|
if j.count() != 1 || len(r.verdicts.vals) != 1 {
|
||||||
t.Fatalf("judged %d times, want once", j.count())
|
t.Fatalf("judged %d times, %d verdicts, want one", j.count(), len(r.verdicts.vals))
|
||||||
}
|
}
|
||||||
keys, _ := r.owed.Keys(ctx)
|
if s := owedState(t, r, hqRepo, 3, head1); s != "success" {
|
||||||
if len(keys) != 1 || !strings.HasPrefix(keys[0], "novox/mesh-delivery.7.") && !strings.Contains(keys[0], "mesh-delivery") {
|
t.Errorf("hq#3 owes %q, want success", s)
|
||||||
t.Errorf("statuses %v, want one for mesh-delivery#7 only", keys)
|
}
|
||||||
|
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(hqRepo, 4, head2), pullOf(repoA, 7, head2)))
|
||||||
|
r.ch.Wait()
|
||||||
|
if s := owedState(t, r, hqRepo, 4, head2); s != "success" {
|
||||||
|
t.Errorf("hq#4 in a group owes %q, want success", s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -298,24 +308,6 @@ func TestATrunkMoveRewritesTheSameVerdictKeyAndTheHeadsStatus(t *testing.T) { //
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1
|
|
||||||
ctx, j := context.Background(), passJudge()
|
|
||||||
r := newRig(t, j)
|
|
||||||
held := true
|
|
||||||
r.ch.Hold = func() bool { return held }
|
|
||||||
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
|
||||||
r.ch.Wait()
|
|
||||||
if j.count() != 0 {
|
|
||||||
t.Fatalf("judged %d times while held", j.count())
|
|
||||||
}
|
|
||||||
held = false
|
|
||||||
r.ch.Reconsider(ctx)
|
|
||||||
r.ch.Wait()
|
|
||||||
if j.count() != 1 {
|
|
||||||
t.Fatalf("judged %d times after the hold ended, want once", j.count())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part)
|
func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part)
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
var seen []string
|
var seen []string
|
||||||
|
|||||||
@@ -0,0 +1,80 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// fakeCompositions answers the assigner's composition of a check from memory, recording the keys asked.
|
||||||
|
type fakeCompositions struct {
|
||||||
|
v delivery.CompositionVerdict
|
||||||
|
err error
|
||||||
|
asked []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) {
|
||||||
|
f.asked = append(f.asked, key)
|
||||||
|
return f.v, f.err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/AC3 (its unit half): a check whose tests pass takes the assigner's composition of it, by the check's own key,
|
||||||
|
// into its verdict: a pass says it passed with the mesh-sdk and machines it composed, and is no longer "not run".
|
||||||
|
func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}}
|
||||||
|
b.ex.Compositions = f
|
||||||
|
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
v := b.judge(t, cs).Verdict
|
||||||
|
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 {
|
||||||
|
t.Fatalf("verdict = %+v, composition %+v", v, v.Composition)
|
||||||
|
}
|
||||||
|
if len(f.asked) != 1 || f.asked[0] != cs.Key {
|
||||||
|
t.Errorf("asked %v, want the check's key %s", f.asked, cs.Key)
|
||||||
|
}
|
||||||
|
if !compositionMentions(v, "passed") {
|
||||||
|
t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func compositionMentions(v delivery.Verdict, word string) bool {
|
||||||
|
return strings.Contains(compositionWords(v.Composition, v.Took.Composition), word)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/AC2 at the checker: a failed composition fails the check, naming the machine it failed at and its words.
|
||||||
|
func TestAFailedCompositionFailsTheCheck(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.ex.Compositions = &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionFail, SDK: "x",
|
||||||
|
Machine: "g14", Words: "g14 runs web, which the change removes from its source"}}
|
||||||
|
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if v.State != delivery.VerdictFail || v.Failed.Machine != "g14" || v.Composition == nil || v.Composition.State != delivery.CompositionFail {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
v.Group, v.Heads = "feat/x", "0123456789ab" // the checker names them when it writes the verdict
|
||||||
|
if _, err := delivery.EncodeVerdict(v); err != nil {
|
||||||
|
t.Errorf("the verdict cannot be written: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1297/B3d: a composition that cannot be read leaves it not run; a try is not composed.
|
||||||
|
func TestACompositionNotHadIsNotRun(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
f := &fakeCompositions{err: errors.New("the check's bound passed")}
|
||||||
|
b.ex.Compositions = f
|
||||||
|
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionNotRun {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
b2 := newBench(t)
|
||||||
|
f2 := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionFail, Words: "w", SDK: "x"}}
|
||||||
|
b2.ex.Compositions = f2
|
||||||
|
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
cs.Try = &TryOptions{}
|
||||||
|
b2.judge(t, cs)
|
||||||
|
if len(f2.asked) != 0 {
|
||||||
|
t.Errorf("a try asked for a composition: %v", f2.asked)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 1930/AC3: given a check ask whose stored end keeps the digest's packages as a list of {package, digest} (issue 1930),
|
||||||
|
// or one stored before it with the packages a map keyed by package path, when the same heads are checked a second time,
|
||||||
|
// then the kept verdicts the first check wrote are read from it: every package it passed is kept, none run again.
|
||||||
|
func TestKeptVerdictsAreReadFromTheDigestsListForm(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
old bool
|
||||||
|
}{{"the list form", false}, {"a map stored before issue 1930", true}} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
var stored []string
|
||||||
|
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||||
|
s := b.answer(a)
|
||||||
|
if tc.old && s.State == delivery.StepPassed {
|
||||||
|
s.Result = oldDigestForm(t, s.Result)
|
||||||
|
}
|
||||||
|
stored = append(stored, string(s.Result))
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
first := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
u := first.Levels[delivery.LevelUnit]
|
||||||
|
if first.State != delivery.VerdictPass || u.Run == 0 || u.Kept != 0 {
|
||||||
|
t.Fatalf("the first check = %+v", first)
|
||||||
|
}
|
||||||
|
if len(stored) == 0 {
|
||||||
|
t.Fatal("no ask's end was stored")
|
||||||
|
}
|
||||||
|
list := strings.Contains(stored[0], `"packages":[{"package":"fixture/slow"`)
|
||||||
|
if list == tc.old || strings.Contains(stored[0], `"fixture/slow":"`) != tc.old {
|
||||||
|
t.Fatalf("the stored end is not in the form under test: %s", stored[0])
|
||||||
|
}
|
||||||
|
b.steps.calls = nil
|
||||||
|
second := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if s := second.Levels[delivery.LevelUnit]; second.State != delivery.VerdictPass || s.Kept != u.Run || s.Run != 0 {
|
||||||
|
t.Fatalf("the second check's unit level = %+v, want %d kept and none run", s, u.Run)
|
||||||
|
}
|
||||||
|
if n := len(b.steps.tool("test")); n != 0 {
|
||||||
|
t.Errorf("%d test steps ran on the second check", n)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// oldDigestForm rewrites a check answer's level digests to the form stored before issue 1930: packages as a map keyed by
|
||||||
|
// package path.
|
||||||
|
func oldDigestForm(t *testing.T, result json.RawMessage) json.RawMessage {
|
||||||
|
t.Helper()
|
||||||
|
var a struct {
|
||||||
|
Fetch json.RawMessage `json:"fetch"`
|
||||||
|
Levels []map[string]any `json:"levels"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(result, &a); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, l := range a.Levels {
|
||||||
|
d, ok := l["digest"].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
m := map[string]any{}
|
||||||
|
list, _ := d["packages"].([]any)
|
||||||
|
for _, p := range list {
|
||||||
|
pd := p.(map[string]any)
|
||||||
|
m[pd["package"].(string)] = pd["digest"]
|
||||||
|
}
|
||||||
|
d["packages"] = m
|
||||||
|
}
|
||||||
|
raw, err := json.Marshal(a)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
@@ -108,6 +109,7 @@ const digD = "dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44"
|
|||||||
|
|
||||||
type bench struct {
|
type bench struct {
|
||||||
ex *Examiner
|
ex *Examiner
|
||||||
|
asks *buildrun.FakeAsks
|
||||||
steps *fakeSteps
|
steps *fakeSteps
|
||||||
kept *memKV
|
kept *memKV
|
||||||
prev *memKV
|
prev *memKV
|
||||||
@@ -116,6 +118,50 @@ type bench struct {
|
|||||||
test func(c stepCall) (any, error)
|
test func(c stepCall) (any, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them.
|
||||||
|
func refsOf(v any) []BesideRef {
|
||||||
|
raw, _ := json.Marshal(v)
|
||||||
|
var refs []BesideRef
|
||||||
|
_ = json.Unmarshal(raw, &refs)
|
||||||
|
return refs
|
||||||
|
}
|
||||||
|
|
||||||
|
// toolPrefix is the start of each build module's tool names, as the fake build modules answer them.
|
||||||
|
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}
|
||||||
|
|
||||||
|
// answer runs a check ask as the build module that pulls it does (buildstep.RunAsk), with the fake tools on shanks and the
|
||||||
|
// bench's kept verdicts, and answers the step it writes.
|
||||||
|
func (b *bench) answer(a delivery.StepAsk) delivery.Step {
|
||||||
|
module, _ := delivery.BuildModuleOf(a.Language)
|
||||||
|
const node = "shanks"
|
||||||
|
tool := func(name string) buildstep.StepFunc {
|
||||||
|
return func(ctx context.Context, args map[string]any) (any, error) {
|
||||||
|
raw, err := b.steps.Run(ctx, module, node, toolPrefix[module]+name, args)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return raw, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
steps := buildstep.AskSteps{Fetch: tool("fetch"), Test: tool("test"),
|
||||||
|
Kept: func(ctx context.Context, level, digest string) (int, bool, error) {
|
||||||
|
k, found, err := (Kept{Store: b.kept}).Lookup(ctx, level, digest)
|
||||||
|
return k.Tests, found, err
|
||||||
|
}}
|
||||||
|
if module == buildrun.GoBuildModule {
|
||||||
|
steps.Digest = tool("digest")
|
||||||
|
}
|
||||||
|
out, err := buildstep.RunAsk(context.Background(), steps, a)
|
||||||
|
began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||||
|
step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)}
|
||||||
|
if err != nil {
|
||||||
|
step.State, step.Reason = delivery.StepFailed, err.Error()
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
step.Result, _ = json.Marshal(out)
|
||||||
|
return step
|
||||||
|
}
|
||||||
|
|
||||||
// newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass
|
// newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass
|
||||||
// the captured slow fixture's result at both levels, on shanks.
|
// the captured slow fixture's result at both levels, on shanks.
|
||||||
func newBench(t testing.TB) *bench {
|
func newBench(t testing.TB) *bench {
|
||||||
@@ -137,12 +183,13 @@ func newBench(t testing.TB) *bench {
|
|||||||
b.steps.handlers = map[string]func(stepCall) (any, error){
|
b.steps.handlers = map[string]func(stepCall) (any, error){
|
||||||
"build_go_fetch": func(stepCall) (any, error) { return b.fetch, nil },
|
"build_go_fetch": func(stepCall) (any, error) { return b.fetch, nil },
|
||||||
"build_node_fetch": func(stepCall) (any, error) { return b.fetch, nil },
|
"build_node_fetch": func(stepCall) (any, error) { return b.fetch, nil },
|
||||||
"build_go_digest": func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil },
|
"build_go_digest": func(c stepCall) (any, error) { return echoBeside(b, b.digest[c.args["level"].(string)]), nil },
|
||||||
"build_go_test": func(c stepCall) (any, error) { return b.test(c) },
|
"build_go_test": func(c stepCall) (any, error) { return b.test(c) },
|
||||||
"build_node_test": func(c stepCall) (any, error) { return b.test(c) },
|
"build_node_test": func(c stepCall) (any, error) { return b.test(c) },
|
||||||
}
|
}
|
||||||
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
|
||||||
b.ex = &Examiner{Steps: b.steps, Toolchains: goEntries("shanks"), Places: &buildrun.Places{}, Kept: Kept{Store: b.kept}, Previews: b.prev,
|
b.asks = &buildrun.FakeAsks{Answer: b.answer}
|
||||||
|
b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev,
|
||||||
Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }}
|
Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }}
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
@@ -353,33 +400,6 @@ func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestNoQualifyingNodeIsAFailureOfTheNodes(t *testing.T) { // N6/AC4
|
|
||||||
b := newBench(t)
|
|
||||||
b.ex.Toolchains = entries{}
|
|
||||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
|
||||||
if !IsNodesFailure(out.Verdict) {
|
|
||||||
t.Fatalf("verdict = %+v, want a failure of the nodes", out.Verdict)
|
|
||||||
}
|
|
||||||
b.ex.Toolchains = goEntries("shanks")
|
|
||||||
b.ex.Presence = offline{}
|
|
||||||
if out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1))); !IsNodesFailure(out.Verdict) {
|
|
||||||
t.Fatalf("an offline node: verdict = %+v", out.Verdict)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
type offline struct{}
|
|
||||||
|
|
||||||
func (offline) Online(string) bool { return false }
|
|
||||||
|
|
||||||
func TestAStepThatFailsIsAVerdictNamingTheNode(t *testing.T) {
|
|
||||||
b := newBench(t)
|
|
||||||
b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { return nil, fmt.Errorf("the step failed: ssh refused") }
|
|
||||||
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
|
||||||
if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "ssh refused") {
|
|
||||||
t.Fatalf("verdict = %+v", out.Verdict)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4
|
func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, fmt.Errorf("go list failed") }
|
b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, fmt.Errorf("go list failed") }
|
||||||
@@ -441,7 +461,7 @@ func TestAGroupIsOneVerdictAndEachMembersTestsReadTheOthersAtTheirHeads(t *testi
|
|||||||
var forA []BesideRef
|
var forA []BesideRef
|
||||||
for _, f := range fetch {
|
for _, f := range fetch {
|
||||||
if f.args["repository"] == repoA {
|
if f.args["repository"] == repoA {
|
||||||
forA, _ = f.args["beside"].([]BesideRef)
|
forA, _ = refsOf(f.args["beside"]), true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
found := false
|
found := false
|
||||||
@@ -571,6 +591,8 @@ func trunksOf(t *testing.T, heads map[string]string) *Trunks {
|
|||||||
|
|
||||||
func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversThem(t *testing.T) { // review D1, ADR 0444, issue 1287
|
func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversThem(t *testing.T) { // review D1, ADR 0444, issue 1287
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
|
// A build module whose digest does not echo the siblings (before DigestVersion 3).
|
||||||
|
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil }
|
||||||
b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA})
|
b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA})
|
||||||
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
if got := len(b.steps.tool("fetch")); got != 1 || b.steps.tool("fetch")[0].args["beside"] == nil {
|
if got := len(b.steps.tool("fetch")); got != 1 || b.steps.tool("fetch")[0].args["beside"] == nil {
|
||||||
@@ -592,12 +614,27 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT
|
|||||||
|
|
||||||
// besideDigest stands for the Go build module's digest over a workspace with siblings (DigestVersion 3, lead decision L5):
|
// besideDigest stands for the Go build module's digest over a workspace with siblings (DigestVersion 3, lead decision L5):
|
||||||
// each package's digest folds the siblings' commits in, and the answer echoes the lines it folded.
|
// each package's digest folds the siblings' commits in, and the answer echoes the lines it folded.
|
||||||
|
// echoBeside is d with the siblings the fetches placed, as the Go build module's digest echoes them (its DigestResult's
|
||||||
|
// Beside): every check now places the change's own repository at its trunk (issue 1887 A12).
|
||||||
|
func echoBeside(b *bench, d Digested) Digested {
|
||||||
|
var lines []string
|
||||||
|
for _, f := range b.steps.tool("fetch") {
|
||||||
|
refs, _ := refsOf(f.args["beside"]), true
|
||||||
|
for _, r := range refs {
|
||||||
|
lines = append(lines, r.Repository+" "+r.Commit)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(lines)
|
||||||
|
d.Beside = lines
|
||||||
|
return d
|
||||||
|
}
|
||||||
|
|
||||||
func besideDigest(b *bench) func(c stepCall) (any, error) {
|
func besideDigest(b *bench) func(c stepCall) (any, error) {
|
||||||
return func(c stepCall) (any, error) {
|
return func(c stepCall) (any, error) {
|
||||||
level := c.args["level"].(string)
|
level := c.args["level"].(string)
|
||||||
var lines []string
|
var lines []string
|
||||||
for _, f := range b.steps.tool("fetch") {
|
for _, f := range b.steps.tool("fetch") {
|
||||||
refs, _ := f.args["beside"].([]BesideRef)
|
refs, _ := refsOf(f.args["beside"]), true
|
||||||
for _, r := range refs {
|
for _, r := range refs {
|
||||||
lines = append(lines, r.Repository+" "+r.Commit)
|
lines = append(lines, r.Repository+" "+r.Commit)
|
||||||
}
|
}
|
||||||
@@ -723,3 +760,86 @@ func TestNoStepOfACheckCarriesPublish(t *testing.T) { // ADR 0500 §4, 1309 (the
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1890/AC1, 1887/A5a: an ask whose step never ends (the build node stopped with it running, and nothing redelivered it in
|
||||||
|
// time) ends the check loudly at the asker's bound; the checker holds no place to free.
|
||||||
|
func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
// A node took the ask and its step stays running: the bound runs from its start (review round 2 item 1).
|
||||||
|
b.asks.Answer = func(a delivery.StepAsk) delivery.Step {
|
||||||
|
return delivery.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()}
|
||||||
|
}
|
||||||
|
b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond}
|
||||||
|
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
|
if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") {
|
||||||
|
t.Fatalf("verdict = %+v", out.Verdict)
|
||||||
|
}
|
||||||
|
if n := len(b.asks.Asks()); n != 1 || b.asks.Asks()[0].Kind != delivery.AskCheck || b.asks.Asks()[0].Check == nil {
|
||||||
|
t.Fatalf("asks = %+v, want one check ask", b.asks.Asks())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/A5a, ADR 0536 §4: a check is one ask per pull request and build module on the check stream, a re-judge after a
|
||||||
|
// trunk move one on the recheck stream, each carrying both levels with the kept verdicts taken by the build module.
|
||||||
|
func TestACheckIsOneAskAndAReJudgeGoesOnTheRecheckStream(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
|
asks := b.asks.Asks()
|
||||||
|
if len(asks) != 1 || asks[0].Kind != delivery.AskCheck || asks[0].Language != delivery.AskGo || asks[0].Asker != delivery.CheckerModule {
|
||||||
|
t.Fatalf("asks = %+v, want one Go check ask", asks)
|
||||||
|
}
|
||||||
|
if j := asks[0].Check; j == nil || !j.Kept || strings.Join(j.Levels, ",") != "unit,integration" || j.Fetch["repository"] != repoA {
|
||||||
|
t.Fatalf("the job = %+v", asks[0].Check)
|
||||||
|
}
|
||||||
|
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
cs.Kind = KindTrunk
|
||||||
|
b.judge(t, cs)
|
||||||
|
if asks := b.asks.Asks(); len(asks) != 2 || asks[1].Kind != delivery.AskRecheck {
|
||||||
|
t.Fatalf("asks = %+v, want the re-judge on recheck", asks)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Review round 2 item 1: a check ask that waits in its stream longer than the step bound (no node has a free slot during a
|
||||||
|
// large rebuild) is not failed: the check goes on when a node takes it, and no second ask is put.
|
||||||
|
func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
answer := b.asks.Answer
|
||||||
|
b.asks.Answer = nil
|
||||||
|
b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond}
|
||||||
|
done := make(chan Outcome, 1)
|
||||||
|
go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }()
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) {
|
||||||
|
if time.Now().After(deadline) {
|
||||||
|
t.Fatal("no ask was put")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(100 * time.Millisecond) // five bounds in the stream
|
||||||
|
select {
|
||||||
|
case out := <-done:
|
||||||
|
t.Fatalf("the check ended while its ask waited: %+v", out.Verdict)
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
b.asks.End(answer(b.asks.Asks()[0])) // a node took it and ran it
|
||||||
|
if out := <-done; out.Verdict.State != delivery.VerdictPass || len(b.asks.Asks()) != 1 {
|
||||||
|
t.Fatalf("verdict %+v after %d asks, want a pass from the one ask", out.Verdict, len(b.asks.Asks()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1917: the digest is taken inside the check ask, as the build module's own digest step answers it (no step key is asked
|
||||||
|
// of it), so a second check of the same heads keeps every package the first one passed: "N kept", nothing run again.
|
||||||
|
func TestASecondCheckOfTheSameHeadsKeepsWhatTheFirstPassed(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
first := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
u := first.Levels[delivery.LevelUnit]
|
||||||
|
if first.State != delivery.VerdictPass || u.Run == 0 || u.Kept != 0 {
|
||||||
|
t.Fatalf("the first check = %+v", first)
|
||||||
|
}
|
||||||
|
b.steps.calls = nil
|
||||||
|
second := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
||||||
|
if s := second.Levels[delivery.LevelUnit]; second.State != delivery.VerdictPass || s.Kept != u.Run || s.Run != 0 {
|
||||||
|
t.Fatalf("the second check's unit level = %+v, want %d kept and none run", s, u.Run)
|
||||||
|
}
|
||||||
|
if n := len(b.steps.tool("test")); n != 0 {
|
||||||
|
t.Errorf("%d test steps ran on the second check", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The failure the build module reported for group r4g on 2026-10-23: the test logged a line, then failed with a multi-line
|
||||||
|
// t.Fatalf. Its output is what the build module now sends (its last lines, with the error line as Cause).
|
||||||
|
func fatalAfterLog() Failure {
|
||||||
|
return Failure{
|
||||||
|
Package: "github.com/novox/mesh-controller/internal/broker", Test: "TestTheCataloguesDerivedSubjectsMeet",
|
||||||
|
Cause: "3 subscription(s) derive to a subject no emitter owns:",
|
||||||
|
Output: " agreement_catalogue_test.go:130: judged against /var/lib/mesh-build-go-work/id/beside/mesh-catalog as cloned\n" +
|
||||||
|
" agreement_catalogue_test.go:186: 3 subscription(s) derive to a subject no emitter owns:\n" +
|
||||||
|
" a.b.c\n d.e.f\n g.h.i\n--- FAIL: TestTheCataloguesDerivedSubjectsMeet (0.00s)",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAFailedTestsVerdictKeepsItsWholeFailureMessage(t *testing.T) { // 1901/AC1
|
||||||
|
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{fatalAfterLog()}})
|
||||||
|
for _, want := range []string{"3 subscription(s) derive to a subject no emitter owns:", "a.b.c", "d.e.f", "g.h.i"} {
|
||||||
|
if !strings.Contains(v.Failed.Words, want) {
|
||||||
|
t.Errorf("words lack %q: %q", want, v.Failed.Words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(v.Failed.Words) > MaxWords+200 {
|
||||||
|
t.Errorf("words are %d bytes, bound %d", len(v.Failed.Words), MaxWords)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheStatusLineNamesTheFailureNotTheFirstLoggedLine(t *testing.T) { // 1901/AC2
|
||||||
|
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{fatalAfterLog()}})
|
||||||
|
got := firstFailed(*v)
|
||||||
|
if !strings.Contains(got, "3 subscription(s) derive to a subject no emitter owns:") || strings.Contains(got, "judged against") || strings.Contains(got, "\n") {
|
||||||
|
t.Errorf("firstFailed = %q", got)
|
||||||
|
}
|
||||||
|
// With no cause told apart (an older build module), the words still start at the first line, as before.
|
||||||
|
f := fatalAfterLog()
|
||||||
|
f.Cause = ""
|
||||||
|
v = levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||||
|
if !strings.HasPrefix(v.Failed.Words, "agreement_catalogue_test.go:130") {
|
||||||
|
t.Errorf("words = %q", v.Failed.Words)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestWordsLongerThanTheBoundKeepTheirLastLines(t *testing.T) { // 1901/AC1
|
||||||
|
f := Failure{Output: strings.Repeat("log line of some length\n", 500) + "the cause is here"}
|
||||||
|
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||||
|
if !strings.HasSuffix(v.Failed.Words, "the cause is here") || len(v.Failed.Words) > MaxWords+200 {
|
||||||
|
t.Errorf("%d bytes, ends %q", len(v.Failed.Words), v.Failed.Words[len(v.Failed.Words)-30:])
|
||||||
|
}
|
||||||
|
_ = delivery.VerdictFail
|
||||||
|
}
|
||||||
|
|
||||||
|
// An older build module sends no "cause": its answer is still accepted and the verdict's words are its output, as before.
|
||||||
|
func TestAnOlderBuildModulesFailureIsAcceptedWithItsWholeOutput(t *testing.T) { // 1901/AC3
|
||||||
|
old := `{"package":"p","test":"TestX","output":"x_test.go:1: logged\nx_test.go:2: the old kept lines"}`
|
||||||
|
var f Failure
|
||||||
|
if err := json.Unmarshal([]byte(old), &f); err != nil || f.Cause != "" {
|
||||||
|
t.Fatalf("1901/AC3: err = %v, cause = %q", err, f.Cause)
|
||||||
|
}
|
||||||
|
v := levelFailure(&target{}, "unit", Tested{Failed: []Failure{f}})
|
||||||
|
if v.Failed.Words != "x_test.go:1: logged\nx_test.go:2: the old kept lines" {
|
||||||
|
t.Errorf("1901/AC3: words = %q", v.Failed.Words)
|
||||||
|
}
|
||||||
|
if got := firstFailed(*v); !strings.Contains(got, "logged") || strings.Contains(got, "\n") {
|
||||||
|
t.Errorf("1901/AC3: status = %q", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,6 +4,7 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
@@ -46,11 +47,11 @@ func TestTheVerdictsHeadsAreTheGatesHashOverLowercaseHeads(t *testing.T) { // ga
|
|||||||
func TestAGroupsVerdictHeadsCoverEveryMember(t *testing.T) { // gate (b): hq's member too, as the gate's ask covers it
|
func TestAGroupsVerdictHeadsCoverEveryMember(t *testing.T) { // gate (b): hq's member too, as the gate's ask covers it
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
r := newRig(t, passJudge())
|
r := newRig(t, passJudge())
|
||||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(HQRepository, 3, head2))
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(hqRepo, 3, head2))
|
||||||
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
r.ch.Candidate(ctx, "feat%2Fx", cand)
|
||||||
r.ch.Wait()
|
r.ch.Wait()
|
||||||
want, _ := delivery.HeadsHash([]delivery.Head{{Repository: repoA, Number: 7, Head: head1}, {Repository: repoSDK, Number: 12, Head: head2},
|
want, _ := delivery.HeadsHash([]delivery.Head{{Repository: repoA, Number: 7, Head: head1}, {Repository: repoSDK, Number: 12, Head: head2},
|
||||||
{Repository: HQRepository, Number: 3, Head: head2}})
|
{Repository: hqRepo, Number: 3, Head: head2}})
|
||||||
if v := r.verdict(t, keyOf(t, "feat/x", cand)); v.Heads != want {
|
if v := r.verdict(t, keyOf(t, "feat/x", cand)); v.Heads != want {
|
||||||
t.Errorf("heads = %q, want %q", v.Heads, want)
|
t.Errorf("heads = %q, want %q", v.Heads, want)
|
||||||
}
|
}
|
||||||
@@ -71,11 +72,12 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
|
|||||||
t.Fatalf("%s#%d is %q before the move", p.repo, p.n, s)
|
t.Fatalf("%s#%d is %q before the move", p.repo, p.n, s)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// A delivery builds, so no check starts; mesh-delivery's trunk moves.
|
// mesh-delivery's trunk moves; the re-judges it asks are held in their judge (in the stream, since R4g) meanwhile.
|
||||||
r.ch.Hold = func() bool { return true }
|
release := make(chan struct{})
|
||||||
|
r.ch.Judge = &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { <-release; return Outcome{Verdict: passing()}, nil }}
|
||||||
|
defer func() { close(release); r.ch.Wait() }()
|
||||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||||
r.ch.TrunkMoved(ctx)
|
r.ch.TrunkMoved(ctx)
|
||||||
r.ch.Wait()
|
|
||||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||||
t.Errorf("mesh-delivery#7, into the moved trunk, is %q, want pending", s)
|
t.Errorf("mesh-delivery#7, into the moved trunk, is %q, want pending", s)
|
||||||
}
|
}
|
||||||
@@ -90,17 +92,25 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) {
|
|||||||
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
|
func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
var r *rig
|
var r *rig
|
||||||
|
release, judged := make(chan struct{}), make(chan struct{})
|
||||||
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) {
|
||||||
if call == 1 {
|
if call == 1 {
|
||||||
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs
|
||||||
r.ch.Hold = func() bool { return true } // and a delivery holds the next check
|
defer close(judged)
|
||||||
|
} else {
|
||||||
|
<-release // the check asked again for the new trunk waits in the stream meanwhile
|
||||||
}
|
}
|
||||||
return Outcome{Verdict: passing()}, nil
|
return Outcome{Verdict: passing()}, nil
|
||||||
}}
|
}}
|
||||||
r = newRig(t, j)
|
r = newRig(t, j)
|
||||||
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
||||||
r.ch.Wait()
|
<-judged
|
||||||
|
for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) {
|
||||||
|
}
|
||||||
|
time.Sleep(20 * time.Millisecond)
|
||||||
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
if s := owedState(t, r, repoA, 7, head1); s != "pending" {
|
||||||
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
|
t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s)
|
||||||
}
|
}
|
||||||
|
close(release)
|
||||||
|
r.ch.Wait()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,18 +1,14 @@
|
|||||||
package check
|
package check
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Unheard is the error of a call that reached nobody: the bus had no responder for it, or none answered within the call's
|
// Unheard is the error of a call that reached nobody: the bus had no responder for it, or none answered within the call's
|
||||||
// bound. The step it asked for never began, so the same step may be asked of another node (issue 1820). The program wraps
|
// bound. The program wraps the errors of its forge calls in it where the bus gives a timeout; a no-responders error is
|
||||||
// the errors of its calls in it where the bus gives a timeout; a no-responders error is known as it is.
|
// known as it is. A build step is no call since R4g: an ask whose node stops is delivered again by its stream (ADR 0525 §6).
|
||||||
type Unheard struct{ Err error }
|
type Unheard struct{ Err error }
|
||||||
|
|
||||||
func (u *Unheard) Error() string { return u.Err.Error() }
|
func (u *Unheard) Error() string { return u.Err.Error() }
|
||||||
@@ -23,79 +19,3 @@ func isUnheard(err error) bool {
|
|||||||
var u *Unheard
|
var u *Unheard
|
||||||
return errors.As(err, &u) || errors.Is(err, nats.ErrNoResponders)
|
return errors.As(err, &u) || errors.Is(err, nats.ErrNoResponders)
|
||||||
}
|
}
|
||||||
|
|
||||||
// deafNode is a node that did not answer a build module's call during this judging.
|
|
||||||
type deafNode struct{ node, why string }
|
|
||||||
|
|
||||||
// markDeaf takes a node out of the choice for a build module for the rest of the judging.
|
|
||||||
func (st *state) markDeaf(bm, node string, err error) {
|
|
||||||
if st.deaf == nil {
|
|
||||||
st.deaf = map[string][]deafNode{}
|
|
||||||
}
|
|
||||||
st.deaf[bm] = append(st.deaf[bm], deafNode{node, err.Error()})
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *state) isDeaf(bm, node string) bool {
|
|
||||||
for _, d := range st.deaf[bm] {
|
|
||||||
if d.node == node {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// triedWords names every node that did not answer a build module, with what the bus said.
|
|
||||||
func (st *state) triedWords(bm string) string {
|
|
||||||
var parts []string
|
|
||||||
for _, d := range st.deaf[bm] {
|
|
||||||
parts = append(parts, fmt.Sprintf("%s (%s)", d.node, d.why))
|
|
||||||
}
|
|
||||||
return strings.Join(parts, "; ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// moveStep is called when the target's node did not answer: the node is left out of the choice, and another build node of the
|
|
||||||
// same build module takes the target's place. It answers an error naming every node tried when none is left (the words hold
|
|
||||||
// NodesWords, so the verdict is judged again when the nodes change).
|
|
||||||
func (st *state) moveStep(t *target, err error) error {
|
|
||||||
old := t.node
|
|
||||||
st.markDeaf(t.module, old, err)
|
|
||||||
node, _, cerr := st.choose(t.module)
|
|
||||||
if cerr != nil {
|
|
||||||
return fmt.Errorf("%s; tried: %s", cerr, st.triedWords(t.module))
|
|
||||||
}
|
|
||||||
st.j.logf("%s#%d: %s did not answer (%v), the step goes to %s", t.pull.Repository, t.pull.Number, old, err, node)
|
|
||||||
t.node = node
|
|
||||||
other := false
|
|
||||||
for _, o := range st.targets {
|
|
||||||
if o != t && o.node == old {
|
|
||||||
other = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !other {
|
|
||||||
delete(st.nodes, old)
|
|
||||||
}
|
|
||||||
st.nodes[node] = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// refetch makes the target's workspace again on its new node: the workspace of a step that was lost is on the node that did
|
|
||||||
// not answer.
|
|
||||||
func (st *state) refetch(ctx context.Context, t *target) error {
|
|
||||||
raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", t.pull.Repository, t.pull.Number, t.node), t.fetchArgs)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
var f Fetched
|
|
||||||
if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" {
|
|
||||||
return fmt.Errorf("the fetch of %s#%d on %s answered no workspace: %s", t.pull.Repository, t.pull.Number, t.node, raw)
|
|
||||||
}
|
|
||||||
t.fetched = f
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// stepOnWorkspace sets the workspace a step reads to the target's current one.
|
|
||||||
func stepOnWorkspace(t *target, args map[string]any) {
|
|
||||||
if _, ok := args["workspace"]; ok {
|
|
||||||
args["workspace"] = t.fetched.Workspace
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,9 +1,7 @@
|
|||||||
package check
|
package check
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -11,127 +9,8 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Issue 1820: a build step goes to a node whose build module is not heard (a laptop asleep, not yet marked offline by the
|
// Issue 1820 AC5: a read of the forge that nobody answers is the forge's failure, not a build node's. (AC1-AC4 moved a
|
||||||
// watchman), and the bus answers "no responders". The check moves the step to another node of the same build module.
|
// step to another node; since R4g an ask whose node stops is delivered again by its stream: ADR 0525 §6.)
|
||||||
|
|
||||||
// deafNodes makes every tool on the named nodes fail as the bus does when nobody listens there; count says how many
|
|
||||||
// times the node was called.
|
|
||||||
func deafNodes(b *bench, deaf ...string) {
|
|
||||||
for tool, h := range b.steps.handlers {
|
|
||||||
h := h
|
|
||||||
b.steps.handlers[tool] = func(c stepCall) (any, error) {
|
|
||||||
for _, n := range deaf {
|
|
||||||
if c.node == n {
|
|
||||||
return nil, fmt.Errorf("%s on %s: %w", c.tool, c.node, nats.ErrNoResponders)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return h(c)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func nodesOf(calls []stepCall) []string {
|
|
||||||
var out []string
|
|
||||||
for _, c := range calls {
|
|
||||||
out = append(out, c.node)
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAFetchThatNobodyAnswersMovesToAnotherBuildNode(t *testing.T) { // 1820/AC1
|
|
||||||
b := newBench(t)
|
|
||||||
b.ex.Toolchains = goEntries("ace", "g14")
|
|
||||||
deafNodes(b, "ace")
|
|
||||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
|
||||||
if v.State != delivery.VerdictPass {
|
|
||||||
t.Fatalf("verdict = %+v, want a pass on the node that answers", v)
|
|
||||||
}
|
|
||||||
if got := nodesOf(b.steps.tool("fetch")); len(got) != 2 || got[0] != "ace" || got[1] != "g14" {
|
|
||||||
t.Errorf("fetch went to %v, want ace (no responders) then g14", got)
|
|
||||||
}
|
|
||||||
for _, c := range b.steps.tool("test") {
|
|
||||||
if c.node != "g14" {
|
|
||||||
t.Errorf("a test went to %s, want g14 only", c.node)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(v.Machines) != 1 || v.Machines[0] != "g14" {
|
|
||||||
t.Errorf("machines = %v, want [g14]: ace ran nothing", v.Machines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestATestStepThatNobodyAnswersIsFetchedAgainOnAnotherNode(t *testing.T) { // 1820/AC2
|
|
||||||
b := newBench(t)
|
|
||||||
b.ex.Toolchains = goEntries("ace", "g14")
|
|
||||||
// ace answers the fetch and the digest, then goes to sleep before the test.
|
|
||||||
test := b.steps.handlers["build_go_test"]
|
|
||||||
b.steps.handlers["build_go_test"] = func(c stepCall) (any, error) {
|
|
||||||
if c.node == "ace" {
|
|
||||||
return nil, fmt.Errorf("test on ace: %w", nats.ErrNoResponders)
|
|
||||||
}
|
|
||||||
return test(c)
|
|
||||||
}
|
|
||||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
|
||||||
if v.State != delivery.VerdictPass {
|
|
||||||
t.Fatalf("verdict = %+v, want a pass", v)
|
|
||||||
}
|
|
||||||
if got := nodesOf(b.steps.tool("fetch")); len(got) != 2 || got[1] != "g14" {
|
|
||||||
t.Errorf("fetch went to %v, want ace then g14 (the workspace was on ace)", got)
|
|
||||||
}
|
|
||||||
for _, c := range b.steps.tool("test") {
|
|
||||||
if c.node == "g14" && c.args["workspace"] == nil {
|
|
||||||
t.Errorf("a test on g14 has no workspace")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(v.Machines) != 1 || v.Machines[0] != "g14" {
|
|
||||||
t.Errorf("machines = %v, want [g14]", v.Machines)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestWhenNoBuildNodeAnswersTheFailureNamesEveryNodeTried(t *testing.T) { // 1820/AC3
|
|
||||||
b := newBench(t)
|
|
||||||
b.ex.Toolchains = goEntries("ace", "g14", "shanks")
|
|
||||||
deafNodes(b, "ace", "g14", "shanks")
|
|
||||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
|
||||||
if v.State != delivery.VerdictFail {
|
|
||||||
t.Fatalf("verdict = %+v, want a failure", v)
|
|
||||||
}
|
|
||||||
for _, n := range []string{"ace", "g14", "shanks"} {
|
|
||||||
if !strings.Contains(v.Failed.Words, n) {
|
|
||||||
t.Errorf("failure %q does not name %s", v.Failed.Words, n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !IsNodesFailure(v) {
|
|
||||||
t.Errorf("failure %q is not a failure of the nodes, so it would not be judged again when they change", v.Failed.Words)
|
|
||||||
}
|
|
||||||
if got := len(b.steps.tool("fetch")); got != 3 {
|
|
||||||
t.Errorf("fetch was tried %d times, want once on each of 3 nodes", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAStepThatFailedForTheChangeIsNotMovedToAnotherNode(t *testing.T) { // 1820/AC4
|
|
||||||
b := newBench(t)
|
|
||||||
b.ex.Toolchains = goEntries("ace", "g14")
|
|
||||||
b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { return nil, fmt.Errorf("fetch on ace: the step failed: git exploded") }
|
|
||||||
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
|
|
||||||
if v.State != delivery.VerdictFail || len(b.steps.tool("fetch")) != 1 {
|
|
||||||
t.Errorf("verdict = %+v after %d fetches, want a failure after one", v, len(b.steps.tool("fetch")))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestACanceledCheckIsNotMovedToAnotherNode(t *testing.T) { // 1820/AC4
|
|
||||||
b := newBench(t)
|
|
||||||
b.ex.Toolchains = goEntries("ace", "g14")
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) {
|
|
||||||
cancel()
|
|
||||||
return nil, fmt.Errorf("fetch on ace: %w", nats.ErrNoResponders)
|
|
||||||
}
|
|
||||||
_, _ = b.ex.Judge(ctx, caseOf(t, pullOf(repoA, 7, head1)))
|
|
||||||
if n := len(b.steps.tool("fetch")); n != 1 {
|
|
||||||
t.Errorf("fetch was tried %d times after the check ended, want 1", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestAPinReadThatNobodyAnswersIsTheForgesFailureNotABuildNodes(t *testing.T) { // 1820/AC5
|
func TestAPinReadThatNobodyAnswersIsTheForgesFailureNotABuildNodes(t *testing.T) { // 1820/AC5
|
||||||
b := groupBench(t, "cb54756c3865")
|
b := groupBench(t, "cb54756c3865")
|
||||||
b.ex.Forge.(*fakeForge).err = fmt.Errorf("gitea.gitea_get_file: %w", nats.ErrNoResponders)
|
b.ex.Forge.(*fakeForge).err = fmt.Errorf("gitea.gitea_get_file: %w", nats.ErrNoResponders)
|
||||||
|
|||||||
@@ -1,7 +0,0 @@
|
|||||||
package check
|
|
||||||
|
|
||||||
// PlansBuildingBucket is the controller's bucket of the delivery plans in their build phase, one key for each plan, written
|
|
||||||
// by the controller alone (issue 1652): a plan key there means a delivery builds on the build nodes, and no new check starts
|
|
||||||
// meanwhile (ADR 0484 clause 12, ADR 0483 clause 6). The keys age out with no delete marker, so the keys are listed, not watched. The bucket may be absent: the controller's issue 1652 is delivered
|
|
||||||
// before R4c-5b, and if it is not, the bucket is read as empty.
|
|
||||||
const PlansBuildingBucket = "mesh-controller_plans-building"
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
package check
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
func TestARunningCheckIsNotStoppedWhenADeliveryStartsToBuild(t *testing.T) { // ADR 0484 clause 12, ADR 0483 clause 4
|
|
||||||
ctx := context.Background()
|
|
||||||
held := false
|
|
||||||
j := &judgeFunc{f: func(_ context.Context, _ *Case, call int) (Outcome, error) {
|
|
||||||
if call == 1 {
|
|
||||||
held = true // a delivery starts while this check runs
|
|
||||||
}
|
|
||||||
return Outcome{Verdict: passing()}, nil
|
|
||||||
}}
|
|
||||||
r := newRig(t, j)
|
|
||||||
r.ch.Hold = func() bool { return held }
|
|
||||||
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
|
||||||
r.ch.Wait()
|
|
||||||
if v := r.verdict(t, keyOf(t, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))); v.State != delivery.VerdictPass {
|
|
||||||
t.Errorf("verdict = %+v, want the running check to end as it was", v)
|
|
||||||
}
|
|
||||||
// A new candidate waits while the plan builds.
|
|
||||||
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf(repoA, 8, head2)))
|
|
||||||
r.ch.Wait()
|
|
||||||
if j.count() != 1 {
|
|
||||||
t.Errorf("judged %d times while a delivery builds, want only the one already running", j.count())
|
|
||||||
}
|
|
||||||
held = false
|
|
||||||
r.ch.Reconsider(ctx)
|
|
||||||
r.ch.Wait()
|
|
||||||
if j.count() != 2 {
|
|
||||||
t.Errorf("judged %d times after the delivery ended, want the waiting candidate too", j.count())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -3,6 +3,7 @@ package check
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"path"
|
"path"
|
||||||
"slices"
|
"slices"
|
||||||
@@ -10,6 +11,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/buildstep"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
||||||
)
|
)
|
||||||
@@ -17,14 +19,45 @@ import (
|
|||||||
// goModule is the build module the Go repositories are judged by.
|
// goModule is the build module the Go repositories are judged by.
|
||||||
const goModule = buildrun.GoBuildModule
|
const goModule = buildrun.GoBuildModule
|
||||||
|
|
||||||
|
// RunnerWorkspace is the runner the Go build module's digest and test steps are asked with when the merge changes the build
|
||||||
|
// module itself: the installed module compiles the candidate's program from the merged tree and runs the step as it, shut
|
||||||
|
// off (ADR 0535 §1).
|
||||||
|
const RunnerWorkspace = "workspace"
|
||||||
|
|
||||||
|
// buildGoDir is the Go build module's directory in the catalogue: a merge changing a path under it is judged by its
|
||||||
|
// candidate (ADR 0535 §1). mesh-build-node's own directory is judged by the installed module until its part of issue 1898.
|
||||||
|
const buildGoDir = "modules/mesh-build-go/"
|
||||||
|
|
||||||
|
// changesBuildGo says whether a merge of the repository with these changed paths changes the Go build module.
|
||||||
|
func changesBuildGo(repository string, changed []string) bool {
|
||||||
|
if repository != catalogueRepository {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, p := range changed {
|
||||||
|
if strings.HasPrefix(p, buildGoDir) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// The words of the line on the program that judged a change to the Go build module (ADR 0535 §2, §5).
|
||||||
|
const (
|
||||||
|
judgedByOldInstalled = "judged by the installed mesh-build-go: it does not run a candidate yet"
|
||||||
|
judgedByInstalled = "judged by the installed mesh-build-go"
|
||||||
|
judgedByCandidate = "judged by the candidate's mesh-build-go at "
|
||||||
|
)
|
||||||
|
|
||||||
// verdictPtr is a verdict of failure a step of the check ends with.
|
// verdictPtr is a verdict of failure a step of the check ends with.
|
||||||
type verdictPtr = *delivery.Verdict
|
type verdictPtr = *delivery.Verdict
|
||||||
|
|
||||||
func failed(machine, words string) verdictPtr { return fail(machine, words) }
|
func failed(machine, words string) verdictPtr { return fail(machine, words) }
|
||||||
|
|
||||||
// Steps runs a build module's long step on a node and answers its result (buildrun.Runner).
|
// Asks puts a check's ask on its build seat's stream and waits for its step (buildrun.Asker; ADR 0525 §1, §6): the
|
||||||
type Steps interface {
|
// checker chooses no node, starts no step and holds none; whichever build node pulls the ask runs the check.
|
||||||
Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error)
|
type Asks interface {
|
||||||
|
Put(ctx context.Context, a delivery.StepAsk) error
|
||||||
|
Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
|
||||||
@@ -38,24 +71,18 @@ type BuildModules interface {
|
|||||||
BuildModules(repository string) []string
|
BuildModules(repository string) []string
|
||||||
}
|
}
|
||||||
|
|
||||||
// toolPrefix is the start of each build module's tool names.
|
// askLanguage is the build seat language of each build module the checker asks.
|
||||||
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}
|
var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode}
|
||||||
|
|
||||||
// Examiner judges a case through the build modules (ADR 0484 clauses 3, 4, 10, 13): for each pull request it fetches the head
|
// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request
|
||||||
// merged onto its trunk head on a build node, then runs the unit level and the integration level, taking the kept verdicts
|
// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level
|
||||||
// of unchanged packages, running the change's own tests first and stopping at the first failure.
|
// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and
|
||||||
|
// stopping at the first failure; it judges the step's answer as it judged the tools' answers.
|
||||||
type Examiner struct {
|
type Examiner struct {
|
||||||
Steps Steps
|
Asks Asks
|
||||||
|
// Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on.
|
||||||
Toolchains ToolchainEntries
|
Toolchains ToolchainEntries
|
||||||
// Presence says which nodes are online; Health whether a node's build module is running and healthy. A node that is not
|
Kept Kept
|
||||||
// is not chosen; nil means every node is.
|
|
||||||
Presence interface{ Online(node string) bool }
|
|
||||||
Health interface {
|
|
||||||
Healthy(node, module string) bool
|
|
||||||
}
|
|
||||||
// Places holds a step back while its node is at the checker's capacity (ADR 0484 clause 12).
|
|
||||||
Places *buildrun.Places
|
|
||||||
Kept Kept
|
|
||||||
// Previews is the previews bucket; nil writes none.
|
// Previews is the previews bucket; nil writes none.
|
||||||
Previews KV
|
Previews KV
|
||||||
Modules BuildModules
|
Modules BuildModules
|
||||||
@@ -76,8 +103,18 @@ type Examiner struct {
|
|||||||
}
|
}
|
||||||
// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
|
// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
|
||||||
Trunks interface{ Heads() map[string]string }
|
Trunks interface{ Heads() map[string]string }
|
||||||
Now func() time.Time
|
// Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey),
|
||||||
Logf func(format string, args ...any)
|
// waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil
|
||||||
|
// leaves the composition not run.
|
||||||
|
Compositions CompositionReader
|
||||||
|
Now func() time.Time
|
||||||
|
Logf func(format string, args ...any)
|
||||||
|
}
|
||||||
|
|
||||||
|
// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote
|
||||||
|
// a verdict that does not wait, or ctx ends.
|
||||||
|
type CompositionReader interface {
|
||||||
|
Await(ctx context.Context, key string) (delivery.CompositionVerdict, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (j *Examiner) now() time.Time {
|
func (j *Examiner) now() time.Time {
|
||||||
@@ -103,35 +140,56 @@ func besideLines(refs []BesideRef) []string {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// target is a pull request on one build module: the node chosen, the workspace its fetch made and what it said.
|
// target is a pull request on one build module: the node that pulled its ask, the workspace its fetch made and what it
|
||||||
|
// said.
|
||||||
type target struct {
|
type target struct {
|
||||||
pull Pull
|
pull Pull
|
||||||
module string
|
module string
|
||||||
node string
|
node string
|
||||||
toolchain string
|
toolchain string
|
||||||
fetched Fetched
|
fetched Fetched
|
||||||
// fetchArgs are the arguments of the target's fetch, so a workspace lost with its node is made again on another.
|
// fetchArgs are the arguments of the fetch its check ask runs.
|
||||||
fetchArgs map[string]any
|
fetchArgs map[string]any
|
||||||
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
|
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
|
||||||
changed []string
|
changed []string
|
||||||
// beside are the lines `<owner/repo> <commit>` of the siblings the workspace was given, sorted: its tests may read them,
|
// beside are the lines `<owner/repo> <commit>` of the siblings the workspace was given, sorted: its tests may read them,
|
||||||
// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
|
// so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5).
|
||||||
beside []string
|
beside []string
|
||||||
plans map[string]*goPlan // by level
|
// runner is RunnerWorkspace when the target's ask runs the candidate's own build module (ADR 0535 §1), empty otherwise.
|
||||||
|
runner string
|
||||||
|
// judged is what the line says of the program that judged a change to the build module; empty for any other change.
|
||||||
|
judged string
|
||||||
}
|
}
|
||||||
|
|
||||||
// goPlan is what a Go target has to run at one level: the digests, the packages without a kept verdict and the tests taken.
|
// judgedBy takes what a step under runner workspace answered of the program that ran it (judged_by): the candidate, or the
|
||||||
type goPlan struct {
|
// installed module (a module that does not take the runner says nothing, and is asked without it from now on: ADR 0535 §2).
|
||||||
digests map[string]string
|
// It says whether the candidate ran.
|
||||||
todo []string
|
func (t *target) judgedBy(answer string) bool {
|
||||||
// all says the digest step gave nothing to key a kept verdict by, so the level runs whole and keeps nothing.
|
switch {
|
||||||
all bool
|
case strings.HasPrefix(answer, "candidate "):
|
||||||
|
t.judged = judgedByCandidate + short12(strings.TrimPrefix(answer, "candidate "))
|
||||||
|
return true
|
||||||
|
case answer == "":
|
||||||
|
t.judged, t.runner = judgedByOldInstalled, ""
|
||||||
|
default:
|
||||||
|
t.judged, t.runner = judgedByInstalled, ""
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// stepArgs are a step's arguments with the target's runner.
|
||||||
|
func (t *target) stepArgs(args map[string]any) map[string]any {
|
||||||
|
if t.runner != "" {
|
||||||
|
args["runner"] = t.runner
|
||||||
|
}
|
||||||
|
return args
|
||||||
}
|
}
|
||||||
|
|
||||||
// acc collects a level's counts across targets.
|
// acc collects a level's counts across targets.
|
||||||
type acc struct {
|
type acc struct {
|
||||||
run, kept, skipped int
|
run, kept, skipped int
|
||||||
elapsed, setup float64
|
elapsed, setup float64
|
||||||
|
lint float64
|
||||||
over bool
|
over bool
|
||||||
slowest []TestTime
|
slowest []TestTime
|
||||||
}
|
}
|
||||||
@@ -143,7 +201,6 @@ type state struct {
|
|||||||
targets []*target
|
targets []*target
|
||||||
levels map[string]*acc
|
levels map[string]*acc
|
||||||
nodes map[string]bool
|
nodes map[string]bool
|
||||||
deaf map[string][]deafNode // by build module: the nodes that did not answer during this judging
|
|
||||||
clone time.Duration
|
clone time.Duration
|
||||||
slow []string
|
slow []string
|
||||||
}
|
}
|
||||||
@@ -152,34 +209,42 @@ type state struct {
|
|||||||
// be judged at all.
|
// be judged at all.
|
||||||
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
|
||||||
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
|
||||||
if v := st.fetchAll(ctx); v != nil {
|
if v := st.judgeAll(ctx); v != nil {
|
||||||
return st.finish(ctx, *v), nil
|
return st.finish(ctx, *v), nil
|
||||||
}
|
}
|
||||||
if v := st.checkPins(ctx); v != nil {
|
|
||||||
return st.finish(ctx, *v), nil
|
|
||||||
}
|
|
||||||
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
|
|
||||||
if c.Try != nil && c.Try.Level != "" && c.Try.Level != level {
|
|
||||||
continue // a try of one level runs that level alone
|
|
||||||
}
|
|
||||||
for _, priority := range []bool{true, false} {
|
|
||||||
for _, t := range st.targets {
|
|
||||||
if v := st.runTarget(ctx, t, level, priority); v != nil {
|
|
||||||
return st.finish(ctx, *v), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
|
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry.
|
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check
|
||||||
|
// whose tests passed, the assigner's composition of it.
|
||||||
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
|
||||||
out := st.outcome(v)
|
out := st.outcome(v)
|
||||||
|
if out.Verdict.State == delivery.VerdictPass {
|
||||||
|
st.composed(ctx, &out.Verdict)
|
||||||
|
}
|
||||||
out.Builds = st.j.buildNotes(ctx, st.c)
|
out.Builds = st.j.buildNotes(ctx, st.c)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// composed reads the assigner's composition of the check into its verdict (ADR 0528 clause 5): a failed composition
|
||||||
|
// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that
|
||||||
|
// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once.
|
||||||
|
func (st *state) composed(ctx context.Context, v *delivery.Verdict) {
|
||||||
|
if st.j.Compositions == nil || st.c.Try != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
comp, err := st.j.Compositions.Await(ctx, st.c.Key)
|
||||||
|
switch {
|
||||||
|
case errors.Is(err, ErrAssignerNotRunning):
|
||||||
|
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}
|
||||||
|
return
|
||||||
|
case err != nil:
|
||||||
|
st.j.logf("%s: the composition could not be read, so it is said as not run: %v", st.c.Key, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
withComposition(v, comp)
|
||||||
|
}
|
||||||
|
|
||||||
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
// outcome fills the verdict's levels, times, machines and composition from what ran.
|
||||||
func (st *state) outcome(v delivery.Verdict) Outcome {
|
func (st *state) outcome(v delivery.Verdict) Outcome {
|
||||||
if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
|
if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
|
||||||
@@ -190,7 +255,7 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
|
|||||||
}
|
}
|
||||||
ms := func(s float64) int64 { return int64(s * 1000) }
|
ms := func(s float64) int64 { return int64(s * 1000) }
|
||||||
u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
|
u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
|
||||||
v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
|
v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), Lint: ms(u.lint), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
|
||||||
for node := range st.nodes {
|
for node := range st.nodes {
|
||||||
v.Machines = append(v.Machines, node)
|
v.Machines = append(v.Machines, node)
|
||||||
}
|
}
|
||||||
@@ -216,7 +281,16 @@ func (st *state) outcome(v delivery.Verdict) Outcome {
|
|||||||
slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
|
slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return Outcome{Verdict: v, Slowest: slow}
|
out := Outcome{Verdict: v, Slowest: slow}
|
||||||
|
for _, t := range st.targets {
|
||||||
|
if t.judged != "" {
|
||||||
|
if out.Judged == nil {
|
||||||
|
out.Judged = map[string]string{}
|
||||||
|
}
|
||||||
|
out.Judged[pullKey(Pull{Repository: t.pull.Repository, Number: t.pull.Number})] = t.judged
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// fail is a verdict of failure at a machine.
|
// fail is a verdict of failure at a machine.
|
||||||
@@ -225,55 +299,18 @@ func fail(machine, words string) *delivery.Verdict {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
// step runs one step of a build module on the node the target is on, holding a place there while it runs. A node that does
|
// levels are the levels a case's asks run: both, or the one a try names.
|
||||||
// not answer (the bus has no responder, or none answered) is left out and the step goes to another node of the same build
|
func (st *state) levelsOf() []string {
|
||||||
// module; a step after the fetch makes its workspace again there first (issue 1820).
|
if st.c.Try != nil && st.c.Try.Level != "" {
|
||||||
func (st *state) step(ctx context.Context, t *target, tool, label string, args map[string]any) (json.RawMessage, error) {
|
return []string{st.c.Try.Level} // a try of one level runs that level alone
|
||||||
for {
|
|
||||||
raw, err := st.stepOnce(ctx, t, tool, label, args)
|
|
||||||
if err == nil || ctx.Err() != nil || !isUnheard(err) {
|
|
||||||
return raw, err
|
|
||||||
}
|
|
||||||
if merr := st.moveStep(t, err); merr != nil {
|
|
||||||
return nil, merr
|
|
||||||
}
|
|
||||||
if tool != "fetch" {
|
|
||||||
if rerr := st.refetch(ctx, t); rerr != nil {
|
|
||||||
return nil, rerr
|
|
||||||
}
|
|
||||||
stepOnWorkspace(t, args)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
return []string{delivery.LevelUnit, delivery.LevelIntegration}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (st *state) stepOnce(ctx context.Context, t *target, tool, label string, args map[string]any) (json.RawMessage, error) {
|
// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and
|
||||||
st.c.Progress.Set(label, t.node)
|
// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once
|
||||||
release, err := st.j.Places.Acquire(ctx, t.node)
|
// (ADR 0484 clause 13).
|
||||||
if err != nil {
|
func (st *state) judgeAll(ctx context.Context) *delivery.Verdict {
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
return st.j.Steps.Run(ctx, t.module, t.node, toolPrefix[t.module]+tool, args)
|
|
||||||
}
|
|
||||||
|
|
||||||
// choose picks the build node for a pull request's repository on a build module.
|
|
||||||
func (st *state) choose(bm string) (node, toolchain string, err error) {
|
|
||||||
language := buildrun.LanguageOf[bm]
|
|
||||||
qualifies := func(node string) bool {
|
|
||||||
return !st.isDeaf(bm, node) && (st.j.Presence == nil || st.j.Presence.Online(node)) && (st.j.Health == nil || st.j.Health.Healthy(node, bm))
|
|
||||||
}
|
|
||||||
nodes, tc, err := buildrun.ChooseNodes(language, st.j.Toolchains.Entries(language), st.j.Places, qualifies, 1)
|
|
||||||
if err != nil {
|
|
||||||
return "", "", err
|
|
||||||
}
|
|
||||||
return nodes[0], tc.Version, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// fetchAll fetches every pull request's head merged onto its trunk on a build node of each build module the repository
|
|
||||||
// needs. A head that does not merge, or a step that fails, ends the check at once, running no test (ADR 0484 clause 13).
|
|
||||||
func (st *state) fetchAll(ctx context.Context) *delivery.Verdict {
|
|
||||||
began := st.j.now()
|
|
||||||
defer func() { st.clone = st.j.now().Sub(began) }()
|
|
||||||
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
// The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members'
|
||||||
// Node fetches take the group's package (ADR 0500 §5).
|
// Node fetches take the group's package (ADR 0500 §5).
|
||||||
pulls := slices.Clone(st.c.Pulls)
|
pulls := slices.Clone(st.c.Pulls)
|
||||||
@@ -290,44 +327,32 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict {
|
|||||||
for _, p := range pulls {
|
for _, p := range pulls {
|
||||||
var merged []Fetched
|
var merged []Fetched
|
||||||
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
|
||||||
node, tc, err := st.choose(bm)
|
t := &target{pull: p, module: bm}
|
||||||
if err != nil {
|
if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" {
|
||||||
return fail(bm, err.Error())
|
t.toolchain = tc.Version
|
||||||
}
|
}
|
||||||
t := &target{pull: p, module: bm, node: node, toolchain: tc, plans: map[string]*goPlan{}}
|
|
||||||
st.nodes[node] = true
|
|
||||||
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
|
||||||
t.fetchArgs = args
|
|
||||||
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository {
|
||||||
args["package"] = groupPackage
|
args["package"] = groupPackage
|
||||||
}
|
}
|
||||||
if bm == buildrun.GoBuildModule {
|
if bm == buildrun.GoBuildModule {
|
||||||
beside, err := st.j.besideFor(ctx, st.c, p)
|
beside, err := st.j.besideFor(ctx, st.c, p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fail(forgeMachine(err, node), err.Error()+unheardWords(err))
|
return fail(forgeMachine(err, bm), err.Error()+unheardWords(err))
|
||||||
}
|
}
|
||||||
if len(beside) > 0 {
|
if len(beside) > 0 {
|
||||||
args["beside"] = beside
|
args["beside"] = beside
|
||||||
t.beside = besideLines(beside)
|
t.beside = besideLines(beside)
|
||||||
}
|
}
|
||||||
}
|
if p.Repository == catalogueRepository {
|
||||||
raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args)
|
t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1)
|
||||||
node = t.node // the step may have moved to another node
|
|
||||||
if err != nil {
|
|
||||||
return fail(node, fmt.Sprintf("fetch of %s#%d: %v%s", p.Repository, p.Number, err, pinHint(st.c, err.Error())))
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &t.fetched); err != nil || t.fetched.Workspace == "" {
|
|
||||||
return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw))
|
|
||||||
}
|
|
||||||
if len(t.fetched.Conflicts) > 0 {
|
|
||||||
if st.c.Try == nil {
|
|
||||||
st.j.writePreview(ctx, p, append(merged, t.fetched))
|
|
||||||
}
|
}
|
||||||
return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
|
|
||||||
}
|
}
|
||||||
t.changed = t.fetched.Changed
|
t.fetchArgs = args
|
||||||
merged = append(merged, t.fetched)
|
|
||||||
st.targets = append(st.targets, t)
|
st.targets = append(st.targets, t)
|
||||||
|
if v := st.ask(ctx, t, &merged); v != nil {
|
||||||
|
return v
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
|
if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head
|
||||||
st.j.writePreview(ctx, p, merged)
|
st.j.writePreview(ctx, p, merged)
|
||||||
@@ -339,6 +364,137 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the
|
||||||
|
// pins of its go.mod files, then each level's digest, kept verdicts and tests.
|
||||||
|
func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict {
|
||||||
|
p := t.pull
|
||||||
|
job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true}
|
||||||
|
if t.runner != "" {
|
||||||
|
job.Runner, job.RunnerWhen = t.runner, buildGoDir
|
||||||
|
}
|
||||||
|
if st.c.Try != nil && st.c.Try.Run != "" {
|
||||||
|
job.Run, job.Kept = st.c.Try.Run, false // a try of some tests runs them alone: a partial run proves no package
|
||||||
|
}
|
||||||
|
kind := delivery.AskCheck
|
||||||
|
if st.c.Kind == KindTrunk {
|
||||||
|
kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4)
|
||||||
|
}
|
||||||
|
a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule,
|
||||||
|
At: st.j.now().UTC(), Check: job}
|
||||||
|
st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "")
|
||||||
|
if err := st.j.Asks.Put(ctx, a); err != nil {
|
||||||
|
return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err))
|
||||||
|
}
|
||||||
|
step, err := st.j.Asks.Wait(ctx, a)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return fail(t.module, ctx.Err().Error())
|
||||||
|
}
|
||||||
|
return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err))
|
||||||
|
}
|
||||||
|
// The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone).
|
||||||
|
took := time.Duration(0)
|
||||||
|
if !step.Started.IsZero() && step.Ended.After(step.Started) {
|
||||||
|
took = step.Ended.Sub(step.Started)
|
||||||
|
}
|
||||||
|
t.node = step.Node
|
||||||
|
st.nodes[step.Node] = true
|
||||||
|
st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node)
|
||||||
|
if step.State != delivery.StepPassed {
|
||||||
|
return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s%s", p.Repository, p.Number, step.State, step.Reason, pinHint(st.c, step.Reason)))
|
||||||
|
}
|
||||||
|
var answer delivery.CheckAnswer
|
||||||
|
if err := json.Unmarshal(step.Result, &answer); err != nil {
|
||||||
|
return fail(step.Node, fmt.Sprintf("the check of %s#%d answered what is not a check: %s", p.Repository, p.Number, step.Result))
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(answer.Fetch, &t.fetched); err != nil || t.fetched.Workspace == "" {
|
||||||
|
return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch))
|
||||||
|
}
|
||||||
|
if len(t.fetched.Conflicts) > 0 {
|
||||||
|
if st.c.Try == nil {
|
||||||
|
st.j.writePreview(ctx, p, append(*merged, t.fetched))
|
||||||
|
}
|
||||||
|
return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
|
||||||
|
}
|
||||||
|
t.changed = t.fetched.Changed
|
||||||
|
if !changesBuildGo(p.Repository, t.changed) {
|
||||||
|
t.runner = "" // the build module used no runner: the merge does not change it
|
||||||
|
}
|
||||||
|
*merged = append(*merged, t.fetched)
|
||||||
|
if v := st.checkPinsOf(ctx, t); v != nil {
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
ran := 0.0
|
||||||
|
for _, la := range answer.Levels {
|
||||||
|
v, elapsed := st.level(ctx, t, la)
|
||||||
|
ran += elapsed
|
||||||
|
if v != nil {
|
||||||
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
st.clone += max(0, took-time.Duration(ran*float64(time.Second)))
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts
|
||||||
|
// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the
|
||||||
|
// verdict. It answers the level's running time (its tests' and setup's).
|
||||||
|
func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) {
|
||||||
|
level := la.Level
|
||||||
|
a := st.levels[level]
|
||||||
|
if a == nil {
|
||||||
|
return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0
|
||||||
|
}
|
||||||
|
var digests map[string]string
|
||||||
|
if t.module == buildrun.GoBuildModule && la.Digest != nil {
|
||||||
|
// The packages come as a list, never keyed by a package path (issue 1930); an answer stored before reads too.
|
||||||
|
digests = la.Digest.Map()
|
||||||
|
if t.runner != "" {
|
||||||
|
t.judgedBy(la.Digest.JudgedBy) // the installed module judged it: the runner is not used again
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if t.runner != "" && la.Judged != "" {
|
||||||
|
t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4)
|
||||||
|
}
|
||||||
|
for _, k := range la.Kept {
|
||||||
|
a.kept += k.Tests
|
||||||
|
}
|
||||||
|
elapsed := 0.0
|
||||||
|
for _, raw := range la.Tests {
|
||||||
|
var res Tested
|
||||||
|
if err := json.Unmarshal(raw, &res); err != nil {
|
||||||
|
return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)), elapsed
|
||||||
|
}
|
||||||
|
if t.runner != "" {
|
||||||
|
t.judgedBy(res.JudgedBy)
|
||||||
|
}
|
||||||
|
elapsed += res.Elapsed + res.Setup + res.Lint
|
||||||
|
if t.module == buildrun.NodeBuildModule {
|
||||||
|
if !res.NoTests {
|
||||||
|
a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
a.run += res.Tests
|
||||||
|
a.skipped += res.Skipped
|
||||||
|
a.slowest = append(a.slowest, res.Slowest...)
|
||||||
|
}
|
||||||
|
a.elapsed += res.Elapsed
|
||||||
|
a.setup += res.Setup
|
||||||
|
a.lint += res.Lint
|
||||||
|
a.over = a.over || res.OverBound
|
||||||
|
if len(digests) > 0 {
|
||||||
|
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, digests, st.j.now()); err != nil {
|
||||||
|
st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !res.Passed {
|
||||||
|
return levelFailure(t, level, res), elapsed
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil, elapsed
|
||||||
|
}
|
||||||
|
|
||||||
// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
|
// changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's
|
||||||
// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5).
|
// TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5).
|
||||||
func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
|
func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
|
||||||
@@ -359,51 +515,6 @@ func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
// isTestFile says whether a repository-relative path is a test file of the languages the checker judges: Go's _test.go and
|
|
||||||
// the TypeScript module's .test. and .spec. files and test directories.
|
|
||||||
func isTestFile(p string) bool {
|
|
||||||
switch {
|
|
||||||
case strings.HasSuffix(p, "_test.go"):
|
|
||||||
return true
|
|
||||||
case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."):
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
for _, seg := range strings.Split(path.Dir(p), "/") {
|
|
||||||
if seg == "__tests__" || seg == "test" || seg == "tests" {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the directory of the
|
|
||||||
// file and the import path agree on their last segments, whichever holds the module's directory and path in front.
|
|
||||||
func changesTestOf(changed []string, pkg string) bool {
|
|
||||||
for _, f := range changed {
|
|
||||||
if !strings.HasSuffix(f, "_test.go") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
dir := path.Dir(f)
|
|
||||||
if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// changesTestIn says whether the merge changes a test file inside the directory of a Node module.
|
|
||||||
func changesTestIn(changed []string, dir string) bool {
|
|
||||||
for _, f := range changed {
|
|
||||||
if !isTestFile(f) || strings.HasSuffix(f, "_test.go") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// failedName names a failure: its package and test.
|
// failedName names a failure: its package and test.
|
||||||
func failedName(f Failure) string {
|
func failedName(f Failure) string {
|
||||||
if f.Test == "" {
|
if f.Test == "" {
|
||||||
@@ -412,13 +523,35 @@ func failedName(f Failure) string {
|
|||||||
return f.Package + "." + f.Test
|
return f.Package + "." + f.Test
|
||||||
}
|
}
|
||||||
|
|
||||||
// firstLine is the first non-empty line of a failure's output, shortened.
|
// MaxWords is the most a failed test's words in a verdict hold (issue 1901). **The words keep the failure's last lines**,
|
||||||
|
// where a test says why it failed: the first line is often something it logged before (the group r4g verdict kept a
|
||||||
|
// "judged against" line and lost the t.Fatalf after it), and the 120 characters kept before cut a docker command's echo
|
||||||
|
// before docker's own error. Four kilobytes holds a long message and is small beside a verdict's other parts.
|
||||||
|
const MaxWords = 4096
|
||||||
|
|
||||||
|
// failureWords is a failure's words: its output's last lines up to MaxWords, with the line the build module told apart as
|
||||||
|
// the test's error first when the output does not start with it, so the first line of the words is the cause and the
|
||||||
|
// status line, which is cut short, names it.
|
||||||
|
func failureWords(f Failure) string {
|
||||||
|
out := strings.TrimSpace(f.Output)
|
||||||
|
if len(out) > MaxWords {
|
||||||
|
out = out[len(out)-MaxWords:]
|
||||||
|
// Start on a whole line, and a whole character.
|
||||||
|
if i := strings.IndexByte(out, '\n'); i >= 0 {
|
||||||
|
out = "... " + strings.TrimLeft(out[i:], "\n")
|
||||||
|
}
|
||||||
|
out = strings.ToValidUTF8(out, "")
|
||||||
|
}
|
||||||
|
if c := strings.TrimSpace(f.Cause); c != "" && !strings.HasPrefix(firstLine(out), c) {
|
||||||
|
out = c + "\n" + out
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// firstLine is the first non-empty line of a failure's words.
|
||||||
func firstLine(s string) string {
|
func firstLine(s string) string {
|
||||||
for _, l := range strings.Split(s, "\n") {
|
for _, l := range strings.Split(s, "\n") {
|
||||||
if l = strings.TrimSpace(l); l != "" {
|
if l = strings.TrimSpace(l); l != "" {
|
||||||
if len(l) > 120 {
|
|
||||||
l = l[:120]
|
|
||||||
}
|
|
||||||
return l
|
return l
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -435,7 +568,7 @@ func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
|
|||||||
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
if t.module == buildrun.NodeBuildModule && f.Package == "" {
|
||||||
name = res.Module
|
name = res.Module
|
||||||
}
|
}
|
||||||
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: firstLine(f.Output)}
|
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: failureWords(f)}
|
||||||
default:
|
default:
|
||||||
why := res.Why
|
why := res.Why
|
||||||
if why == "" {
|
if why == "" {
|
||||||
@@ -445,153 +578,3 @@ func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
|
|||||||
}
|
}
|
||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
// runTarget runs one level of a target: first the packages the change's own tests are in (priority), then the others.
|
|
||||||
func (st *state) runTarget(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict {
|
|
||||||
if t.module == buildrun.NodeBuildModule {
|
|
||||||
return st.runNode(ctx, t, level, priority)
|
|
||||||
}
|
|
||||||
return st.runGo(ctx, t, level, priority)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A level's work is planned once for a target: the digest of each package, the kept verdicts taken, the packages left to run.
|
|
||||||
func (st *state) planGo(ctx context.Context, t *target, level string) *delivery.Verdict {
|
|
||||||
if t.plans[level] != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
plan := &goPlan{digests: map[string]string{}}
|
|
||||||
t.plans[level] = plan
|
|
||||||
if st.c.Try != nil && st.c.Try.Run != "" {
|
|
||||||
// A try of some tests runs them alone: a partial run proves no package, so nothing is taken or kept.
|
|
||||||
plan.all = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
raw, err := st.step(ctx, t, "digest", fmt.Sprintf("digest the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node),
|
|
||||||
map[string]any{"workspace": t.fetched.Workspace, "level": level})
|
|
||||||
var d Digested
|
|
||||||
if err == nil {
|
|
||||||
err = json.Unmarshal(raw, &d)
|
|
||||||
}
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return fail(t.node, ctx.Err().Error())
|
|
||||||
}
|
|
||||||
if err != nil || len(d.Packages) == 0 {
|
|
||||||
// The build module named no digest (a level it cannot digest here): the level runs whole and keeps nothing.
|
|
||||||
if err != nil {
|
|
||||||
st.j.logf("%s#%d: no digest of the %s level, so it runs whole: %v", t.pull.Repository, t.pull.Number, level, err)
|
|
||||||
}
|
|
||||||
plan.all = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
if !slices.Equal(d.Beside, t.beside) {
|
|
||||||
// The siblings the tests read are not all in the digest (a build module before DigestVersion 3, or another set): a
|
|
||||||
// package whose tests read one is never kept, nor taken from a verdict kept for other inputs (ADR 0444, L5).
|
|
||||||
st.j.logf("%s#%d: the %s level's digest covers the siblings %v, not %v, so it runs whole and keeps nothing", t.pull.Repository, t.pull.Number, level, d.Beside, t.beside)
|
|
||||||
plan.all = true
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
plan.digests = d.Packages
|
|
||||||
a := st.levels[level]
|
|
||||||
pkgs := make([]string, 0, len(d.Packages))
|
|
||||||
for p := range d.Packages {
|
|
||||||
pkgs = append(pkgs, p)
|
|
||||||
}
|
|
||||||
sort.Strings(pkgs)
|
|
||||||
for _, p := range pkgs {
|
|
||||||
k, found, err := st.j.Kept.Lookup(ctx, level, d.Packages[p])
|
|
||||||
if err != nil {
|
|
||||||
st.j.logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err)
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
a.kept += k.Tests
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
plan.todo = append(plan.todo, p)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *state) runGo(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict {
|
|
||||||
if v := st.planGo(ctx, t, level); v != nil {
|
|
||||||
return v
|
|
||||||
}
|
|
||||||
plan := t.plans[level]
|
|
||||||
var run []string
|
|
||||||
switch {
|
|
||||||
case plan.all:
|
|
||||||
if priority {
|
|
||||||
return nil // nothing to put first: the level runs whole after
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
for _, p := range plan.todo {
|
|
||||||
if changesTestOf(t.changed, p) == priority {
|
|
||||||
run = append(run, p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(run) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
args := map[string]any{"workspace": t.fetched.Workspace, "level": level}
|
|
||||||
if run != nil {
|
|
||||||
args["packages"] = run
|
|
||||||
}
|
|
||||||
if st.c.Try != nil && st.c.Try.Run != "" {
|
|
||||||
args["run"] = st.c.Try.Run // build_go_test's go test -run pattern
|
|
||||||
}
|
|
||||||
raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), args)
|
|
||||||
if err != nil {
|
|
||||||
return fail(t.node, fmt.Sprintf("the %s level of %s#%d: %v", level, t.pull.Repository, t.pull.Number, err))
|
|
||||||
}
|
|
||||||
var res Tested
|
|
||||||
if err := json.Unmarshal(raw, &res); err != nil {
|
|
||||||
return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err))
|
|
||||||
}
|
|
||||||
a := st.levels[level]
|
|
||||||
a.run += res.Tests
|
|
||||||
a.skipped += res.Skipped
|
|
||||||
a.elapsed += res.Elapsed
|
|
||||||
a.setup += res.Setup
|
|
||||||
a.over = a.over || res.OverBound
|
|
||||||
a.slowest = append(a.slowest, res.Slowest...)
|
|
||||||
if len(plan.digests) > 0 {
|
|
||||||
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, plan.digests, st.j.now()); err != nil {
|
|
||||||
st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !res.Passed {
|
|
||||||
return levelFailure(t, level, res)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (st *state) runNode(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict {
|
|
||||||
for _, dir := range t.fetched.Modules {
|
|
||||||
if changesTestIn(t.changed, dir) != priority {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d (%s) on %s", level, t.pull.Repository, t.pull.Number, dir, t.node),
|
|
||||||
map[string]any{"workspace": t.fetched.Workspace, "module": dir, "level": level})
|
|
||||||
if err != nil {
|
|
||||||
return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s): %v", level, t.pull.Repository, t.pull.Number, dir, err))
|
|
||||||
}
|
|
||||||
var res Tested
|
|
||||||
if err := json.Unmarshal(raw, &res); err != nil {
|
|
||||||
return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s) answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, dir, err))
|
|
||||||
}
|
|
||||||
if res.Module == "" {
|
|
||||||
res.Module = dir
|
|
||||||
}
|
|
||||||
a := st.levels[level]
|
|
||||||
if !res.NoTests {
|
|
||||||
a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
|
|
||||||
}
|
|
||||||
a.elapsed += res.Elapsed
|
|
||||||
a.setup += res.Setup
|
|
||||||
a.over = a.over || res.OverBound
|
|
||||||
if !res.Passed {
|
|
||||||
return levelFailure(t, level, res)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,207 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The judged_by answer of mesh-build-go's digest and test steps (ADR 0535 §5; the milestone file's interface for issue
|
||||||
|
// 1898): "installed" for a step its installed program ran, "candidate <merge commit>" for one the candidate's program ran
|
||||||
|
// under runner workspace; an installed module older than the runner names nothing (it ignores the unknown argument: the
|
||||||
|
// SDK's tool server checks no argument against the input schema).
|
||||||
|
|
||||||
|
const buildGoFile = "modules/mesh-build-go/cmd/mesh-build-go/digest.go"
|
||||||
|
|
||||||
|
// candidateBench is a bench whose fetch changes the Go build module's directory, and whose build module runs the candidate
|
||||||
|
// when asked to (a module with the runner) or ignores the runner (one without).
|
||||||
|
func candidateBench(t *testing.T, takesRunner bool) *bench {
|
||||||
|
b := newBench(t)
|
||||||
|
b.fetch.Changed = []string{"README.md", buildGoFile}
|
||||||
|
judged := func(c stepCall) string {
|
||||||
|
if !takesRunner {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
if c.args["runner"] == RunnerWorkspace {
|
||||||
|
return "candidate " + b.fetch.Merge
|
||||||
|
}
|
||||||
|
return "installed"
|
||||||
|
}
|
||||||
|
b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) {
|
||||||
|
d := echoBeside(b, b.digest[c.args["level"].(string)])
|
||||||
|
d.JudgedBy = judged(c)
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
b.test = func(c stepCall) (any, error) {
|
||||||
|
r := captured[Tested](t, "go-test-slow")
|
||||||
|
r.Level, _ = c.args["level"].(string)
|
||||||
|
r.JudgedBy = judged(c)
|
||||||
|
return r, nil
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMergeChangingTheGoBuildModuleIsAskedWithTheCandidateRunner(t *testing.T) { // 1898/AC1
|
||||||
|
b := candidateBench(t, true)
|
||||||
|
b.judge(t, caseOf(t, pullOf(catalogueRepository, 9, head1)))
|
||||||
|
asked := append(b.steps.tool("digest"), b.steps.tool("test")...)
|
||||||
|
if len(asked) < 3 {
|
||||||
|
t.Fatalf("asked %+v, want the digest and both levels", asked)
|
||||||
|
}
|
||||||
|
for _, c := range asked {
|
||||||
|
if c.args["runner"] != RunnerWorkspace {
|
||||||
|
t.Errorf("%s %v was asked with runner %v, want %s", c.tool, c.args["level"], c.args["runner"], RunnerWorkspace)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAMergeNotChangingTheGoBuildModuleIsAskedWithoutARunner(t *testing.T) { // 1898/AC1
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name, repo, path string
|
||||||
|
}{
|
||||||
|
{"another module of the catalogue", catalogueRepository, "modules/mesh-build-node/cmd/mesh-build-node/main.go"},
|
||||||
|
{"a look-alike path in another repository", repoA, buildGoFile},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
b := candidateBench(t, true)
|
||||||
|
b.fetch.Changed = []string{tc.path}
|
||||||
|
out := b.judge(t, caseOf(t, pullOf(tc.repo, 9, head1)))
|
||||||
|
for _, c := range append(b.steps.tool("digest"), b.steps.tool("test")...) {
|
||||||
|
if _, ok := c.args["runner"]; ok {
|
||||||
|
t.Errorf("%s was asked with runner %v", c.tool, c.args["runner"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(out.Judged) != 0 {
|
||||||
|
t.Errorf("judged = %v, want nothing said of the program", out.Judged)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnInstalledModuleWithoutTheRunnerJudgesAsTodayAndTheLineSaysSo(t *testing.T) { // 1898/AC2
|
||||||
|
b := candidateBench(t, false)
|
||||||
|
cs := caseOf(t, pullOf(catalogueRepository, 9, head1))
|
||||||
|
out := b.judge(t, cs)
|
||||||
|
if out.Verdict.State != delivery.VerdictPass {
|
||||||
|
t.Fatalf("verdict = %+v", out.Verdict)
|
||||||
|
}
|
||||||
|
// As today: the digests key kept verdicts, and the levels are asked without the runner the module ignored.
|
||||||
|
for _, c := range b.steps.tool("test") {
|
||||||
|
if _, ok := c.args["runner"]; ok {
|
||||||
|
t.Errorf("the %v level was asked again with the runner the module does not take", c.args["level"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if keys, _ := b.kept.Keys(context.Background()); len(keys) != 2 {
|
||||||
|
t.Errorf("kept %v, want the passing package at both levels, as today", keys)
|
||||||
|
}
|
||||||
|
p := cs.Pulls[0]
|
||||||
|
want := "judged by the installed mesh-build-go: it does not run a candidate yet"
|
||||||
|
if got := out.Judged[pullKey(p)]; got != want {
|
||||||
|
t.Fatalf("judged = %q, want %q", got, want)
|
||||||
|
}
|
||||||
|
if l := Line(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged); !strings.Contains(l, want) {
|
||||||
|
t.Errorf("line %q does not say %q", l, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestACandidateJudgedCheckTakesNoKeptVerdictAndWritesNone(t *testing.T) { // 1898/AC3
|
||||||
|
b := candidateBench(t, true)
|
||||||
|
if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digA, keptValue("fixture/slow", 2)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 9, head1)))
|
||||||
|
if out.Verdict.State != delivery.VerdictPass {
|
||||||
|
t.Fatalf("verdict = %+v", out.Verdict)
|
||||||
|
}
|
||||||
|
tests := b.steps.tool("test")
|
||||||
|
if len(tests) != 2 {
|
||||||
|
t.Fatalf("test calls = %+v, want both levels", tests)
|
||||||
|
}
|
||||||
|
for _, c := range tests {
|
||||||
|
if _, ok := c.args["packages"]; ok {
|
||||||
|
t.Errorf("the %v level was asked for %v, want it whole", c.args["level"], c.args["packages"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Kept != 0 {
|
||||||
|
t.Errorf("unit = %+v, want nothing taken", u)
|
||||||
|
}
|
||||||
|
if keys, _ := b.kept.Keys(context.Background()); len(keys) != 1 || keys[0] != "unit."+digA {
|
||||||
|
t.Errorf("kept keys = %v, want only the one kept before", keys)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheLineNamesTheCandidateThatJudged(t *testing.T) { // 1898/AC6
|
||||||
|
b := candidateBench(t, true)
|
||||||
|
cs := caseOf(t, pullOf(catalogueRepository, 9, head1))
|
||||||
|
out := b.judge(t, cs)
|
||||||
|
p := cs.Pulls[0]
|
||||||
|
want := "judged by the candidate's mesh-build-go at " + b.fetch.Merge[:12]
|
||||||
|
for name, l := range map[string]string{
|
||||||
|
"line": Line(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged),
|
||||||
|
"report": Report(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged),
|
||||||
|
} {
|
||||||
|
if !strings.Contains(l, want) {
|
||||||
|
t.Errorf("the %s %q does not say %q", name, l, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A long failing line keeps the program it was judged by; only the lesser parts go.
|
||||||
|
v := out.Verdict
|
||||||
|
v.State = delivery.VerdictFail
|
||||||
|
v.Failed = delivery.FailedFirst{Test: "unit: git.novox.be/novox/mesh-catalog/modules/mesh-build-go/cmd/mesh-build-go.TestADigestFoldsTheSiblings", Words: strings.Repeat("x", 60)}
|
||||||
|
if l := Line(v, cs, p, out.Builds, out.Slowest, out.Judged); !strings.Contains(l, "judged by the candidate's") {
|
||||||
|
t.Errorf("the long line %q dropped the program", l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInAGroupOnlyTheBuildModulesPullRequestNamesItsProgram(t *testing.T) { // 1898/AC6
|
||||||
|
b := candidateBench(t, true)
|
||||||
|
cs := caseOf(t, pullOf(catalogueRepository, 9, head1), pullOf(repoA, 7, head1))
|
||||||
|
out := b.judge(t, cs)
|
||||||
|
for _, p := range cs.Pulls {
|
||||||
|
l := Report(out.Verdict, cs, p, out.Builds, out.Slowest, out.Judged)
|
||||||
|
says := strings.Contains(l, "judged by")
|
||||||
|
if says != (p.Repository == catalogueRepository) {
|
||||||
|
t.Errorf("%s#%d: %q", p.Repository, p.Number, l)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestACandidateThatDoesNotCompileFailsTheCheckWithTheCompilersWords(t *testing.T) { // ADR 0535 §3, checker side
|
||||||
|
b := candidateBench(t, true)
|
||||||
|
refusal := "the candidate's mesh-build-go does not compile: modules/mesh-build-go/cmd/mesh-build-go/digest.go:12:2: undefined: x"
|
||||||
|
b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, errorString(refusal) }
|
||||||
|
b.test = func(stepCall) (any, error) { return nil, errorString(refusal) }
|
||||||
|
out := b.judge(t, caseOf(t, pullOf(catalogueRepository, 9, head1)))
|
||||||
|
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Words, "undefined: x") {
|
||||||
|
t.Fatalf("verdict = %+v, want a failure naming the compile error", out.Verdict)
|
||||||
|
}
|
||||||
|
if n := len(b.steps.tool("test")); n != 1 {
|
||||||
|
t.Errorf("%d test steps, want the first one's failure to end the check", n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type errorString string
|
||||||
|
|
||||||
|
func (e errorString) Error() string { return string(e) }
|
||||||
|
|
||||||
|
func TestTheStatusTheForgeShowsNamesTheProgramThatJudged(t *testing.T) { // 1898/AC6, the forge's line
|
||||||
|
ctx := context.Background()
|
||||||
|
words := judgedByCandidate + "fedcba987654"
|
||||||
|
j := &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) {
|
||||||
|
return Outcome{Verdict: passing(), Judged: map[string]string{repoA + "#7": words}}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
|
||||||
|
r.ch.Wait()
|
||||||
|
key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
|
||||||
|
raw, _, _, _ := r.owed.Get(ctx, key)
|
||||||
|
o, err := delivery.DecodeCheckerOwed(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if o.State != "success" || !strings.Contains(o.Text, words) {
|
||||||
|
t.Errorf("status %+v, want success saying %q", o, words)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -37,30 +37,38 @@ func firstFailed(v delivery.Verdict) string {
|
|||||||
f := v.Failed
|
f := v.Failed
|
||||||
switch {
|
switch {
|
||||||
case f.Test != "":
|
case f.Test != "":
|
||||||
if f.Words != "" {
|
if w := firstWordsLine(f.Words); w != "" {
|
||||||
return f.Test + ": " + f.Words
|
return f.Test + ": " + w
|
||||||
}
|
}
|
||||||
return f.Test
|
return f.Test
|
||||||
case f.Machine != "" && f.Words != "":
|
case f.Machine != "" && firstWordsLine(f.Words) != "":
|
||||||
return f.Machine + ": " + f.Words
|
return f.Machine + ": " + firstWordsLine(f.Words)
|
||||||
case f.Machine != "":
|
case f.Machine != "":
|
||||||
return f.Machine
|
return f.Machine
|
||||||
}
|
}
|
||||||
return f.Words
|
return firstWordsLine(f.Words)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// firstWordsLine is the first line of a failure's words: the words are several lines (issue 1901) and the status line one.
|
||||||
|
func firstWordsLine(w string) string { return firstLine(w) }
|
||||||
|
|
||||||
// pieces are the parts of a line, from the most to the least needed.
|
// pieces are the parts of a line, from the most to the least needed.
|
||||||
type pieces struct {
|
type pieces struct {
|
||||||
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, shortComposition, group string
|
||||||
|
// judged names the program that judged a change to a build module (ADR 0535 §5); never dropped.
|
||||||
|
judged string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pullKey names a pull request of a case: `<owner/repo>#<number>`.
|
||||||
|
func pullKey(p Pull) string { return fmt.Sprintf("%s#%d", p.Repository, p.Number) }
|
||||||
|
|
||||||
func (p pieces) join(drop int) string {
|
func (p pieces) join(drop int) string {
|
||||||
parts := []string{p.head, p.commits, p.counts}
|
parts := []string{p.head, p.commits, p.counts, p.judged}
|
||||||
switch {
|
switch {
|
||||||
case drop < 3 && p.times != "":
|
case drop < 3 && p.times != "":
|
||||||
parts = append(parts, p.times)
|
parts = append(parts, p.times)
|
||||||
case drop == 3 && p.shortTimes != "":
|
case drop == 3 && p.shortTimes != "":
|
||||||
parts = []string{p.head, p.commits, p.shortCounts, p.shortTimes}
|
parts = []string{p.head, p.commits, p.shortCounts, p.judged, p.shortTimes}
|
||||||
}
|
}
|
||||||
if p.over != "" {
|
if p.over != "" {
|
||||||
parts = append(parts, p.over)
|
parts = append(parts, p.over)
|
||||||
@@ -77,13 +85,7 @@ func (p pieces) join(drop int) string {
|
|||||||
case p.composition != "":
|
case p.composition != "":
|
||||||
parts = append(parts, p.composition)
|
parts = append(parts, p.composition)
|
||||||
}
|
}
|
||||||
out := make([]string, 0, len(parts))
|
return joinSet(parts...)
|
||||||
for _, s := range parts {
|
|
||||||
if s != "" {
|
|
||||||
out = append(out, s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return strings.Join(out, " | ")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func levelWords(name string, r delivery.LevelResult) string {
|
func levelWords(name string, r delivery.LevelResult) string {
|
||||||
@@ -100,6 +102,9 @@ func compositionWords(c *delivery.Composition, took int64) string { return compo
|
|||||||
|
|
||||||
// composition is compositionWords; short counts the machines and leaves the SDK commit to the report.
|
// composition is compositionWords; short counts the machines and leaves the SDK commit to the report.
|
||||||
func composition(c *delivery.Composition, took int64, short bool) string {
|
func composition(c *delivery.Composition, took int64, short bool) string {
|
||||||
|
if c != nil && c.State == delivery.CompositionNotRun && c.Words != "" {
|
||||||
|
return "composition not judged: " + c.Words
|
||||||
|
}
|
||||||
if c == nil || c.State == delivery.CompositionNotRun {
|
if c == nil || c.State == delivery.CompositionNotRun {
|
||||||
return "composition not yet run"
|
return "composition not yet run"
|
||||||
}
|
}
|
||||||
@@ -139,8 +144,9 @@ func buildWords(builds []BuildNote) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parts makes the pieces of the line of a pull request of a case.
|
// parts makes the pieces of the line of a pull request of a case.
|
||||||
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) pieces {
|
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) pieces {
|
||||||
var pc pieces
|
var pc pieces
|
||||||
|
pc.judged = judged[pullKey(p)]
|
||||||
commits := short12(p.Head) + " on " + short12(p.Trunk)
|
commits := short12(p.Head) + " on " + short12(p.Trunk)
|
||||||
switch v.State {
|
switch v.State {
|
||||||
case delivery.VerdictRunning:
|
case delivery.VerdictRunning:
|
||||||
@@ -176,7 +182,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
|||||||
for _, x := range []struct {
|
for _, x := range []struct {
|
||||||
name string
|
name string
|
||||||
ms int64
|
ms int64
|
||||||
}{{"queue", t.Queue}, {"clone", t.Clone}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}} {
|
}{{"queue", t.Queue}, {"clone", t.Clone}, {"lint", t.Lint}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}} {
|
||||||
if x.ms > 0 {
|
if x.ms > 0 {
|
||||||
times = append(times, x.name+" "+seconds(x.ms))
|
times = append(times, x.name+" "+seconds(x.ms))
|
||||||
}
|
}
|
||||||
@@ -189,6 +195,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
|||||||
if t.Clone > 0 {
|
if t.Clone > 0 {
|
||||||
short = append(short, "clone "+seconds(t.Clone))
|
short = append(short, "clone "+seconds(t.Clone))
|
||||||
}
|
}
|
||||||
|
if t.Lint > 0 {
|
||||||
|
short = append(short, "lint "+seconds(t.Lint))
|
||||||
|
}
|
||||||
if t.Unit > 0 {
|
if t.Unit > 0 {
|
||||||
short = append(short, "unit "+seconds(t.Unit))
|
short = append(short, "unit "+seconds(t.Unit))
|
||||||
}
|
}
|
||||||
@@ -221,22 +230,48 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Report is the full line of a pull request's status, with no limit: what the read tools and the full report say.
|
// Report is the full line of a pull request's status, with no limit: what the read tools and the full report say.
|
||||||
func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string {
|
func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
|
||||||
return parts(v, cs, p, builds, slowest).join(0)
|
return parts(v, cs, p, builds, slowest, judged).join(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7): the verdict first (a
|
// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7): the verdict first (a
|
||||||
// failure names the conflict, the refused step or the test before anything), the head and the trunk commit as twelve
|
// failure names the conflict, the refused step or the test before anything), the head and the trunk commit as twelve
|
||||||
// characters each, the tests run, kept and skipped per level, the time of each part, a level over its bound with its slowest
|
// characters each, the tests run, kept and skipped per level, the time of each part, a level over its bound with its slowest
|
||||||
// tests, the builds of the touched modules, and that the composition was not yet run. It is at most MaxLine characters: the
|
// tests, the builds of the touched modules, and that the composition was not yet run; for a change to a build module, the
|
||||||
|
// program that judged it (judged, by pullKey; ADR 0535 §5). It is at most MaxLine characters: the
|
||||||
// builds, then the group drop first, then the times are given in their compact form, then dropped, and the line is cut at last.
|
// builds, then the group drop first, then the times are given in their compact form, then dropped, and the line is cut at last.
|
||||||
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string {
|
// The program that judged is never dropped: a line naming it gives the counts in their compact form, then drops them and
|
||||||
pc := parts(v, cs, p, builds, slowest)
|
// what follows, and is cut in its first part rather than lose it.
|
||||||
|
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string, judged map[string]string) string {
|
||||||
|
pc := parts(v, cs, p, builds, slowest, judged)
|
||||||
for drop := 0; drop <= 4; drop++ {
|
for drop := 0; drop <= 4; drop++ {
|
||||||
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
|
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if pc.judged != "" {
|
||||||
|
for _, counts := range []string{pc.shortCounts, ""} {
|
||||||
|
if s := joinSet(pc.head, pc.commits, counts, pc.judged); utf8.RuneCountInString(s) <= MaxLine {
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
rest := " | " + joinSet(pc.commits, pc.judged)
|
||||||
|
room := MaxLine - utf8.RuneCountInString(rest) - 1
|
||||||
|
if head := []rune(pc.head); room > 0 && len(head) > room {
|
||||||
|
return string(head[:room]) + "…" + rest
|
||||||
|
}
|
||||||
|
}
|
||||||
runes := []rune(pc.join(4))
|
runes := []rune(pc.join(4))
|
||||||
return string(runes[:MaxLine-1]) + "…"
|
return string(runes[:MaxLine-1]) + "…"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// joinSet joins the parts that are not empty.
|
||||||
|
func joinSet(parts ...string) string {
|
||||||
|
out := make([]string, 0, len(parts))
|
||||||
|
for _, s := range parts {
|
||||||
|
if s != "" {
|
||||||
|
out = append(out, s)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(out, " | ")
|
||||||
|
}
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ func passVerdict() delivery.Verdict {
|
|||||||
|
|
||||||
func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCompositionWasNotRun(t *testing.T) { // 1300/AC1
|
func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCompositionWasNotRun(t *testing.T) { // 1300/AC1
|
||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
full := Report(passVerdict(), cs, p, nil, nil)
|
full := Report(passVerdict(), cs, p, nil, nil, nil)
|
||||||
for _, want := range []string{"pass 111111111111 on aaaaaaaaaaaa", "unit 12 run, 301 kept", "integration 3 run, 40 kept", "queue 800ms", "clone 4.1s", "unit 400ms", "int setup 3.2s", "int run 12s", "composition not yet run"} {
|
for _, want := range []string{"pass 111111111111 on aaaaaaaaaaaa", "unit 12 run, 301 kept", "integration 3 run, 40 kept", "queue 800ms", "clone 4.1s", "unit 400ms", "int setup 3.2s", "int run 12s", "composition not yet run"} {
|
||||||
if !strings.Contains(full, want) {
|
if !strings.Contains(full, want) {
|
||||||
t.Errorf("the report %q lacks %q", full, want)
|
t.Errorf("the report %q lacks %q", full, want)
|
||||||
@@ -31,7 +31,7 @@ func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCom
|
|||||||
if !strings.HasPrefix(full, "pass ") || strings.Contains(full, "composition passed") {
|
if !strings.HasPrefix(full, "pass ") || strings.Contains(full, "composition passed") {
|
||||||
t.Errorf("report = %q", full)
|
t.Errorf("report = %q", full)
|
||||||
}
|
}
|
||||||
line := Line(passVerdict(), cs, p, nil, nil)
|
line := Line(passVerdict(), cs, p, nil, nil, nil)
|
||||||
if n := utf8.RuneCountInString(line); n > MaxLine {
|
if n := utf8.RuneCountInString(line); n > MaxLine {
|
||||||
t.Errorf("the line is %d characters, want at most %d: %q", n, MaxLine, line)
|
t.Errorf("the line is %d characters, want at most %d: %q", n, MaxLine, line)
|
||||||
}
|
}
|
||||||
@@ -51,7 +51,7 @@ func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { //
|
|||||||
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
|
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
|
||||||
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
|
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
|
||||||
} {
|
} {
|
||||||
line := Line(c.v, cs, p, nil, nil)
|
line := Line(c.v, cs, p, nil, nil, nil)
|
||||||
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
|
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
|
||||||
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
|
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
|
||||||
}
|
}
|
||||||
@@ -62,8 +62,8 @@ func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndTheVerdictIsNotChanged(t
|
|||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
v := passVerdict()
|
v := passVerdict()
|
||||||
v.OverBound = []string{"unit"}
|
v.OverBound = []string{"unit"}
|
||||||
line := Line(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"})
|
line := Line(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}, nil)
|
||||||
if !strings.HasPrefix(line, "pass ") || !strings.Contains(Report(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}), "over bound: unit (slowest unit pkg.TestSlow 7.0s)") {
|
if !strings.HasPrefix(line, "pass ") || !strings.Contains(Report(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}, nil), "over bound: unit (slowest unit pkg.TestSlow 7.0s)") {
|
||||||
t.Errorf("line = %q", line)
|
t.Errorf("line = %q", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -72,16 +72,16 @@ func TestSkippedTestsAreSaidAndNeverAFlatPass(t *testing.T) { // 1300/AC6
|
|||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
v := passVerdict()
|
v := passVerdict()
|
||||||
v.Levels[delivery.LevelUnit] = delivery.LevelResult{Run: 12, Kept: 301, Skipped: 2}
|
v.Levels[delivery.LevelUnit] = delivery.LevelResult{Run: 12, Kept: 301, Skipped: 2}
|
||||||
if line := Line(v, cs, p, nil, nil); !strings.Contains(line, "unit 12 run, 301 kept, 2 skipped") {
|
if line := Line(v, cs, p, nil, nil, nil); !strings.Contains(line, "unit 12 run, 301 kept, 2 skipped") {
|
||||||
t.Errorf("line = %q", line)
|
t.Errorf("line = %q", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4
|
func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4
|
||||||
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
|
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
|
||||||
a := Line(passVerdict(), cs, cs.Pulls[0], nil, nil)
|
a := Line(passVerdict(), cs, cs.Pulls[0], nil, nil, nil)
|
||||||
b := Line(passVerdict(), cs, cs.Pulls[1], nil, nil)
|
b := Line(passVerdict(), cs, cs.Pulls[1], nil, nil, nil)
|
||||||
if !strings.HasPrefix(a, "pass ") || !strings.HasPrefix(b, "pass ") || !strings.Contains(Report(passVerdict(), cs, cs.Pulls[0], nil, nil), "group feat/x of 2 pull requests") {
|
if !strings.HasPrefix(a, "pass ") || !strings.HasPrefix(b, "pass ") || !strings.Contains(Report(passVerdict(), cs, cs.Pulls[0], nil, nil, nil), "group feat/x of 2 pull requests") {
|
||||||
t.Errorf("lines %q and %q", a, b)
|
t.Errorf("lines %q and %q", a, b)
|
||||||
}
|
}
|
||||||
if !strings.Contains(a, "111111111111") || !strings.Contains(b, "222222222222") {
|
if !strings.Contains(a, "111111111111") || !strings.Contains(b, "222222222222") {
|
||||||
@@ -92,16 +92,16 @@ func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4
|
|||||||
func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(t *testing.T) { // ADR 0484 clause 7
|
func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(t *testing.T) { // ADR 0484 clause 7
|
||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
builds := []BuildNote{{"avahi", "built"}, {"mesh-tools", "waiting"}}
|
builds := []BuildNote{{"avahi", "built"}, {"mesh-tools", "waiting"}}
|
||||||
if r := Report(passVerdict(), cs, p, builds, nil); !strings.Contains(r, "builds: avahi built, mesh-tools waiting") {
|
if r := Report(passVerdict(), cs, p, builds, nil, nil); !strings.Contains(r, "builds: avahi built, mesh-tools waiting") {
|
||||||
t.Errorf("report = %q", r)
|
t.Errorf("report = %q", r)
|
||||||
}
|
}
|
||||||
line := Line(passVerdict(), cs, p, builds, nil)
|
line := Line(passVerdict(), cs, p, builds, nil, nil)
|
||||||
if utf8.RuneCountInString(line) > MaxLine {
|
if utf8.RuneCountInString(line) > MaxLine {
|
||||||
t.Errorf("line %q is too long", line)
|
t.Errorf("line %q is too long", line)
|
||||||
}
|
}
|
||||||
short := passVerdict()
|
short := passVerdict()
|
||||||
short.Took = delivery.Took{}
|
short.Took = delivery.Took{}
|
||||||
if l := Line(short, cs, p, builds[:1], nil); !strings.Contains(l, "builds: avahi built") {
|
if l := Line(short, cs, p, builds[:1], nil, nil); !strings.Contains(l, "builds: avahi built") {
|
||||||
t.Errorf("a short line keeps its builds: %q", l)
|
t.Errorf("a short line keeps its builds: %q", l)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -109,14 +109,14 @@ func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(
|
|||||||
func TestALineIsNeverLongerThanTheLimit(t *testing.T) { // 1300/AC1
|
func TestALineIsNeverLongerThanTheLimit(t *testing.T) { // 1300/AC1
|
||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + ".TestSomething", Words: strings.Repeat("because ", 40)}}
|
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + ".TestSomething", Words: strings.Repeat("because ", 40)}}
|
||||||
if line := Line(v, cs, p, nil, nil); utf8.RuneCountInString(line) > MaxLine || !strings.HasPrefix(line, "failed: integration: ") {
|
if line := Line(v, cs, p, nil, nil, nil); utf8.RuneCountInString(line) > MaxLine || !strings.HasPrefix(line, "failed: integration: ") {
|
||||||
t.Errorf("line = %q (%d)", line, utf8.RuneCountInString(line))
|
t.Errorf("line = %q (%d)", line, utf8.RuneCountInString(line))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestARunningCheckIsPending(t *testing.T) {
|
func TestARunningCheckIsPending(t *testing.T) {
|
||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
if line := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil); line != "checking 111111111111 on aaaaaaaaaaaa" {
|
if line := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil, nil); line != "checking 111111111111 on aaaaaaaaaaaa" {
|
||||||
t.Errorf("line = %q", line)
|
t.Errorf("line = %q", line)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -125,11 +125,11 @@ func TestThePassedCompositionIsSaidOnlyWhenItWasRun(t *testing.T) { // ADR 0484
|
|||||||
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
cs, p := lineCase(t, pullOf(repoA, 7, head1))
|
||||||
v := passVerdict()
|
v := passVerdict()
|
||||||
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab"}
|
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab"}
|
||||||
if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition passed with afbe371003a5") {
|
if r := Report(v, cs, p, nil, nil, nil); !strings.Contains(r, "composition passed with afbe371003a5") {
|
||||||
t.Errorf("report = %q", r)
|
t.Errorf("report = %q", r)
|
||||||
}
|
}
|
||||||
v.Composition = nil
|
v.Composition = nil
|
||||||
if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition not yet run") || strings.Contains(r, "composition passed") {
|
if r := Report(v, cs, p, nil, nil, nil); !strings.Contains(r, "composition not yet run") || strings.Contains(r, "composition passed") {
|
||||||
t.Errorf("a verdict with no composition: %q", r)
|
t.Errorf("a verdict with no composition: %q", r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -139,18 +139,41 @@ func TestTheCompositionsTimeAndMachinesAreSaidOnceTheAssignerReportsThem(t *test
|
|||||||
v := passVerdict()
|
v := passVerdict()
|
||||||
v.Took.Composition = 2400
|
v.Took.Composition = 2400
|
||||||
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab", Machines: []string{"g14", "ace", "novox", "shanks"}}
|
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab", Machines: []string{"g14", "ace", "novox", "shanks"}}
|
||||||
r := Report(v, cs, p, nil, nil)
|
r := Report(v, cs, p, nil, nil, nil)
|
||||||
for _, want := range []string{"composition passed in 2.4s with afbe371003a5 on 4 machines: ace, g14, novox, shanks"} {
|
for _, want := range []string{"composition passed in 2.4s with afbe371003a5 on 4 machines: ace, g14, novox, shanks"} {
|
||||||
if !strings.Contains(r, want) {
|
if !strings.Contains(r, want) {
|
||||||
t.Errorf("report %q lacks %q", r, want)
|
t.Errorf("report %q lacks %q", r, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if l := Line(v, cs, p, nil, nil); !strings.Contains(l, "composition passed in 2.4s") || !strings.Contains(l, "on 4 machines") || utf8.RuneCountInString(l) > MaxLine {
|
if l := Line(v, cs, p, nil, nil, nil); !strings.Contains(l, "composition passed in 2.4s") || !strings.Contains(l, "on 4 machines") || utf8.RuneCountInString(l) > MaxLine {
|
||||||
t.Errorf("line = %q", l)
|
t.Errorf("line = %q", l)
|
||||||
}
|
}
|
||||||
v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: "novox", Words: "composition refused"}
|
v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: "novox", Words: "composition refused"}
|
||||||
v.Composition = &delivery.Composition{State: delivery.CompositionFail, Words: "novox: two seats claim dns", Machines: []string{"novox"}}
|
v.Composition = &delivery.Composition{State: delivery.CompositionFail, Words: "novox: two seats claim dns", Machines: []string{"novox"}}
|
||||||
if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition failed in 2.4s on novox: novox: two seats claim dns") {
|
if r := Report(v, cs, p, nil, nil, nil); !strings.Contains(r, "composition failed in 2.4s on novox: novox: two seats claim dns") {
|
||||||
t.Errorf("failed composition report = %q", r)
|
t.Errorf("failed composition report = %q", r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1887/D7-go, review round 2 item 5: the line says the unit level's gofmt and go vet time beside its tests'.
|
||||||
|
func TestTheLineSaysTheLintTime(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
test := b.test
|
||||||
|
b.test = func(c stepCall) (any, error) {
|
||||||
|
r, err := test(c)
|
||||||
|
if res, ok := r.(Tested); ok && res.Level == delivery.LevelUnit {
|
||||||
|
res.Lint = 1.5
|
||||||
|
return res, err
|
||||||
|
}
|
||||||
|
return r, err
|
||||||
|
}
|
||||||
|
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
|
if out.Verdict.Took.Lint != 1500 {
|
||||||
|
t.Fatalf("took = %+v, want the lint's 1.5 s", out.Verdict.Took)
|
||||||
|
}
|
||||||
|
cs := caseOf(t, pullOf(repoA, 7, head1))
|
||||||
|
pc := parts(out.Verdict, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||||
|
if !strings.Contains(pc.times, "lint 1.5s") || !strings.Contains(pc.shortTimes, "lint 1.5s") {
|
||||||
|
t.Errorf("times = %q, short %q, want the lint's time in both forms of the line", pc.times, pc.shortTimes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -55,19 +55,19 @@ func SDKPin(goMod string) (commit string, ok bool) {
|
|||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
// checkPins refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the
|
// checkPinsOf refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the
|
||||||
// group's mesh-sdk pull request. It reads each go.mod the fetch named from the forge: the one the check builds, which is the
|
// group's mesh-sdk pull request. It reads each go.mod the fetch named from the forge: the one the check builds, which is the
|
||||||
// head's when the head holds the file and the trunk's when it does not, since a module added on the trunk after the branch was
|
// head's when the head holds the file and the trunk's when it does not, since a module added on the trunk after the branch was
|
||||||
// cut is in the merge and not at the head (issue 1845). It answers nil when every pin is the group's.
|
// cut is in the merge and not at the head (issue 1845). It answers nil when every pin is the group's.
|
||||||
func (st *state) checkPins(ctx context.Context) verdictPtr {
|
func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr {
|
||||||
if st.j.Forge == nil {
|
if st.j.Forge == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
sdk := st.c.member(SDKRepository)
|
sdk := st.c.member(SDKRepository)
|
||||||
for _, t := range st.targets {
|
if t.module != goModule || t.pull.Repository == SDKRepository {
|
||||||
if t.module != goModule || t.pull.Repository == SDKRepository {
|
return nil
|
||||||
continue
|
}
|
||||||
}
|
{
|
||||||
for _, dir := range t.fetched.Modules {
|
for _, dir := range t.fetched.Modules {
|
||||||
file := path.Join(dir, "go.mod")
|
file := path.Join(dir, "go.mod")
|
||||||
st.c.Progress.Set(fmt.Sprintf("read %s of %s#%d", file, t.pull.Repository, t.pull.Number), t.node)
|
st.c.Progress.Set(fmt.Sprintf("read %s of %s#%d", file, t.pull.Repository, t.pull.Number), t.node)
|
||||||
|
|||||||
@@ -97,7 +97,7 @@ func TestAMemberPinnedToTheSDKMembersHeadPassesAndIsBuiltAgainstIt(t *testing.T)
|
|||||||
var besideSDK bool
|
var besideSDK bool
|
||||||
for _, f := range b.steps.tool("fetch") {
|
for _, f := range b.steps.tool("fetch") {
|
||||||
if f.args["repository"] == repoA {
|
if f.args["repository"] == repoA {
|
||||||
refs, _ := f.args["beside"].([]BesideRef)
|
refs, _ := refsOf(f.args["beside"]), true
|
||||||
for _, r := range refs {
|
for _, r := range refs {
|
||||||
besideSDK = besideSDK || (r.Repository == repoSDK && r.Commit == head2)
|
besideSDK = besideSDK || (r.Repository == repoSDK && r.Commit == head2)
|
||||||
}
|
}
|
||||||
@@ -116,9 +116,8 @@ func TestAMemberPinnedToAnOlderHeadOfTheSDKMemberFailsNamingTheMemberAndTheCommi
|
|||||||
!strings.Contains(f.Words, "neither on mesh-sdk's trunk nor the group's head") || !strings.Contains(f.Words, short12(head2)) {
|
!strings.Contains(f.Words, "neither on mesh-sdk's trunk nor the group's head") || !strings.Contains(f.Words, short12(head2)) {
|
||||||
t.Fatalf("verdict = %+v", v)
|
t.Fatalf("verdict = %+v", v)
|
||||||
}
|
}
|
||||||
if n := len(b.steps.tool("test")); n != 0 {
|
// Since R4g the levels run in the member's one check ask before its go.mod pins are judged (ADR 0525 §3); the verdict is
|
||||||
t.Errorf("%d test steps ran after a stale pin", n)
|
// still the pin's, whatever the levels said.
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAMemberPinnedToACommitOnTheTrunkPasses(t *testing.T) { // 1624/AC2, AC4
|
func TestAMemberPinnedToACommitOnTheTrunkPasses(t *testing.T) { // 1624/AC2, AC4
|
||||||
@@ -213,7 +212,7 @@ func besideOf(t *testing.T, b *bench, repository string) map[string]BesideRef {
|
|||||||
out := map[string]BesideRef{}
|
out := map[string]BesideRef{}
|
||||||
for _, f := range b.steps.tool("fetch") {
|
for _, f := range b.steps.tool("fetch") {
|
||||||
if f.args["repository"] == repository {
|
if f.args["repository"] == repository {
|
||||||
refs, _ := f.args["beside"].([]BesideRef)
|
refs, _ := refsOf(f.args["beside"]), true
|
||||||
for _, r := range refs {
|
for _, r := range refs {
|
||||||
out[r.Repository] = r
|
out[r.Repository] = r
|
||||||
}
|
}
|
||||||
@@ -228,7 +227,8 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
|||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"})
|
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"})
|
||||||
b.ex.Runs = runsAt{"mesh-host": hostRuns}
|
b.ex.Runs = runsAt{"mesh-host": hostRuns}
|
||||||
forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865")}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865"),
|
||||||
|
catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}}
|
||||||
b.ex.Forge = forge
|
b.ex.Forge = forge
|
||||||
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
|
||||||
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
|
||||||
@@ -244,6 +244,39 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1295/R4g: the node-engine's tests read the controller's manifest, so a mesh-host pull request has mesh-controller beside it:
|
||||||
|
// at the group member's head when the group holds one, else at its trunk.
|
||||||
|
func hostCase(t *testing.T, pulls ...delivery.CandidatePull) *Case {
|
||||||
|
t.Helper()
|
||||||
|
c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pulls...))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cs := CaseOf(c, map[string]string{"novox/mesh-host": trunkA, controllerRepo: trunkA2})
|
||||||
|
if cs == nil {
|
||||||
|
t.Fatal("no case")
|
||||||
|
}
|
||||||
|
return cs
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestTheNodeEnginesTestsHaveTheControllerBesideThem(t *testing.T) {
|
||||||
|
const host = "novox/mesh-host"
|
||||||
|
b := newBench(t)
|
||||||
|
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
|
||||||
|
b.judge(t, hostCase(t, pullOf(host, 5, head1)))
|
||||||
|
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != trunkA2 || c.Ref != "" {
|
||||||
|
t.Errorf("the controller beside the node-engine alone = %+v, want its trunk %s", c, trunkA2)
|
||||||
|
}
|
||||||
|
b = newBench(t)
|
||||||
|
b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS})
|
||||||
|
b.ex.Forge = &fakeForge{goMods: map[string]string{controllerRepo + " go.mod " + head2: capturedGoMod(t, "cb54756c3865")},
|
||||||
|
onTrunk: map[string]bool{"cb54756c3865": true}}
|
||||||
|
b.judge(t, hostCase(t, pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2)))
|
||||||
|
if c := besideOf(t, b, host)[controllerRepo]; c.Commit != head2 || c.Ref != "refs/pull/9/head" {
|
||||||
|
t.Errorf("the controller beside the node-engine in a group = %+v, want the member's head %s", c, head2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2
|
func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA})
|
b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA})
|
||||||
@@ -265,3 +298,51 @@ func TestAForgeThatIsNotHeardFailsAtTheForgeAndIsJudgedAgainWhenTheNodesChange(t
|
|||||||
t.Errorf("a forge that was not heard is not the change's failure: %+v is not judged again when the nodes change", v.Failed)
|
t.Errorf("a forge that was not heard is not the change's failure: %+v is not judged again when the nodes change", v.Failed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1887/A12: the change's own repository is placed beside it at the trunk it is merged onto, for every repository, so a
|
||||||
|
// shrink-only list test reads the trunk's copy at $MESH_CHECK_BESIDE/<name>.
|
||||||
|
func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
|
||||||
|
got := besideOf(t, b, repoA)
|
||||||
|
if own := got[repoA]; own.Commit != trunkA || own.Ref != "" {
|
||||||
|
t.Fatalf("the repository beside its own change = %+v, want its trunk %s", own, trunkA)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk and
|
||||||
|
// mesh-controller, so the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog.
|
||||||
|
func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) {
|
||||||
|
b := newBench(t)
|
||||||
|
b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2})
|
||||||
|
b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1)))
|
||||||
|
got := besideOf(t, b, catalogueRepository)
|
||||||
|
if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" {
|
||||||
|
t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA)
|
||||||
|
}
|
||||||
|
if _, ok := got[SDKRepository]; !ok {
|
||||||
|
t.Errorf("mesh-sdk is no longer beside a catalogue change: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/A12: a mesh-lab head gets a check: with its trunk known it is judged like any repository, its Go fetch finding
|
||||||
|
// the nested modules itself.
|
||||||
|
func TestAMeshLabHeadGetsACheck(t *testing.T) {
|
||||||
|
const lab = "novox/mesh-lab"
|
||||||
|
raw := candidateOf(t, "fix/x", pullOf(lab, 3, head1))
|
||||||
|
c, err := delivery.DecodeCandidate(raw)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cs := CaseOf(c, map[string]string{lab: trunkA})
|
||||||
|
if cs == nil || len(cs.Pulls) != 1 || cs.Pulls[0].Repository != lab || len(cs.Left) != 0 {
|
||||||
|
t.Fatalf("the case = %+v, want mesh-lab#3 judged", cs)
|
||||||
|
}
|
||||||
|
b := newBench(t)
|
||||||
|
if v := b.judge(t, cs).Verdict; v.State != delivery.VerdictPass {
|
||||||
|
t.Fatalf("verdict = %+v", v)
|
||||||
|
}
|
||||||
|
if fetched := b.steps.tool("fetch"); len(fetched) == 0 || fetched[0].args["repository"] != lab {
|
||||||
|
t.Fatalf("mesh-lab was not fetched: %+v", fetched)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package check
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The checker's ranked start of checks (ADR 0536 §4, issue 1897): a freed place goes to a head never judged (or asked again
|
||||||
|
// after a failure of the nodes) before a head asked again only because its trunk moved; among equals the oldest asked first.
|
||||||
|
// A running check is never stopped.
|
||||||
|
|
||||||
|
// rankRig is a checker of one check at a time whose judge records the order checks start in and holds the first check
|
||||||
|
// after hold is set until release is closed.
|
||||||
|
type rankRig struct {
|
||||||
|
*rig
|
||||||
|
mu sync.Mutex
|
||||||
|
started []*Case
|
||||||
|
hold bool
|
||||||
|
held chan struct{} // closed when the held check started
|
||||||
|
release chan struct{}
|
||||||
|
errs []error // ctx errors the held check saw when released
|
||||||
|
}
|
||||||
|
|
||||||
|
func group(n int) string { return fmt.Sprintf("feat/g%d", n) }
|
||||||
|
|
||||||
|
func groups(cs []*Case) []string {
|
||||||
|
out := make([]string, len(cs))
|
||||||
|
for i, c := range cs {
|
||||||
|
out[i] = c.Group
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4
|
||||||
|
ctx := context.Background()
|
||||||
|
j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) {
|
||||||
|
if c.Group == group(2) {
|
||||||
|
return Outcome{Verdict: failure("the forge", "go.mod cannot be read: no responders (the forge was "+UnheardWords+")")}, nil
|
||||||
|
}
|
||||||
|
return Outcome{Verdict: passing()}, nil
|
||||||
|
}}
|
||||||
|
r := newRig(t, j)
|
||||||
|
cands := fakeState{}
|
||||||
|
put := func(n int, head string) {
|
||||||
|
cand := candidateOf(t, group(n), pullOf(repoA, n, head))
|
||||||
|
ckey, _ := delivery.CandidatesKey(group(n))
|
||||||
|
cands[ckey] = cand
|
||||||
|
r.ch.Candidate(ctx, group(n), cand)
|
||||||
|
}
|
||||||
|
put(1, head1) // judged on the old trunk: a re-judge waits
|
||||||
|
put(2, head1) // failed for the nodes: waits for them
|
||||||
|
r.ch.Wait()
|
||||||
|
r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS})
|
||||||
|
put(4, head1) // judged on the new trunk: stands
|
||||||
|
r.ch.Wait()
|
||||||
|
cand3 := candidateOf(t, group(3), pullOf(repoA, 3, head1)) // never judged
|
||||||
|
ckey3, _ := delivery.CandidatesKey(group(3))
|
||||||
|
cands[ckey3] = cand3
|
||||||
|
verdicts := fakeState{}
|
||||||
|
for k, v := range r.verdicts.vals {
|
||||||
|
verdicts[k] = v
|
||||||
|
}
|
||||||
|
rd := Reader{CandidateState: cands.read(), VerdictState: verdicts.read(),
|
||||||
|
Trunks: func() (map[string]string, error) { return map[string]string{repoA: trunkA2, repoSDK: trunkS}, nil }}
|
||||||
|
st, err := rd.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
want := map[string][]string{KindNew: {group(3)}, KindNodes: {group(2)}, KindTrunk: {group(1)}}
|
||||||
|
if fmt.Sprint(st.Waiting) != fmt.Sprint(want) {
|
||||||
|
t.Errorf("waiting = %v, want %v", st.Waiting, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,9 +5,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
|
||||||
@@ -18,7 +18,7 @@ const (
|
|||||||
VerdictsReadTool = "checker_verdicts"
|
VerdictsReadTool = "checker_verdicts"
|
||||||
PullReadTool = "checker_pull"
|
PullReadTool = "checker_pull"
|
||||||
KeptReadTool = "checker_kept"
|
KeptReadTool = "checker_kept"
|
||||||
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685).
|
// PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker runs (issue 1685: nothing holds it since R4g).
|
||||||
PreviewsReadTool = "checker_previews"
|
PreviewsReadTool = "checker_previews"
|
||||||
StatusReadTool = "checker_status"
|
StatusReadTool = "checker_status"
|
||||||
)
|
)
|
||||||
@@ -33,10 +33,13 @@ type StateRead struct {
|
|||||||
type Reader struct {
|
type Reader struct {
|
||||||
VerdictState, KeptState, CandidateState StateRead
|
VerdictState, KeptState, CandidateState StateRead
|
||||||
|
|
||||||
PreviewState StateRead // the checker's merge previews
|
PreviewState StateRead // the checker's merge previews
|
||||||
PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent
|
StepStates map[string]StateRead // the build modules' steps states, by module
|
||||||
StepStates map[string]StateRead // the build modules' steps states, by module
|
HaveState StateRead // the node-engine's have: whether a build module is on a node and running
|
||||||
Capacity func() (string, error) // the capacity setting as written, `node=steps,...`
|
StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0
|
||||||
|
// Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move.
|
||||||
|
Trunks func() (map[string]string, error)
|
||||||
|
Now func() time.Time
|
||||||
}
|
}
|
||||||
|
|
||||||
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
|
||||||
@@ -78,7 +81,7 @@ func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
for _, p := range cs.Pulls {
|
for _, p := range cs.Pulls {
|
||||||
out.Reports[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = Report(v, cs, p, nil, nil)
|
out.Reports[pullKey(p)] = Report(v, cs, p, nil, nil, nil)
|
||||||
}
|
}
|
||||||
for _, l := range cs.Left {
|
for _, l := range cs.Left {
|
||||||
out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
|
out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
|
||||||
@@ -119,8 +122,8 @@ func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request,
|
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds
|
||||||
// that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does.
|
// in no candidate, that none does.
|
||||||
func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
||||||
if r.CandidateState.Keys == nil || r.CandidateState.Get == nil {
|
if r.CandidateState.Keys == nil || r.CandidateState.Get == nil {
|
||||||
return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here")
|
return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here")
|
||||||
@@ -143,9 +146,6 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
|
|||||||
if p.Repository != repository || p.Number != number {
|
if p.Repository != repository || p.Number != number {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if repository == HQRepository {
|
|
||||||
return VerdictRead{Key: k, Note: HQNote}, nil
|
|
||||||
}
|
|
||||||
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return VerdictRead{}, err
|
return VerdictRead{}, err
|
||||||
@@ -254,48 +254,42 @@ type RunningStep struct {
|
|||||||
Since string `json:"since,omitempty"`
|
Since string `json:"since,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in
|
// LeftStep is a key a build module left running that no step behind it runs any more, and why.
|
||||||
// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a
|
type LeftStep struct {
|
||||||
// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters.
|
RunningStep
|
||||||
type Status struct {
|
Why string `json:"why"`
|
||||||
Capacity map[string]int `json:"capacity"`
|
|
||||||
CapacityNote string `json:"capacity_note,omitempty"`
|
|
||||||
DefaultPerNode int `json:"default_per_node"`
|
|
||||||
PlansBuilding struct {
|
|
||||||
Readable bool `json:"readable"`
|
|
||||||
Keys []string `json:"keys"`
|
|
||||||
Holds bool `json:"holds"`
|
|
||||||
} `json:"plans_building"`
|
|
||||||
Running []RunningStep `json:"running_steps"`
|
|
||||||
RunningByNode map[string]int `json:"running_by_node"`
|
|
||||||
Unjudged []string `json:"unjudged_candidates"`
|
|
||||||
Notes []string `json:"notes,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts.
|
// Status is what runs now, read from the buckets: the steps in flight on the build modules and the candidates without a
|
||||||
|
// verdict for their heads. Nothing holds a check since R4g: a check is an ask on the build seats' streams, whose backlog
|
||||||
|
// the delivery view shows (ADR 0525 §9).
|
||||||
|
type Status struct {
|
||||||
|
Running []RunningStep `json:"running_steps"`
|
||||||
|
LeftRunning []LeftStep `json:"left_running"`
|
||||||
|
RunningByNode map[string]int `json:"running_by_node"`
|
||||||
|
Unjudged []string `json:"unjudged_candidates"`
|
||||||
|
// Waiting are the candidates whose check waits, by kind (ADR 0536 §4): new (no verdict for its heads) and nodes (failed
|
||||||
|
// for the nodes, asked again when they change) start before trunk (judged on trunk heads that moved since).
|
||||||
|
Waiting map[string][]string `json:"waiting_by_kind"`
|
||||||
|
Notes []string `json:"notes,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Status reads the build modules' steps and the candidates against their verdicts.
|
||||||
func (r Reader) Status() (Status, error) {
|
func (r Reader) Status() (Status, error) {
|
||||||
st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}}
|
st := Status{RunningByNode: map[string]int{}}
|
||||||
if r.Capacity == nil {
|
|
||||||
st.CapacityNote = "the capacity setting cannot be read from here"
|
|
||||||
} else if setting, err := r.Capacity(); err != nil {
|
|
||||||
st.CapacityNote = "the capacity setting cannot be read: " + err.Error()
|
|
||||||
} else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil {
|
|
||||||
st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error()
|
|
||||||
}
|
|
||||||
if r.PlansState.Keys == nil {
|
|
||||||
st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker")
|
|
||||||
} else if keys, err := r.PlansState.Keys(); err != nil {
|
|
||||||
st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error())
|
|
||||||
} else {
|
|
||||||
sort.Strings(keys)
|
|
||||||
st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0
|
|
||||||
}
|
|
||||||
mods := make([]string, 0, len(r.StepStates))
|
mods := make([]string, 0, len(r.StepStates))
|
||||||
for m := range r.StepStates {
|
for m := range r.StepStates {
|
||||||
mods = append(mods, m)
|
mods = append(mods, m)
|
||||||
}
|
}
|
||||||
sort.Strings(mods)
|
sort.Strings(mods)
|
||||||
st.Running = []RunningStep{}
|
st.Running, st.LeftRunning = []RunningStep{}, []LeftStep{}
|
||||||
|
now, bound := time.Now(), r.StepBound
|
||||||
|
if r.Now != nil {
|
||||||
|
now = r.Now()
|
||||||
|
}
|
||||||
|
if bound <= 0 {
|
||||||
|
bound = DefaultStepBound
|
||||||
|
}
|
||||||
for _, m := range mods {
|
for _, m := range mods {
|
||||||
sr := r.StepStates[m]
|
sr := r.StepStates[m]
|
||||||
if sr.Keys == nil || sr.Get == nil {
|
if sr.Keys == nil || sr.Get == nil {
|
||||||
@@ -320,11 +314,30 @@ func (r Reader) Status() (Status, error) {
|
|||||||
if !step.Started.IsZero() {
|
if !step.Started.IsZero() {
|
||||||
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
rs.Since = step.Started.UTC().Format("2006-01-02T15:04:05Z")
|
||||||
}
|
}
|
||||||
|
if why := r.leftWhy(m, step, now, bound); why != "" {
|
||||||
|
st.LeftRunning = append(st.LeftRunning, LeftStep{RunningStep: rs, Why: why})
|
||||||
|
continue
|
||||||
|
}
|
||||||
st.Running = append(st.Running, rs)
|
st.Running = append(st.Running, rs)
|
||||||
st.RunningByNode[step.Node]++
|
st.RunningByNode[step.Node]++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if r.HaveState.Get == nil {
|
||||||
|
st.Notes = append(st.Notes, "the node-engine's have is not readable here, so a step is not checked against its build module being on its node")
|
||||||
|
}
|
||||||
|
if len(st.LeftRunning) > 0 {
|
||||||
|
st.Notes = append(st.Notes, "left_running are keys a build module left running (it stopped, restarted or was unassigned before it wrote the step's end): the checker's wait for each ended at the step bound, so none holds a place; the build module never ends them")
|
||||||
|
}
|
||||||
st.Unjudged = []string{}
|
st.Unjudged = []string{}
|
||||||
|
st.Waiting = map[string][]string{KindNew: {}, KindNodes: {}, KindTrunk: {}}
|
||||||
|
var heads map[string]string
|
||||||
|
if r.Trunks == nil {
|
||||||
|
st.Notes = append(st.Notes, "the trunk heads are not readable here, so no check is named as waiting for a trunk move")
|
||||||
|
} else if h, err := r.Trunks(); err != nil {
|
||||||
|
st.Notes = append(st.Notes, "the trunk heads cannot be read, so no check is named as waiting for a trunk move: "+err.Error())
|
||||||
|
} else {
|
||||||
|
heads = h
|
||||||
|
}
|
||||||
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
if r.CandidateState.Keys != nil && r.CandidateState.Get != nil && r.VerdictState.Get != nil {
|
||||||
keys, err := r.CandidateState.Keys()
|
keys, err := r.CandidateState.Keys()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -344,11 +357,59 @@ func (r Reader) Status() (Status, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if v, err := r.VerdictState.Get(vkey); err == nil && len(v) == 0 {
|
raw, err = r.VerdictState.Get(vkey)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(raw) == 0 {
|
||||||
st.Unjudged = append(st.Unjudged, c.Group)
|
st.Unjudged = append(st.Unjudged, c.Group)
|
||||||
|
st.Waiting[KindNew] = append(st.Waiting[KindNew], c.Group)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
v, err := delivery.DecodeVerdict(raw)
|
||||||
|
switch {
|
||||||
|
case err != nil || v.State == delivery.VerdictRunning:
|
||||||
|
case IsNodesFailure(v):
|
||||||
|
st.Waiting[KindNodes] = append(st.Waiting[KindNodes], c.Group)
|
||||||
|
case heads != nil:
|
||||||
|
if cs := CaseOf(c, heads); cs != nil && !sameTrunk(v.Trunk, cs.Trunk) {
|
||||||
|
st.Waiting[KindTrunk] = append(st.Waiting[KindTrunk], c.Group)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
st.Notes = append(st.Notes, "the steps the checker holds waiting for a place are in its process and in no bucket: a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held")
|
st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)")
|
||||||
return st, nil
|
return st, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch,
|
||||||
|
// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node.
|
||||||
|
const DefaultStepBound = 30 * time.Minute
|
||||||
|
|
||||||
|
// leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module
|
||||||
|
// is not on its node or is not running there (the node-engine's have), or it started longer ago than the step bound, when
|
||||||
|
// the checker stopped waiting for it.
|
||||||
|
func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound time.Duration) string {
|
||||||
|
if r.HaveState.Get != nil {
|
||||||
|
key, err := delivery.HaveKey(step.Node, module)
|
||||||
|
if err == nil {
|
||||||
|
raw, err := r.HaveState.Get(key)
|
||||||
|
switch {
|
||||||
|
case err != nil:
|
||||||
|
case len(raw) == 0:
|
||||||
|
return fmt.Sprintf("%s is not on %s", module, step.Node)
|
||||||
|
default:
|
||||||
|
if h, err := delivery.DecodeHave(raw); err == nil && h.Phase != delivery.PhaseRunning {
|
||||||
|
return fmt.Sprintf("%s on %s is %s", module, step.Node, h.Phase)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if step.Started.IsZero() {
|
||||||
|
return fmt.Sprintf("it says no start, so no wait of the checker's (bound %s) can be behind it", bound)
|
||||||
|
}
|
||||||
|
if end := step.Started.Add(bound); now.After(end) {
|
||||||
|
return fmt.Sprintf("started %s, past the step bound of %s: the checker stopped waiting for it at %s", step.Started.UTC().Format(time.RFC3339), bound, end.UTC().Format(time.RFC3339))
|
||||||
|
}
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package check
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
@@ -35,7 +34,7 @@ func (f fakeState) read() StateRead {
|
|||||||
func readerOf(t *testing.T) (Reader, string) {
|
func readerOf(t *testing.T) (Reader, string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
r := newRig(t, passJudge())
|
r := newRig(t, passJudge())
|
||||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(HQRepository, 3, head2))
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(hqRepo, 3, head2))
|
||||||
r.ch.Candidate(context.Background(), "feat%2Fx", cand)
|
r.ch.Candidate(context.Background(), "feat%2Fx", cand)
|
||||||
r.ch.Wait()
|
r.ch.Wait()
|
||||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||||
@@ -67,10 +66,12 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestAnHqPullRequestIsSaidToBeJudgedByMergeCheckUntilIssue1296(t *testing.T) { // ADR 0484 clause 8
|
// 1887/A10: an hq pull request is answered as any other, its report its verdict's (ADR 0527 §2), never a note that a script
|
||||||
|
// judges it.
|
||||||
|
func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) {
|
||||||
rd, _ := readerOf(t)
|
rd, _ := readerOf(t)
|
||||||
hq, err := rd.Pull(HQRepository, 3)
|
hq, err := rd.Pull(hqRepo, 3)
|
||||||
if err != nil || hq.Note != "hq: judged by merge-check.sh until issue 1296" || hq.Verdict != nil {
|
if err != nil || hq.Note != "" || hq.Verdict == nil {
|
||||||
t.Errorf("hq pull = %+v, %v", hq, err)
|
t.Errorf("hq pull = %+v, %v", hq, err)
|
||||||
}
|
}
|
||||||
p, err := rd.Pull(repoA, 7)
|
p, err := rd.Pull(repoA, 7)
|
||||||
@@ -129,39 +130,106 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1685 live acceptance: checker_status reads the capacity, whether a plan key holds the checker, the steps in flight by node and
|
// checker_status reads the steps in flight by node and the candidates without a verdict; since R4g nothing holds the checker
|
||||||
// the candidates without a verdict.
|
// and it has no capacity of its own (ADR 0525 §4, §7; 1685 closed by the hold's deletion).
|
||||||
func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) {
|
func TestTheStatusToolReadsRunningSteps(t *testing.T) {
|
||||||
running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)})
|
running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)})
|
||||||
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
|
passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed})
|
||||||
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
|
||||||
ckey, _ := delivery.CandidatesKey("feat/x")
|
ckey, _ := delivery.CandidatesKey("feat/x")
|
||||||
rd := Reader{
|
rd := Reader{
|
||||||
Capacity: func() (string, error) { return "shanks=2,ace=0", nil },
|
|
||||||
PlansState: fakeState{"p1": []byte("{}")}.read(),
|
|
||||||
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
|
StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()},
|
||||||
CandidateState: fakeState{ckey: cand}.read(),
|
CandidateState: fakeState{ckey: cand}.read(),
|
||||||
VerdictState: fakeState{}.read(),
|
VerdictState: fakeState{}.read(),
|
||||||
|
HaveState: fakeState{"shanks.mesh-build-go": haveOf(t, delivery.PhaseRunning)}.read(),
|
||||||
|
Now: func() time.Time { return time.Date(2026, 10, 22, 15, 10, 0, 0, time.UTC) },
|
||||||
}
|
}
|
||||||
st, err := rd.Status()
|
st, err := rd.Status()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if st.Capacity["shanks"] != 2 || st.Capacity["ace"] != 0 || st.DefaultPerNode != 1 {
|
|
||||||
t.Errorf("capacity = %v default %d", st.Capacity, st.DefaultPerNode)
|
|
||||||
}
|
|
||||||
if !st.PlansBuilding.Holds || len(st.PlansBuilding.Keys) != 1 {
|
|
||||||
t.Errorf("plans = %+v", st.PlansBuilding)
|
|
||||||
}
|
|
||||||
if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 {
|
if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 {
|
||||||
t.Errorf("running = %+v %v", st.Running, st.RunningByNode)
|
t.Errorf("running = %+v %v", st.Running, st.RunningByNode)
|
||||||
}
|
}
|
||||||
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" {
|
||||||
t.Errorf("unjudged = %v", st.Unjudged)
|
t.Errorf("unjudged = %v", st.Unjudged)
|
||||||
}
|
}
|
||||||
// An absent plan bucket is read as empty and said; an unreadable capacity is said, not guessed.
|
// What cannot be read is said, not guessed.
|
||||||
st, err = Reader{Capacity: func() (string, error) { return "", os.ErrNotExist }}.Status()
|
st, err = Reader{}.Status()
|
||||||
if err != nil || st.PlansBuilding.Holds || st.CapacityNote == "" || len(st.Notes) == 0 {
|
if err != nil || len(st.Notes) == 0 {
|
||||||
t.Errorf("empty status = %+v, %v", st, err)
|
t.Errorf("empty status = %+v, %v", st, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func haveOf(t *testing.T, phase delivery.Phase) []byte {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := delivery.EncodeHave(delivery.Have{Phase: phase, Health: delivery.HealthOK})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
func stepOf(t *testing.T, id, node string, started time.Time) []byte {
|
||||||
|
t.Helper()
|
||||||
|
raw, err := delivery.EncodeStep(delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: delivery.StepRunning, Started: started})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return raw
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1890: a key a build module left running (it stopped, restarted or was unassigned before it wrote the step's end) holds
|
||||||
|
// no place in the checker, whose wait for it ended at the step bound; the status says it is left running and why, and
|
||||||
|
// counts only the steps that can still run.
|
||||||
|
func TestTheStatusCountsOnlyStepsThatCanStillRunAndSaysWhyTheOthersAreLeft(t *testing.T) { // 1890/AC2, AC3
|
||||||
|
now := time.Date(2026, 10, 23, 7, 0, 0, 0, time.UTC)
|
||||||
|
rd := Reader{
|
||||||
|
StepStates: map[string]StateRead{
|
||||||
|
"mesh-build-go": fakeState{
|
||||||
|
"g14.fresh1": stepOf(t, "fresh1", "g14", now.Add(-5*time.Minute)),
|
||||||
|
"g14.old1": stepOf(t, "old1", "g14", now.Add(-33*time.Hour)),
|
||||||
|
"ace.gone1": stepOf(t, "gone1", "ace", now.Add(-2*time.Minute)),
|
||||||
|
"shanks.down1": stepOf(t, "down1", "shanks", now.Add(-time.Minute)),
|
||||||
|
}.read(),
|
||||||
|
},
|
||||||
|
HaveState: fakeState{
|
||||||
|
"g14.mesh-build-go": haveOf(t, delivery.PhaseRunning),
|
||||||
|
"shanks.mesh-build-go": haveOf(t, delivery.PhaseFailed),
|
||||||
|
}.read(),
|
||||||
|
StepBound: 30 * time.Minute,
|
||||||
|
Now: func() time.Time { return now },
|
||||||
|
}
|
||||||
|
st, err := rd.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(st.Running) != 1 || st.Running[0].ID != "fresh1" || len(st.RunningByNode) != 1 || st.RunningByNode["g14"] != 1 {
|
||||||
|
t.Fatalf("running = %+v by node %v", st.Running, st.RunningByNode)
|
||||||
|
}
|
||||||
|
why := map[string]string{}
|
||||||
|
for _, l := range st.LeftRunning {
|
||||||
|
why[l.ID] = l.Why
|
||||||
|
}
|
||||||
|
if len(why) != 3 {
|
||||||
|
t.Fatalf("left running = %+v", st.LeftRunning)
|
||||||
|
}
|
||||||
|
for id, want := range map[string]string{"old1": "past the step bound of 30m0s", "gone1": "mesh-build-go is not on ace", "down1": "mesh-build-go on shanks is failed"} {
|
||||||
|
if !strings.Contains(why[id], want) {
|
||||||
|
t.Errorf("why %s = %q, want it to say %q", id, why[id], want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 1887/A10: a group of hq's pull requests alone is judged by the checker since R4g, so without a verdict it is unjudged.
|
||||||
|
func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) {
|
||||||
|
cand := candidateOf(t, "pb-x", pullOf(hqRepo, 3, head2))
|
||||||
|
ckey, _ := delivery.CandidatesKey("pb-x")
|
||||||
|
st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{}.read()}.Status()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" {
|
||||||
|
t.Errorf("unjudged = %v, want pb-x", st.Unjudged)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -59,6 +59,8 @@ type Failure struct {
|
|||||||
Package string `json:"package"`
|
Package string `json:"package"`
|
||||||
Test string `json:"test,omitempty"`
|
Test string `json:"test,omitempty"`
|
||||||
Output string `json:"output,omitempty"`
|
Output string `json:"output,omitempty"`
|
||||||
|
// Cause is the first line of what the test reported as its error, when the build module told it apart (issue 1901).
|
||||||
|
Cause string `json:"cause,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// TestTime is one test and the seconds it took.
|
// TestTime is one test and the seconds it took.
|
||||||
@@ -71,22 +73,27 @@ type TestTime struct {
|
|||||||
// Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one
|
// Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one
|
||||||
// names the package directory as Module and counts none).
|
// names the package directory as Module and counts none).
|
||||||
type Tested struct {
|
type Tested struct {
|
||||||
Level string `json:"level"`
|
Level string `json:"level"`
|
||||||
Module string `json:"module,omitempty"`
|
Module string `json:"module,omitempty"`
|
||||||
Passed bool `json:"passed"`
|
Passed bool `json:"passed"`
|
||||||
Why string `json:"why,omitempty"`
|
Why string `json:"why,omitempty"`
|
||||||
NoTests bool `json:"no_tests,omitempty"`
|
NoTests bool `json:"no_tests,omitempty"`
|
||||||
Tests int `json:"tests"`
|
Tests int `json:"tests"`
|
||||||
Skipped int `json:"skipped"`
|
Skipped int `json:"skipped"`
|
||||||
Packages []GoPackage `json:"packages,omitempty"`
|
Packages []GoPackage `json:"packages,omitempty"`
|
||||||
Elapsed float64 `json:"elapsed"`
|
Elapsed float64 `json:"elapsed"`
|
||||||
Setup float64 `json:"setup,omitempty"`
|
Setup float64 `json:"setup,omitempty"`
|
||||||
Bound float64 `json:"bound"`
|
// Lint is the unit level's gofmt and go vet time, before its tests (mesh-build-go, ADR 0529 §1).
|
||||||
SetupBound float64 `json:"setup_bound,omitempty"`
|
Lint float64 `json:"lint,omitempty"`
|
||||||
Slowest []TestTime `json:"slowest,omitempty"`
|
Bound float64 `json:"bound"`
|
||||||
Failed []Failure `json:"failed,omitempty"`
|
SetupBound float64 `json:"setup_bound,omitempty"`
|
||||||
OverBound bool `json:"over_bound"`
|
Slowest []TestTime `json:"slowest,omitempty"`
|
||||||
OverWhich string `json:"over_which,omitempty"`
|
Failed []Failure `json:"failed,omitempty"`
|
||||||
|
OverBound bool `json:"over_bound"`
|
||||||
|
OverWhich string `json:"over_which,omitempty"`
|
||||||
|
// JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate <merge commit>" under runner
|
||||||
|
// workspace; absent from a build module that does not take the runner.
|
||||||
|
JudgedBy string `json:"judged_by,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
|
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
|
||||||
@@ -98,4 +105,6 @@ type Digested struct {
|
|||||||
Level string `json:"level"`
|
Level string `json:"level"`
|
||||||
Packages map[string]string `json:"packages"`
|
Packages map[string]string `json:"packages"`
|
||||||
WithoutTests []string `json:"without_tests,omitempty"`
|
WithoutTests []string `json:"without_tests,omitempty"`
|
||||||
|
// JudgedBy is the program that ran the digest, as Tested.JudgedBy says.
|
||||||
|
JudgedBy string `json:"judged_by,omitempty"`
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -108,15 +108,15 @@ type TryStore interface {
|
|||||||
Create(key string, value []byte) error
|
Create(key string, value []byte) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// NotChecked is the error of a try of a repository the checker judges no level of: hq, judged by merge-check.sh (ADR 0484
|
// NotChecked is the error of a try of a repository the checker judges no level of: one whose trunk the gate's source does
|
||||||
// clause 8), and a repository whose trunk the gate's source does not name.
|
// not name.
|
||||||
func NotChecked(repository string) error {
|
func NotChecked(repository string) error {
|
||||||
return fmt.Errorf("no level of %s is checked by the checker", repository)
|
return fmt.Errorf("no level of %s is checked by the checker", repository)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
|
// AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second
|
||||||
// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a
|
// ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a
|
||||||
// repository with none, and hq, are no repository the checker checks. A trunk asked by name is its trunk head now.
|
// repository with none is no repository the checker checks. A trunk asked by name is its trunk head now.
|
||||||
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) {
|
||||||
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
|
repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git"))
|
||||||
head := strings.ToLower(strings.TrimSpace(a.Head))
|
head := strings.ToLower(strings.TrimSpace(a.Head))
|
||||||
@@ -136,7 +136,7 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (
|
|||||||
trunkHead = h
|
trunkHead = h
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if repo == HQRepository || trunkHead == "" {
|
if trunkHead == "" {
|
||||||
return "", NotChecked(repo)
|
return "", NotChecked(repo)
|
||||||
}
|
}
|
||||||
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
|
t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()}
|
||||||
@@ -206,13 +206,11 @@ type TryOptions struct {
|
|||||||
Run string
|
Run string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check, and writes its
|
// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR
|
||||||
// end. It holds no timer: a try starts when its key is written, or when a delivery that held it ends (Reconsider).
|
// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written.
|
||||||
type Tryer struct {
|
type Tryer struct {
|
||||||
Tries KV
|
Tries KV
|
||||||
Judge Judge
|
Judge Judge
|
||||||
// Hold says whether a delivery builds, so no try starts (ADR 0484 clause 12); nil never holds.
|
|
||||||
Hold func() bool
|
|
||||||
Bound time.Duration
|
Bound time.Duration
|
||||||
Now func() time.Time
|
Now func() time.Time
|
||||||
Logf func(format string, args ...any)
|
Logf func(format string, args ...any)
|
||||||
@@ -250,7 +248,6 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
|||||||
if t.Over() {
|
if t.Over() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
held := tr.Hold != nil && tr.Hold()
|
|
||||||
tr.mu.Lock()
|
tr.mu.Lock()
|
||||||
if tr.inflight == nil {
|
if tr.inflight == nil {
|
||||||
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
|
tr.inflight, tr.seen = map[string]bool{}, map[string]bool{}
|
||||||
@@ -259,11 +256,6 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) {
|
|||||||
tr.mu.Unlock()
|
tr.mu.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
if held {
|
|
||||||
tr.mu.Unlock() // not seen yet: a try left running is still taken over when the hold ends
|
|
||||||
tr.logf("%s: not started, a delivery builds", key)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
tr.seen[key] = true
|
tr.seen[key] = true
|
||||||
tr.inflight[key] = true
|
tr.inflight[key] = true
|
||||||
tr.mu.Unlock()
|
tr.mu.Unlock()
|
||||||
@@ -336,7 +328,7 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
|||||||
t.Ended = tr.now().UTC()
|
t.Ended = tr.now().UTC()
|
||||||
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
t.Took = t.Ended.Sub(t.Started).Round(time.Second).String()
|
||||||
t.Levels, t.Machines = v.Levels, v.Machines
|
t.Levels, t.Machines = v.Levels, v.Machines
|
||||||
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest)
|
t.Said = Report(v, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged)
|
||||||
switch {
|
switch {
|
||||||
case !judged || IsNodesFailure(v):
|
case !judged || IsNodesFailure(v):
|
||||||
t.State = TryError
|
t.State = TryError
|
||||||
@@ -361,13 +353,14 @@ func (tr *Tryer) run(ctx context.Context, key string) {
|
|||||||
tr.logf("%s: %s", key, t.State)
|
tr.logf("%s: %s", key, t.State)
|
||||||
}
|
}
|
||||||
|
|
||||||
// judgeWithin runs a case within the bound (DefaultBound when 0). A judge that fails to judge, and a check that ran past its
|
// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A
|
||||||
// bound, are verdicts of failure naming the machine and the step, and judged says false for them.
|
// judge that fails to judge, and a check that ran past its bound, are verdicts of failure naming the machine and the step,
|
||||||
|
// and judged says false for them.
|
||||||
func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) {
|
func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) {
|
||||||
if bound <= 0 {
|
run, cancel := context.WithCancel(ctx)
|
||||||
bound = DefaultBound
|
if bound > 0 {
|
||||||
|
run, cancel = context.WithTimeoutCause(ctx, bound, errBound)
|
||||||
}
|
}
|
||||||
run, cancel := context.WithTimeoutCause(ctx, bound, errBound)
|
|
||||||
defer cancel()
|
defer cancel()
|
||||||
out, err := j.Judge(run, cs)
|
out, err := j.Judge(run, cs)
|
||||||
stage, node := cs.Progress.Now()
|
stage, node := cs.Progress.Now()
|
||||||
|
|||||||
@@ -23,7 +23,7 @@ func (m memTries) Create(key string, value []byte) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, HQRepository: trunkA}
|
var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA}
|
||||||
|
|
||||||
func readTry(t testing.TB, kv *memKV, id string) Try {
|
func readTry(t testing.TB, kv *memKV, id string) Try {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
@@ -67,7 +67,7 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR
|
|||||||
|
|
||||||
func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agreed error of the try
|
func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agreed error of the try
|
||||||
store := memTries{newMemKV()}
|
store := memTries{newMemKV()}
|
||||||
for _, repo := range []string{HQRepository, "novox/mesh-lab"} {
|
for _, repo := range []string{"novox/no-trunk", "novox/mesh-lab"} { // hq is checked since R4g (ADR 0527): see tryTrunks
|
||||||
_, err := AskTry(store, tryTrunks, TryAsk{Repository: repo, Head: head1, Trunk: "main"}, time.Now())
|
_, err := AskTry(store, tryTrunks, TryAsk{Repository: repo, Head: head1, Trunk: "main"}, time.Now())
|
||||||
if err == nil || err.Error() != "no level of "+repo+" is checked by the checker" {
|
if err == nil || err.Error() != "no level of "+repo+" is checked by the checker" {
|
||||||
t.Errorf("%s: %v", repo, err)
|
t.Errorf("%s: %v", repo, err)
|
||||||
@@ -153,21 +153,10 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T)
|
|||||||
if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], "TestThreeIsFour") {
|
if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], "TestThreeIsFour") {
|
||||||
t.Errorf("try = %+v", got)
|
t.Errorf("try = %+v", got)
|
||||||
}
|
}
|
||||||
// No build node: the try could not run, which is an error, not a failure of the change.
|
// No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4).
|
||||||
kv2 := newMemKV()
|
|
||||||
b2 := newBench(t)
|
|
||||||
b2.ex.Toolchains = entries{}
|
|
||||||
id2, _ := AskTry(memTries{kv2}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
|
||||||
tr2 := &Tryer{Tries: kv2, Judge: b2.ex}
|
|
||||||
raw, _, _, _ = kv2.Get(ctx, id2)
|
|
||||||
tr2.Put(ctx, id2, raw)
|
|
||||||
tr2.Wait()
|
|
||||||
if got := readTry(t, kv2, id2); got.State != TryError {
|
|
||||||
t.Errorf("try with no node = %+v, want error", got)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestATryWaitsWhileADeliveryBuildsAndATryLeftRunningIsTakenOver(t *testing.T) {
|
func TestATryLeftRunningIsTakenOver(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
kv := newMemKV()
|
kv := newMemKV()
|
||||||
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now())
|
||||||
@@ -180,14 +169,7 @@ func TestATryWaitsWhileADeliveryBuildsAndATryLeftRunningIsTakenOver(t *testing.T
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
b := newBench(t)
|
b := newBench(t)
|
||||||
held := true
|
tr := &Tryer{Tries: kv, Judge: b.ex}
|
||||||
tr := &Tryer{Tries: kv, Judge: b.ex, Hold: func() bool { return held }}
|
|
||||||
tr.Reconsider(ctx)
|
|
||||||
tr.Wait()
|
|
||||||
if n := len(b.steps.tool("fetch")); n != 0 {
|
|
||||||
t.Fatalf("a try ran while a delivery builds")
|
|
||||||
}
|
|
||||||
held = false
|
|
||||||
tr.Reconsider(ctx)
|
tr.Reconsider(ctx)
|
||||||
tr.Wait()
|
tr.Wait()
|
||||||
if got := readTry(t, kv, id); got.State != TryPass {
|
if got := readTry(t, kv, id); got.State != TryPass {
|
||||||
|
|||||||
@@ -46,6 +46,7 @@ type manifest struct {
|
|||||||
Consumes []string `json:"consumes"`
|
Consumes []string `json:"consumes"`
|
||||||
State []state `json:"state"`
|
State []state `json:"state"`
|
||||||
Reads []string `json:"reads"`
|
Reads []string `json:"reads"`
|
||||||
|
Uses []string `json:"uses"`
|
||||||
Invokes []string `json:"invokes"`
|
Invokes []string `json:"invokes"`
|
||||||
Config map[string]struct {
|
Config map[string]struct {
|
||||||
Type string `json:"type"`
|
Type string `json:"type"`
|
||||||
@@ -95,13 +96,18 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
|||||||
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "tries", History: 1}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
|
||||||
t.Errorf("state = %+v, want %+v", m.State, want)
|
t.Errorf("state = %+v, want %+v", m.State, want)
|
||||||
}
|
}
|
||||||
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
|
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", "mesh-controller.assignments",
|
||||||
"mesh-controller.plans-building", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
|
"mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
|
||||||
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
|
||||||
t.Errorf("reads = %v, want %v", m.Reads, want)
|
t.Errorf("reads = %v, want %v", m.Reads, want)
|
||||||
}
|
}
|
||||||
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
// 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), calls no
|
||||||
t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5) and the forge's two reads for a mesh-sdk pin (clause 9)", m.Invokes)
|
// build module's tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4).
|
||||||
|
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
|
||||||
|
t.Errorf("invokes = %v, want only the forge's reads for a mesh-sdk pin (ADR 0484 clause 9)", m.Invokes)
|
||||||
|
}
|
||||||
|
if want := []string{"node-build-go", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) {
|
||||||
|
t.Errorf("uses = %v, want the Go and Node build seats", m.Uses)
|
||||||
}
|
}
|
||||||
// The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
|
// The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
|
||||||
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
|
||||||
@@ -122,7 +128,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1
|
|||||||
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
|
||||||
t.Errorf("tools = %v, want %v", m.Tools, want)
|
t.Errorf("tools = %v, want %v", m.Tools, want)
|
||||||
}
|
}
|
||||||
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
|
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} {
|
||||||
if _, ok := top[k]; ok {
|
if _, ok := top[k]; ok {
|
||||||
t.Errorf("the manifest declares %q: the checker has none", k)
|
t.Errorf("the manifest declares %q: the checker has none", k)
|
||||||
}
|
}
|
||||||
@@ -148,35 +154,19 @@ func TestManifestProcess(t *testing.T) { // 1678/AC1
|
|||||||
t.Errorf("process = %v", p)
|
t.Errorf("process = %v", p)
|
||||||
}
|
}
|
||||||
env, _ := p["env"].(map[string]any)
|
env, _ := p["env"].(map[string]any)
|
||||||
// The capacity of each node is the module's setting `capacity` (ADR 0484 clause 12, as the builder's, issue 1661), laid into the
|
if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" {
|
||||||
// settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity.
|
t.Errorf("env = %v, want the broker file alone", p["env"])
|
||||||
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_CHECKER_SETTINGS"] != "${dir:state}/settings.json" {
|
|
||||||
t.Errorf("env = %v", p["env"])
|
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) {
|
|
||||||
t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"])
|
|
||||||
}
|
|
||||||
c, ok := m.Config["capacity"]
|
|
||||||
if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "" || c.Description == "" {
|
|
||||||
t.Errorf("config = %+v, want one string setting, capacity, empty (one step at a time on every node) to start", m.Config)
|
|
||||||
}
|
|
||||||
var settings map[string]any
|
|
||||||
for _, r := range m.Resources {
|
for _, r := range m.Resources {
|
||||||
if r["id"] == "settings" {
|
if r["id"] == "settings" {
|
||||||
settings = r
|
t.Errorf("a settings resource %v: the checker has no capacity of its own (ADR 0525 §4)", r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" {
|
|
||||||
t.Errorf("settings resource = %v", settings)
|
|
||||||
}
|
|
||||||
if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": ""`) {
|
|
||||||
t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err)
|
|
||||||
}
|
|
||||||
if _, bus := p["bus"]; bus {
|
if _, bus := p["bus"]; bus {
|
||||||
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
t.Errorf("the process declares a bus: it runs on its own broker credential")
|
||||||
}
|
}
|
||||||
if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[2].Name != "files" || m.Build.Artifacts[2].Kind != "archive" || m.Build.Artifacts[2].From != "files" {
|
if len(m.Build.Artifacts) != 2 {
|
||||||
t.Fatalf("artifacts = %+v, want the program, the tools and the files", m.Build.Artifacts)
|
t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts)
|
||||||
}
|
}
|
||||||
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Language != "go" || a.Binary != "mesh-delivery-checker-tools" ||
|
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Language != "go" || a.Binary != "mesh-delivery-checker-tools" ||
|
||||||
!reflect.DeepEqual(a.Loads, []string{"mesh-delivery-checker-tools"}) {
|
!reflect.DeepEqual(a.Loads, []string{"mesh-delivery-checker-tools"}) {
|
||||||
|
|||||||
@@ -21,7 +21,8 @@
|
|||||||
"mesh-build-go.toolchains",
|
"mesh-build-go.toolchains",
|
||||||
"mesh-build-node.steps",
|
"mesh-build-node.steps",
|
||||||
"mesh-build-node.toolchains",
|
"mesh-build-node.toolchains",
|
||||||
"mesh-controller.plans-building",
|
"mesh-controller.assignments",
|
||||||
|
"mesh-delivery-assigner.compositions",
|
||||||
"mesh-delivery-builder.builds",
|
"mesh-delivery-builder.builds",
|
||||||
"mesh-delivery-gate.source",
|
"mesh-delivery-gate.source",
|
||||||
"mesh-delivery-indexer.candidates",
|
"mesh-delivery-indexer.candidates",
|
||||||
@@ -30,6 +31,10 @@
|
|||||||
"mesh-delivery-watchman.presence",
|
"mesh-delivery-watchman.presence",
|
||||||
"node-engine.have"
|
"node-engine.have"
|
||||||
],
|
],
|
||||||
|
"uses": [
|
||||||
|
"node-build-go",
|
||||||
|
"node-build-node"
|
||||||
|
],
|
||||||
"tools": [
|
"tools": [
|
||||||
"checker_kept",
|
"checker_kept",
|
||||||
"checker_previews",
|
"checker_previews",
|
||||||
@@ -42,21 +47,9 @@
|
|||||||
"invokes": [
|
"invokes": [
|
||||||
"gitea.gitea_commits",
|
"gitea.gitea_commits",
|
||||||
"gitea.gitea_compare",
|
"gitea.gitea_compare",
|
||||||
"gitea.gitea_get_file",
|
"gitea.gitea_get_file"
|
||||||
"mesh-build-go.build_go_digest",
|
|
||||||
"mesh-build-go.build_go_fetch",
|
|
||||||
"mesh-build-go.build_go_test",
|
|
||||||
"mesh-build-node.build_node_fetch",
|
|
||||||
"mesh-build-node.build_node_test"
|
|
||||||
],
|
],
|
||||||
"config": {
|
|
||||||
"capacity": {
|
|
||||||
"type": "string",
|
|
||||||
"description": "How many build steps the checker runs at once on each node, as node=steps pairs separated by commas, such as shanks=2,ace=1 (ADR 0484 clause 12); a node set to 0 never checks, a node not named runs one at a time, and empty is one everywhere. The builder's capacity is the builder's own.",
|
|
||||||
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
|
|
||||||
"default": ""
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"own-secrets": {
|
"own-secrets": {
|
||||||
"broker": "${dir:state}/broker"
|
"broker": "${dir:state}/broker"
|
||||||
},
|
},
|
||||||
@@ -77,17 +70,6 @@
|
|||||||
"place": ".",
|
"place": ".",
|
||||||
"unknown": "program"
|
"unknown": "program"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"id": "settings",
|
|
||||||
"trusted": false,
|
|
||||||
"type": "file",
|
|
||||||
"path": "${dir:state}/settings.json",
|
|
||||||
"mode": "0600",
|
|
||||||
"owner": "mesh-delivery-checker",
|
|
||||||
"takes-config": ["capacity"],
|
|
||||||
"merge": "json",
|
|
||||||
"template": "files/settings.json"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"id": "checker",
|
"id": "checker",
|
||||||
"type": "process",
|
"type": "process",
|
||||||
@@ -97,12 +79,8 @@
|
|||||||
"./mesh-delivery-checker"
|
"./mesh-delivery-checker"
|
||||||
],
|
],
|
||||||
"env": {
|
"env": {
|
||||||
"MESH_BROKER_FILE": "${dir:state}/broker",
|
"MESH_BROKER_FILE": "${dir:state}/broker"
|
||||||
"MESH_CHECKER_SETTINGS": "${dir:state}/settings.json"
|
|
||||||
},
|
},
|
||||||
"restart-on": [
|
|
||||||
"settings"
|
|
||||||
],
|
|
||||||
"user": "mesh-delivery-checker",
|
"user": "mesh-delivery-checker",
|
||||||
"health": {
|
"health": {
|
||||||
"kind": "unit"
|
"kind": "unit"
|
||||||
@@ -129,11 +107,6 @@
|
|||||||
"loads": [
|
"loads": [
|
||||||
"mesh-delivery-checker-tools"
|
"mesh-delivery-checker-tools"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "files",
|
|
||||||
"kind": "archive",
|
|
||||||
"from": "files"
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b
|
|||||||
// TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged
|
// TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged
|
||||||
// message on FORGE makes modules.<m> within 5 s, the fingerprint of the merge's tree and the recorded inputs.
|
// message on FORGE makes modules.<m> within 5 s, the fingerprint of the merge's tree and the recorded inputs.
|
||||||
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851)
|
func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851)
|
||||||
// The bucket holds only build-agent's record, the one the indexer reads until R4g (the builder's own input).
|
// The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1).
|
||||||
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
|
||||||
rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`
|
rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}`
|
||||||
if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil {
|
if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil {
|
||||||
@@ -276,8 +276,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4
|
|||||||
// TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state
|
// TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state
|
||||||
// toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying
|
// toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying
|
||||||
// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build
|
// avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build
|
||||||
// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, none2 "builder's
|
// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, its fingerprint:
|
||||||
// fingerprint not yet known".
|
// no builder input is waited for (ADR 0530 §1; issue 1887 I1).
|
||||||
func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10
|
func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10
|
||||||
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -342,8 +342,8 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
|
|||||||
if _, err := js.Publish(ctx, "mesh.stream.mesh-build-go.event.toolchain.reported", []byte(`{"node":"n1","language":"go"}`)); err != nil {
|
if _, err := js.Publish(ctx, "mesh.stream.mesh-build-go.event.toolchain.reported", []byte(`{"node":"n1","language":"go"}`)); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got := none2("builder's fingerprint not yet known"); got != "builder's fingerprint not yet known" {
|
if got := none2(""); got != "" {
|
||||||
t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got)
|
t.Errorf("none2 = %q within 5 s of the toolchain key, want a fingerprint and no reason", got)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -478,3 +478,159 @@ func TestAMergeOfMeshSDKChangingOnlyGoLeavesItsNamedTreeEqual(t *testing.T) {
|
|||||||
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
|
t.Errorf("the new form moved: %q/%q, then %q/%q", first.FP2, first.None2, second.FP2, second.None2)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestIndexerJudgesAnUnreadableVerdictAgainAtStart: the modules bucket holds avahi as an older SDK wrote it, with the reason
|
||||||
|
// 'the manifest cannot be read', the inputs the merge carried (the tree, the lock hash by file name) and the digest of its
|
||||||
|
// manifest, which manifests.<digest> holds and the running SDK reads; the go toolchain is reported. Within 5 s of the start
|
||||||
|
// and with no repository merge the value carries a fingerprint and no reason (issue 1923).
|
||||||
|
func TestIndexerJudgesAnUnreadableVerdictAgainAtStart(t *testing.T) { // 1923/AC3
|
||||||
|
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||||
|
digest := delivery.ManifestDigest(manifest)
|
||||||
|
_, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||||
|
mkey, err := delivery.ManifestsKey(digest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mdata, err := delivery.EncodeManifest(delivery.Manifest{Module: "avahi", JSON: string(manifest)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
manifests, err := js.KeyValue(ctx, delivery.ManifestsBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := manifests.Put(ctx, mkey, mdata); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
stale, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "89abcdef0123456789abcdef0123456789abcdef", Path: "modules/avahi",
|
||||||
|
Manifest: digest, None2: "the manifest cannot be read: claims node-build-go, which is a seat in the mesh's own namespace that it does not define",
|
||||||
|
Inputs2: &delivery.KeptInputs2{Tree: "tree-avahi", Locks: map[string]string{"go.sum": goSum}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
held, err := js.KeyValue(ctx, delivery.ModulesBucket)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := held.Put(ctx, "avahi", stale); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
key, err := delivery.ToolchainKey("n1", "go")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "1", At: time.Now()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
var got delivery.Module
|
||||||
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err := modules.Get(ctx, "avahi"); err == nil {
|
||||||
|
if m, err := delivery.DecodeModule(e.Value()); err == nil {
|
||||||
|
if got = m; got.FP2 != "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if got.FP2 == "" || got.None2 != "" {
|
||||||
|
t.Errorf("avahi = fp2 %q none2 %q within 5 s of the start, want a fingerprint and no reason", got.FP2, got.None2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestIndexerCountsOnlyTheAssignedBuildNodesToolchains (issue 1940): the toolchains bucket holds the report of n2, where
|
||||||
|
// mesh-build-go was unassigned (its assignment ended) at another version than n1's, as ace's v26.12.0 stayed beside the
|
||||||
|
// v26.12.1 of the nodes mesh-build-node runs on. The indexer gives the merged module a fingerprint from n1's toolchain
|
||||||
|
// alone; when the controller assigns mesh-build-go to n2 again, its report counts and the builders disagree (the guard
|
||||||
|
// stays); when the assignment ends again, the fingerprint comes back. No restart, no timer: the assignments' watch.
|
||||||
|
func TestIndexerCountsOnlyTheAssignedBuildNodesToolchains(t *testing.T) {
|
||||||
|
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
|
||||||
|
var assignments jetstream.KeyValue
|
||||||
|
js, ctx, _, modules := startIndexer(t, func(context.Context, jetstream.KeyValue) {}, func(ctx context.Context, js jetstream.JetStream) {
|
||||||
|
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket("mesh-build-go"), History: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for node, version := range map[string]string{"n1": "go1.27.3", "n2": "go1.27.2"} {
|
||||||
|
key, _ := delivery.ToolchainKey(node, "go")
|
||||||
|
value, err := delivery.EncodeToolchain(delivery.Toolchain{Language: "go", Version: version, Recipe: "1", At: time.Now()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := kv.Put(ctx, key, value); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The controller's assignments mirror, in its layout (mesh-controller internal/chain Assignment).
|
||||||
|
if assignments, err = js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_assignments", History: 1}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for key, value := range map[string]string{
|
||||||
|
"n1.mesh-build-go": `{"node":"n1","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z"}`,
|
||||||
|
"n2.mesh-build-go": `{"node":"n2","module":"mesh-build-go","assigned":"2026-10-20T08:00:00Z","ended":"2026-10-23T08:00:00Z"}`,
|
||||||
|
} {
|
||||||
|
if _, err := assignments.Put(ctx, key, []byte(value)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
data, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgeMerged, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 302,
|
||||||
|
Head: "0123456789abcdef0123456789abcdef01234567", Merge: "89abcdef0123456789abcdef0123456789abcdef", Root: "r0",
|
||||||
|
Trees: map[string]string{"modules/avahi": "newtree"},
|
||||||
|
Manifests: map[string]string{"modules/avahi": string(manifest)},
|
||||||
|
Locks: map[string]map[string]string{"modules/avahi": {"go.sum": goSum}}})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.merged", data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
read := func(ok func(delivery.Module) bool) delivery.Module {
|
||||||
|
var got delivery.Module
|
||||||
|
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
|
||||||
|
if e, err := modules.Get(ctx, "avahi"); err == nil {
|
||||||
|
if m, err := delivery.DecodeModule(e.Value()); err == nil {
|
||||||
|
if got = m; ok(m) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
agreed := func(m delivery.Module) bool { return m.FP2 != "" && m.Toolchains["go"].Version == "go1.27.3" }
|
||||||
|
if m := read(agreed); !agreed(m) {
|
||||||
|
t.Fatalf("within 5 s avahi has fp2 %q, none2 %q, toolchains %v: want a fingerprint of n1's go1.27.3 alone", m.FP2, m.None2, m.Toolchains)
|
||||||
|
}
|
||||||
|
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","assigned":"2026-10-24T08:00:00Z"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
disagree := func(m delivery.Module) bool {
|
||||||
|
return m.FP2 == "" && strings.HasPrefix(m.None2, "builders disagree: go1.27.2 recipe 1 on n2")
|
||||||
|
}
|
||||||
|
if m := read(disagree); !disagree(m) {
|
||||||
|
t.Fatalf("within 5 s of n2's assignment avahi has fp2 %q, none2 %q: want the builders to disagree", m.FP2, m.None2)
|
||||||
|
}
|
||||||
|
if _, err := assignments.Put(ctx, "n2.mesh-build-go", []byte(`{"node":"n2","module":"mesh-build-go","ended":"2026-10-24T09:00:00Z"}`)); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if m := read(agreed); !agreed(m) {
|
||||||
|
t.Fatalf("within 5 s of n2's unassignment avahi has fp2 %q, none2 %q", m.FP2, m.None2)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,9 @@
|
|||||||
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
// bridge's messages on the stream FORGE and keeps the candidate groups (candidates.<group>): the pull requests that share
|
||||||
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
// a branch, their heads and the modules each touches. It reads the controller's builds to know which module a changed
|
||||||
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
|
// directory is. It serves no request subject and holds no timer: every reaction is a message or a watch (ADR 0437 §8).
|
||||||
|
//
|
||||||
|
// Its build of 2026-10-24 00:37 named a program whose push never reached the artifact store, and a retry sends that
|
||||||
|
// same build again; this paragraph was added only to give the indexer a new build (issue 1933).
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -18,7 +21,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"strings"
|
"strings"
|
||||||
"sync/atomic"
|
|
||||||
"syscall"
|
"syscall"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -262,20 +264,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
known := index.NewBuilds()
|
known := index.NewBuilds()
|
||||||
// A build of the module that builds the builder moves the builder's Builder input and so every fingerprint of the new
|
// The controller's build records name the directory of a module whose key names none (fillPaths); no build of any
|
||||||
// form, with no merge (issue 1727): the new-form fingerprints are computed again when one is heard, once the indexer exists.
|
// module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1).
|
||||||
var started atomic.Pointer[index.Indexer]
|
|
||||||
each := func(e jetstream.KeyValueEntry) {
|
each := func(e jetstream.KeyValueEntry) {
|
||||||
if opOf(e) {
|
if opOf(e) {
|
||||||
known.Put(e.Key(), e.Value())
|
known.Put(e.Key(), e.Value())
|
||||||
} else {
|
} else {
|
||||||
known.Delete(e.Key())
|
known.Delete(e.Key())
|
||||||
}
|
}
|
||||||
if ix := started.Load(); ix != nil && strings.HasPrefix(e.Key(), index.BuilderBuiltBy+".build.") {
|
|
||||||
if err := ix.RecomputeForBuild(ctx); err != nil {
|
|
||||||
logger.Printf("computing the new-form fingerprints again after %s: %v", e.Key(), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
w, err := settle(ctx, builds, each)
|
w, err := settle(ctx, builds, each)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -295,7 +291,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
defer calls.Close()
|
defer calls.Close()
|
||||||
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
|
ix := &index.Indexer{Store: kvStore{candidates}, Modules: kvStore{modules}, Manifests: kvStore{manifests}, Builds: known, Logf: logger.Printf,
|
||||||
Named: namedVia(calls.Call)}
|
Named: namedVia(calls.Call), Trunk: trunkVia(calls.Call), Contains: containsVia(calls.Call)}
|
||||||
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
// The gate's source bucket (mesh-delivery-gate_source) comes with the gate (release 3e, novox/hq ADR 0450 §1): its key
|
||||||
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
// main is the mesh's source revision, which stamps the modules (issue 1344). Where it does not exist yet the indexer does
|
||||||
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
// not wait for it (that needs a timer, ADR 0437 §8): it says so, and is started again with the gate.
|
||||||
@@ -317,7 +313,26 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
}
|
}
|
||||||
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
go func() { sourceErrs <- follow(ctx, sw, each) }()
|
||||||
}
|
}
|
||||||
tools := newToolchainWatch(ctx, js, toolchainSink{put: ix.PutToolchain, apply: ix.ApplyToolchain, recompute: ix.Recompute}, logger)
|
// Only the toolchain reports of nodes the build module is assigned to count (issue 1940): the controller's
|
||||||
|
// assignments are settled first, so the toolchains settle through the gate already knowing them.
|
||||||
|
gate := newToolchainGate(sortedModules(), ix.PutToolchain, logger.Printf)
|
||||||
|
assignErrs, err := watchAssignments(ctx, js, gate, func(ctx context.Context) {
|
||||||
|
if err := ix.Recompute(ctx); err != nil {
|
||||||
|
logger.Printf("writing the modules again after an assignment of a build module changed: %v", err)
|
||||||
|
}
|
||||||
|
}, logger)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
tools := newToolchainWatch(ctx, js, toolchainSink{
|
||||||
|
put: gate.Toolchain,
|
||||||
|
apply: func(ctx context.Context, module, key string, value []byte, deleted bool) error {
|
||||||
|
if !gate.Toolchain(module, key, value, deleted) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return ix.Recompute(ctx)
|
||||||
|
},
|
||||||
|
recompute: ix.Recompute}, logger)
|
||||||
if err := tools.start(); err != nil {
|
if err := tools.start(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@@ -327,12 +342,15 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
if err := previews.ensure(); err != nil {
|
if err := previews.ensure(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
// Only now, with the toolchains read, are the builds heard to compute from (a build before it would write every Go
|
// Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as
|
||||||
// module as having no toolchain reported, and back); the builds heard until now are computed once.
|
// having no toolchain reported, and back).
|
||||||
started.Store(ix)
|
if err := ix.RecomputeAtStart(ctx); err != nil {
|
||||||
if err := ix.RecomputeForBuild(ctx); err != nil {
|
|
||||||
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
logger.Printf("computing the new-form fingerprints at start: %v", err)
|
||||||
}
|
}
|
||||||
|
// The modules the trunk no longer holds go from the bucket (issue 1900); a merge does the same from its own message.
|
||||||
|
if err := ix.Reconcile(ctx); err != nil {
|
||||||
|
logger.Printf("dropping the modules the trunk does not hold at start: %v", err)
|
||||||
|
}
|
||||||
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
// A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are
|
||||||
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
// made by the controller as the forge's are. Where the stream is not there yet the indexer says so and is started again
|
||||||
// with the build module, as with the gate's source.
|
// with the build module, as with the gate's source.
|
||||||
@@ -378,6 +396,8 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
|
|||||||
return err
|
return err
|
||||||
case err := <-sourceErrs:
|
case err := <-sourceErrs:
|
||||||
return err
|
return err
|
||||||
|
case err := <-assignErrs:
|
||||||
|
return err
|
||||||
case err := <-tools.errs:
|
case err := <-tools.errs:
|
||||||
return err
|
return err
|
||||||
case err := <-previews.errs:
|
case err := <-previews.errs:
|
||||||
|
|||||||
@@ -46,3 +46,50 @@ func namedVia(call toolCaller) func(ctx context.Context, repository, sha string,
|
|||||||
return a.Paths, nil
|
return a.Paths, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// trunkVia asks the forge for the module directories of a commit (issue 1900): the keys of the `trees` the same tool
|
||||||
|
// answers without paths, one per directory that holds a module.json (the root as ""; the tool lists no other).
|
||||||
|
func trunkVia(call toolCaller) func(ctx context.Context, repository, sha string) ([]string, error) {
|
||||||
|
return func(_ context.Context, repository, sha string) ([]string, error) {
|
||||||
|
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||||
|
}
|
||||||
|
raw, err := call(commitTreesTool, map[string]any{"owner": owner, "repo": repo, "sha": sha})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Trees map[string]string `json:"trees"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil {
|
||||||
|
return nil, fmt.Errorf("the answer of %s is not {root, trees}: %w", commitTreesTool, err)
|
||||||
|
}
|
||||||
|
dirs := make([]string, 0, len(a.Trees))
|
||||||
|
for dir := range a.Trees {
|
||||||
|
dirs = append(dirs, dir)
|
||||||
|
}
|
||||||
|
return dirs, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// containsVia asks the forge whether a commit is in another (gitea.gitea_contains: contains is true for the same commit or an ancestor).
|
||||||
|
func containsVia(call toolCaller) func(ctx context.Context, repository, commit, in string) (bool, error) {
|
||||||
|
return func(_ context.Context, repository, commit, in string) (bool, error) {
|
||||||
|
owner, repo, ok := strings.Cut(delivery.RepositoryName(repository), "/")
|
||||||
|
if !ok {
|
||||||
|
return false, fmt.Errorf("%q is not <owner>/<name>", repository)
|
||||||
|
}
|
||||||
|
raw, err := call("gitea.gitea_contains", map[string]any{"owner": owner, "repo": repo, "commit": commit, "in": in})
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
var a struct {
|
||||||
|
Contains *bool `json:"contains"`
|
||||||
|
}
|
||||||
|
if err := json.Unmarshal(raw, &a); err != nil || a.Contains == nil {
|
||||||
|
return false, fmt.Errorf("the answer of gitea.gitea_contains names no contains: %v", err)
|
||||||
|
}
|
||||||
|
return *a.Contains, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user