novox/mesh-delivery
19
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

The checker runs beside today's check: the checker, the indexer's candidate fingerprints, the bridge's status #17

Merged
mesh-tools merged 19 commits from feat/the-checker into main 2026-10-21 21:46:33 +00:00
Conversation 1 Commits 19 Files Changed 103
+7696 -1115
103 changed files with 7696 additions and 1115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go
+71 -11
View File
@@ -56,6 +56,7 @@ type fakeBuildModules struct {
failBuild atomic.Bool
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
fetches []map[string]any
}
func (f *fakeBuildModules) count(tool string) int {
@@ -83,6 +84,11 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
if strings.HasSuffix(tool, "_fetch") {
f.mu.Lock()
f.fetches = append(f.fetches, args)
f.mu.Unlock()
}
id := fmt.Sprintf("s%d", f.n.Add(1))
key, _ := delivery.StepKey(node, id)
put := func(s delivery.Step) {
@@ -103,7 +109,11 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
step.Ended = time.Now().UTC()
switch {
case strings.HasSuffix(tool, "_fetch"):
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"}`)
merge := ""
if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult)
merge = `,"merge":"` + strings.Repeat("9", 40) + `"`
}
step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`)
case f.failBuild.Load() && node == "n1":
step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"])
default:
@@ -119,16 +129,17 @@ func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
}
type bed struct {
ctx context.Context
js jetstream.JetStream
modules jetstream.KeyValue
builds jetstream.KeyValue
tools map[string]jetstream.KeyValue
presence jetstream.KeyValue
have jetstream.KeyValue
fake *fakeBuildModules
srvURL string
dir string
ctx context.Context
js jetstream.JetStream
modules jetstream.KeyValue
candidates jetstream.KeyValue
builds jetstream.KeyValue
tools map[string]jetstream.KeyValue
presence jetstream.KeyValue
have jetstream.KeyValue
fake *fakeBuildModules
srvURL string
dir string
}
// startBed starts the bus, the buckets the controller makes from the builder's and the build modules' manifests, the
@@ -193,6 +204,7 @@ func startBed(t *testing.T) *bed {
return kv
}
b.modules = mk(delivery.ModulesBucket, delivery.ModulesHistory)
b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory)
mk(delivery.ManifestsBucket, delivery.ManifestsHistory)
b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory)
presence := mk(delivery.PresenceBucket, delivery.PresenceHistory)
@@ -522,3 +534,51 @@ func TestAKeyLeftBuildingForAModuleThatIsGoneIsClosedAtStart(t *testing.T) {
t.Errorf("build = %+v, want it closed naming who left it", v)
}
}
// TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk: the trunk's record of netcheck carries a fingerprint of its
// own; a candidate the indexer wrote gives netcheck the fingerprint fp at a pull request's head; the program writes
// builds.netcheck.<fp> done, its fetch given the head, the ref that carries it and the trunk commit (issue 1684).
func TestBuilderBuildsACandidatesFingerprintFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1
b := startBed(t)
other := "src2:" + strings.Repeat("e", 64)
digest := b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck")
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: commit, FP: "src1:" + strings.Repeat("f", 64), FP2: other,
Manifest: digest, Path: "modules/netcheck", Toolchains: map[string]delivery.ToolchainUsed{"go": {Version: "go1.27.1", Recipe: "1"}, "typescript": {Version: "tsc5", Recipe: "1"}}})
if err != nil {
t.Fatal(err)
}
if _, err := b.modules.Put(b.ctx, "netcheck", data); err != nil {
t.Fatal(err)
}
const head = "1111111111111111111111111111111111111111"
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: 7, Head: head}, Modules: []string{"netcheck"},
Fps: map[string]delivery.CandidateModule{"netcheck": {FP2: fp, Manifest: digest}}}}
hash, _ := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
cand, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: "feat/x", HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
ckey, err := delivery.CandidatesKey("feat/x")
if err != nil {
t.Fatal(err)
}
if _, err := b.candidates.Put(b.ctx, ckey, cand); err != nil {
t.Fatal(err)
}
b.start(t, "n1", nil)
v := b.await(t, "netcheck", isState(delivery.BuildDone))
if len(v.Artifacts) != 2 {
t.Fatalf("build = %+v", v)
}
b.fake.mu.Lock()
defer b.fake.mu.Unlock()
found := false
for _, f := range b.fake.fetches {
if f["commit"] == head && f["ref"] == "refs/pull/7/head" && f["base"] == commit {
found = true
}
}
if !found {
t.Errorf("fetches = %v, want one of the candidate's head with its ref and the trunk commit", b.fake.fetches)
}
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go
+25 -7
View File
@@ -13,6 +13,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"io"
"log"
"os"
@@ -140,6 +141,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return err
}
candidates, err := bucket(delivery.CandidatesBucket)
if err != nil {
return err
}
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
// buckets are read and written on the connection above.
account, err := process.ReadAccount(path)
@@ -162,16 +167,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err != nil {
return fmt.Errorf("the build capacity setting: %w", err)
}
tools := &build.ToolchainSet{}
presence := &build.PresenceSet{}
have := &build.HaveSet{}
tools := &buildrun.ToolchainSet{}
presence := &buildrun.PresenceSet{}
have := &buildrun.HaveSet{}
b := &build.Builder{
Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module},
Runner: build.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound},
Runner: buildrun.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound},
Toolchains: tools,
Presence: presence,
Health: have,
Places: &build.Places{Capacity: capacity},
Places: &buildrun.Places{Capacity: capacity},
Manifests: manifestReader{manifests},
Nodes: nodesPerFingerprint,
Logf: logger.Printf,
@@ -184,7 +189,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
// A build module says each time it wrote its toolchains key: its stream, and the builder's durable consumer on it, are
// made by the controller. Where the stream is not there yet the builder says so and is started again with the build
// module.
for _, module := range []string{build.GoBuildModule, build.NodeBuildModule} {
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
stream := toolchainStreams[module]
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
@@ -255,9 +260,22 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := b.Sweep(ctx); err != nil {
return err
}
// The fingerprints the indexer gives the candidates' modules are built too (issue 1684), once the trunk's modules are known.
eachCandidate := func(e jetstream.KeyValueEntry) {
if opOf(e) {
b.Candidate(ctx, e.Key(), e.Value())
} else {
b.ForgetCandidate(e.Key())
}
}
cw, err := settle(ctx, candidates, eachCandidate)
if err != nil {
return err
}
logger.Printf("started on %s", cred.Node)
errs := make(chan error, 1)
errs := make(chan error, 2)
go func() { errs <- follow(ctx, w, each) }()
go func() { errs <- follow(ctx, cw, eachCandidate) }()
select {
case <-ctx.Done():
b.Wait()
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go
+3 -4
View File
@@ -3,9 +3,8 @@ package main
import (
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"os"
"mesh-delivery-builder/internal/build"
)
// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `<node>=<steps at
@@ -13,7 +12,7 @@ import (
// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch.
func capacityFrom(path, fallback string) (map[string]int, error) {
if path == "" {
return build.ParseCapacity(fallback)
return buildrun.ParseCapacity(fallback)
}
raw, err := os.ReadFile(path)
if err != nil {
@@ -25,5 +24,5 @@ func capacityFrom(path, fallback string) (map[string]int, error) {
if err := json.Unmarshal(raw, &s); err != nil {
return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
}
return build.ParseCapacity(s.Capacity)
return buildrun.ParseCapacity(s.Capacity)
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings_test.go
+2 -3
View File
@@ -1,11 +1,10 @@
package main
import (
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"os"
"path/filepath"
"testing"
"mesh-delivery-builder/internal/build"
)
func settingsFile(t *testing.T, body string) string {
@@ -29,7 +28,7 @@ func TestCapacityIsReadFromTheSettingsFile(t *testing.T) {
t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want)
}
}
p := &build.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 {
t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace"))
}
modules/mesh-delivery-builder/cmd/mesh-delivery-builder/toolchains.go
+8 -9
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"log"
"strings"
"sync"
@@ -11,22 +12,20 @@ import (
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-builder/internal/build"
)
// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it
// (declared by the build modules, heard by this module through consumes).
var toolchainStreams = map[string]string{build.GoBuildModule: "TOOLCHAINS_GO", build.NodeBuildModule: "TOOLCHAINS_NODE"}
var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO", buildrun.NodeBuildModule: "TOOLCHAINS_NODE"}
// toolchainWatch keeps the build modules' toolchains in a build.ToolchainSet and tells the builder when one changed. A
// toolchainWatch keeps the build modules' toolchains in a buildrun.ToolchainSet and tells the builder when one changed. A
// build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
type toolchainWatch struct {
ctx context.Context
js bucketOpener
set *build.ToolchainSet
set *buildrun.ToolchainSet
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
logger *log.Logger
errs chan error
@@ -34,13 +33,13 @@ type toolchainWatch struct {
watched map[string]bool
}
func newToolchainWatch(ctx context.Context, js bucketOpener, set *build.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(build.LanguageOf)), watched: map[string]bool{}}
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
}
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
func (w *toolchainWatch) start() error {
for module := range build.LanguageOf {
for module := range buildrun.LanguageOf {
if err := w.ensure(module); err != nil {
return err
}
@@ -91,7 +90,7 @@ type delivered interface {
// terminated; a failure stays unacknowledged, and the server delivers it again.
func (w *toolchainWatch) Reported(m delivered) {
module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.")
if _, known := build.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
if _, known := buildrun.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
w.logger.Printf("%s: not a build module's event, dropped", m.Subject())
_ = m.Term()
return
modules/mesh-delivery-builder/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-builder
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-builder/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-builder/internal/build/candidates.go
+104
View File
@@ -0,0 +1,104 @@
package build
import (
"context"
"fmt"
"sort"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// How a candidate's fingerprint is built (novox/hq ADR 0484 clause 3, issue 1684): the indexer writes, for each module a
// pull request touches, the fingerprint of the new form it has at the head merged onto the trunk (candidates.<group>), and
// the builder builds each, once, as a key of its own (builds.<module>.<fingerprint>) that the checker's line reads. The
// build is the trunk module's: the same build modules, the same nodes and capacity; what differs is where the source comes
// from: the fetch step is given the pull request's head, the ref that carries it and the trunk commit to merge it onto, as
// the checker's fetch is, and builds the merge's module directory.
// via is how a build's fetch step reaches its source when it is a candidate's: the ref that carries the head and the trunk
// commit the head is merged onto.
type via struct{ Ref, Base string }
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and builds the fingerprints it
// carries. A value that is not a candidate is said and left.
func (b *Builder) Candidate(ctx context.Context, key string, value []byte) {
b.init()
c, err := delivery.DecodeCandidate(value)
if err != nil {
b.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return
}
b.mu.Lock()
b.cands[key] = c
b.mu.Unlock()
b.considerCandidate(ctx, c)
}
// ForgetCandidate takes a candidate out (its key was deleted).
func (b *Builder) ForgetCandidate(key string) {
b.init()
b.mu.Lock()
delete(b.cands, key)
b.mu.Unlock()
}
func (b *Builder) considerCandidate(ctx context.Context, c delivery.Candidate) {
for _, p := range c.Pulls {
names := make([]string, 0, len(p.Fps))
for name := range p.Fps {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
cm := p.Fps[name]
if cm.FP2 == "" || cm.Manifest == "" {
continue // no fingerprint of the new form for it (a reason in none2) or no manifest to build by
}
b.mu.Lock()
trunk, known := b.known[name]
b.mu.Unlock()
if !known || trunk.Path == "" {
b.logf("candidates.%s: the module %s has no record of the trunk yet, so its fingerprint %s is not built", c.Group, name, cm.FP2)
continue
}
key, err := delivery.BuilderBuildKey(name, cm.FP2)
if err != nil {
b.logf("candidates.%s: %v", c.Group, err)
continue
}
m := delivery.Module{Repository: p.Repository, Commit: p.Head.Head, FP2: cm.FP2, Manifest: cm.Manifest, Path: trunk.Path,
Toolchains: b.agreedFor(ctx, cm.Manifest)}
b.mu.Lock()
if _, onTrunk := b.trunkFP[key]; !onTrunk {
b.via[key] = via{Ref: fmt.Sprintf("refs/pull/%d/head", p.Number), Base: trunk.Commit}
}
b.mu.Unlock()
b.consider(ctx, name, m)
}
}
}
// agreedFor are the toolchains the build nodes agree on now, for the languages of the manifest's plan: the toolchains the
// candidate's fingerprint was computed with, as the indexer recomputes it whenever they change. A language with none agreed
// is left out, and the build then waits for it as the trunk's does.
func (b *Builder) agreedFor(ctx context.Context, manifest string) map[string]delivery.ToolchainUsed {
plan, err, _ := b.planOf(ctx, manifest)
if err != nil {
return nil
}
out := map[string]delivery.ToolchainUsed{}
for _, bm := range plan.Modules() {
if !plan.NeedsToolchain(bm) {
continue
}
language := buildrun.LanguageOf[bm]
if tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)); none == "" {
out[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe}
}
}
if len(out) == 0 {
return nil
}
return out
}
modules/mesh-delivery-builder/internal/build/candidates_test.go
+114
View File
@@ -0,0 +1,114 @@
package build
import (
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// trunkModule writes the trunk's record of the module m at a fingerprint of its own, so that the candidate's fingerprint (fp1)
// is one the trunk does not carry.
func trunkModule(t *testing.T, r *rig) {
t.Helper()
data, err := delivery.EncodeModule(delivery.Module{Repository: "novox/mesh-catalog", Commit: "7a74cee83f88e6585a5511c3cc53bdf0f5303d20", FP: "src1:" + hex64('f'),
FP2: "src2:" + hex64('e'), Manifest: mdigest('a'), Path: "modules/m", Toolchains: agreedToolchains})
if err != nil {
t.Fatal(err)
}
r.b.Module(context.Background(), "m", data)
r.b.Wait()
r.bm.mu.Lock()
r.bm.calls = nil
r.bm.mu.Unlock()
}
func candidateValue(t *testing.T, group string, number int, head string) []byte {
t.Helper()
pulls := []delivery.CandidatePull{{Head: delivery.Head{Repository: "novox/mesh-catalog", Number: number, Head: head}, Modules: []string{"m"},
Fps: map[string]delivery.CandidateModule{"m": {FP2: fp1, Manifest: mdigest('a')}}}}
hash, err := delivery.HeadsHash([]delivery.Head{pulls[0].Head})
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
return data
}
const (
headX = "1111111111111111111111111111111111111111"
headY = "2222222222222222222222222222222222222222"
)
func TestACandidatesFingerprintIsBuiltOnceAsAKeyOfItsOwnFromTheHeadMergedOntoTheTrunk(t *testing.T) { // N7/AC1, N7/AC2 (issue 1684)
r := newRig(t, goBundle)
trunkModule(t, r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != delivery.BuildDone || len(v.Artifacts) != 1 {
t.Fatalf("build = %+v, want the candidate's fingerprint built", v)
}
fetches := r.bm.callsOf("build_go_fetch")
if len(fetches) != 2 { // one on each of two nodes
t.Fatalf("%d fetches, want one per node", len(fetches))
}
f := fetches[0].body.(map[string]any)
if f["commit"] != headX || f["ref"] != "refs/pull/7/head" || f["base"] != "7a74cee83f88e6585a5511c3cc53bdf0f5303d20" || f["repository"] != "novox/mesh-catalog" {
t.Errorf("fetch args = %+v, want the head, its ref and the trunk commit to merge onto", f)
}
if b := r.bm.callsOf("build_go_build")[0].body.(map[string]any); b["module"] != "modules/m" {
t.Errorf("build args = %+v", b)
}
// Two heads carrying one fingerprint are one build.
r.b.Candidate(context.Background(), "feat%2Fy", candidateValue(t, "feat/y", 8, headY))
r.b.Wait()
if n := len(r.bm.callsOf("build_go_fetch")); n != 2 {
t.Errorf("%d fetches after a second head with the same fingerprint, want no more", n)
}
}
func TestACandidateWhoseModuleHasNoTrunkRecordIsNotBuilt(t *testing.T) {
r := newRig(t, goBundle)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
if _, ok := r.st.vals[r.key]; ok {
t.Error("a build key was written for a module the builder knows no directory of")
}
}
func TestAFingerprintTheTrunkCarriesIsBuiltFromTheTrunksCommit(t *testing.T) { // the trunk's source wins
r := newRig(t, goBundle)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) // the trunk now carries fp1
r.b.Wait()
for _, c := range r.bm.callsOf("build_go_fetch") {
if f := c.body.(map[string]any); f["ref"] != nil || f["base"] != nil {
t.Errorf("fetch args = %+v: a trunk fingerprint is built from the trunk's commit alone", f)
}
}
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone {
t.Errorf("build = %+v", v)
}
}
func TestACandidateWhoseHeadDoesNotMergeIsFailedAndNoTrunkBuildIsStoredUnderIt(t *testing.T) { // review D4
for name, set := range map[string]func(*rig){
"conflict": func(r *rig) { r.bm.conflicts = "go.mod" },
"no merge": func(r *rig) { r.bm.noMerge = true },
} {
r := newRig(t, goBundle)
trunkModule(t, r)
set(r)
r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX))
r.b.Wait()
v := valueOf(t, r.st, r.key)
if v.State != delivery.BuildFailed || (name == "conflict" && !strings.Contains(v.Error, "go.mod")) || len(r.bm.callsOf("build_go_build")) != 0 {
t.Errorf("%s: build = %+v, builds %d, want failed and nothing built", name, v, len(r.bm.callsOf("build_go_build")))
}
}
}
modules/mesh-delivery-builder/internal/build/choose.go
-88
View File
@@ -1,88 +0,0 @@
package build
import (
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ChooseNodes picks up to want build nodes for a language (ADR 0478 clause 3): the nodes whose toolchains key reports the
// version the nodes agree on, the ones with room under their build capacity first (places), then the fewest lanes the
// builder has running or chosen there, a tie broken by the node's name. A place is reserved on each node chosen, in the
// same step as the choice, so builds choosing at once do not choose the same full node while another has room; the
// caller takes the place (Places.Acquire) or gives it up (Places.Cancel). A node at its capacity is still chosen when
// the others are too (fewer than want have room): its lane waits for a place. A node qualifies only when qualifies is true
// of it (online and its build module healthy; nil means every node does). It answers the toolchain all the reporting nodes
// agree on, qualifying or not, as the indexer computes it. When nothing is reported, or the nodes report different
// versions (so the indexer agreed none), it fails naming the language and why; when no reporting node qualifies it fails
// naming the nodes. A node whose build module refuses on its toolchain refuses the step, and the build fails naming it.
func ChooseNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, delivery.Toolchain2, error) {
tc, none := delivery.AgreedToolchain(entries)
if none != "" {
return nil, delivery.Toolchain2{}, fmt.Errorf("no %s build node qualifies: %s", language, none)
}
nodes, err := choose(language, entries, places, qualifies, want)
return nodes, tc, err
}
// ChooseAnyNodes is ChooseNodes for artifacts that need no toolchain, such as an archive (ADR 0478 clause 1): the
// build module's nodes agree on nothing, so any node of the build module that qualifies is chosen, by capacity and load only.
func ChooseAnyNodes(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
return choose(language, entries, places, qualifies, want)
}
func choose(language string, entries []delivery.NodeToolchain, places *Places, qualifies func(node string) bool, want int) ([]string, error) {
if places == nil {
places = &Places{}
}
nodes := make([]string, 0, len(entries))
var reporting []string
seen := map[string]bool{}
for _, e := range entries {
if seen[e.Node] {
continue
}
seen[e.Node] = true
reporting = append(reporting, e.Node)
if places.CapacityOf(e.Node) > 0 && (qualifies == nil || qualifies(e.Node)) { // a capacity of 0 keeps the builder off a node
nodes = append(nodes, e.Node)
}
}
if len(nodes) == 0 {
sort.Strings(reporting)
return nil, fmt.Errorf("no %s build node qualifies: %s report the toolchain, and none is online with a healthy build module and a build capacity", language, strings.Join(reporting, ", "))
}
return places.choose(nodes, want), nil
}
// DigestsDifferPrefix opens the error of a build whose nodes built an artifact to different digests (ADR 0478 clause 4):
// such a key is not built again as a new revision, since the same source on the same toolchains gave two answers and
// building again settles nothing; it waits for another fingerprint.
const DigestsDifferPrefix = "digests differ:"
// Agree is the one digest every node built an artifact to (ADR 0478 clause 4), or an error naming each node and its
// digest when they differ. One node's digest agrees with itself.
func Agree(artifact string, byNode map[string]string) (string, error) {
nodes := make([]string, 0, len(byNode))
for n := range byNode {
nodes = append(nodes, n)
}
sort.Strings(nodes)
var first string
for i, n := range nodes {
if i == 0 {
first = byNode[n]
continue
}
if byNode[n] != first {
msg := fmt.Sprintf("%s the nodes built %s to different digests:", DigestsDifferPrefix, artifact)
for _, n := range nodes {
msg += fmt.Sprintf(" %s %s;", n, byNode[n])
}
return "", fmt.Errorf("%s", msg[:len(msg)-1])
}
}
return first, nil
}
modules/mesh-delivery-builder/internal/build/choose_test.go
-95
View File
@@ -1,95 +0,0 @@
package build
import (
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
var out []delivery.NodeToolchain
for _, n := range nodes {
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
}
return out
}
func TestChooseTakesTheAgreedNodesWithFewestRunningStepsTiesByName(t *testing.T) { // 1607/AC4, ADR 0478 clause 3
entries := tcs("go", "go1.27.1", "novox", "ace", "g14", "shanks")
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 3, "ace": 1, "g14": 1, "shanks": 0}), nil, 2)
if err != nil {
t.Fatal(err)
}
if len(nodes) != 2 || nodes[0] != "shanks" || nodes[1] != "ace" {
t.Errorf("nodes = %v, want [shanks ace]: fewest steps first, a tie by name", nodes)
}
if tc.Version != "go1.27.1" || tc.Recipe != "1" || tc.Language != "go" {
t.Errorf("toolchain = %+v", tc)
}
}
func TestChooseWithOneNodeAnswersOneNode(t *testing.T) { // 1607/AC4
nodes, _, err := ChooseNodes("go", tcs("go", "go1.27.1", "novox"), nil, nil, 2)
if err != nil || len(nodes) != 1 || nodes[0] != "novox" {
t.Errorf("nodes = %v, err %v, want [novox]", nodes, err)
}
}
func TestChooseFailsNamingTheLanguageAndTheVersionsWhenNoNodeQualifies(t *testing.T) { // 1607/AC4
_, _, err := ChooseNodes("typescript", nil, nil, nil, 2)
if err == nil || !strings.Contains(err.Error(), "typescript") || !strings.Contains(err.Error(), delivery.ToolchainNoReport) {
t.Errorf("no report: error = %v, want it to name typescript and %q", err, delivery.ToolchainNoReport)
}
entries := append(tcs("go", "go1.27.1", "novox"), tcs("go", "go1.26.0", "ace")...)
_, _, err = ChooseNodes("go", entries, nil, nil, 2)
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), "go1.26.0") || !strings.Contains(err.Error(), "go") {
t.Errorf("disagreeing nodes: error = %v, want both versions named", err)
}
}
func TestAgreeNeedsEqualDigestsAndNamesBothNodesWhenTheyDiffer(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
d, err := Agree("tool", map[string]string{"ace": digest, "novox": digest})
if err != nil || d != digest {
t.Errorf("Agree = %q, %v, want %q", d, err, digest)
}
_, err = Agree("tool", map[string]string{"ace": digest, "novox": "sha256:" + hex64('9')})
if err == nil || !strings.Contains(err.Error(), "ace") || !strings.Contains(err.Error(), "novox") || !strings.Contains(err.Error(), digest) || !strings.Contains(err.Error(), hex64('9')) || !strings.Contains(err.Error(), "tool") {
t.Errorf("differing digests: error = %v, want the artifact, both nodes and both digests", err)
}
if d, err := Agree("tool", map[string]string{"ace": digest}); err != nil || d != digest {
t.Errorf("one node: %q, %v", d, err)
}
}
func TestChooseSkipsNodesThatAreNotOnlineAndFailsNamingThemWhenNoneIs(t *testing.T) { // 1607 review 3, ADR 0478 clause 3
entries := tcs("go", "go1.27.1", "ace", "g14", "novox")
online := func(n string) bool { return n != "g14" } // a sleeping laptop
nodes, tc, err := ChooseNodes("go", entries, placesRunning(map[string]int{"novox": 5}), online, 2)
if err != nil || len(nodes) != 2 || nodes[0] != "ace" || nodes[1] != "novox" || tc.Version != "go1.27.1" {
t.Errorf("nodes = %v, toolchain %+v, err %v, want [ace novox] with g14 left out although it runs nothing", nodes, tc, err)
}
_, tc, err = ChooseNodes("go", entries, nil, func(string) bool { return false }, 2)
if err == nil || !strings.Contains(err.Error(), "online") || !strings.Contains(err.Error(), "g14") || !strings.Contains(err.Error(), "go") {
t.Errorf("none online: error = %v, want it to name the nodes that report and that none is online", err)
}
if tc.Version != "go1.27.1" {
t.Errorf("none online: the agreed toolchain %+v is still answered", tc)
}
}
// placesRunning is Places with these steps counted as running, for a test of the choice.
func placesRunning(running map[string]int) *Places {
return &Places{running: running, changed: make(chan struct{})}
}
func TestChoosePrefersNodesBelowTheirCapacity(t *testing.T) { // round 3
entries := tcs("go", "go1.27.1", "ace", "novox", "shanks")
// shanks has capacity 3 with 2 running: room; ace has the default 1 with 1 running: none; novox has none running.
p := placesRunning(map[string]int{"shanks": 2, "ace": 1})
p.Capacity = map[string]int{"shanks": 3}
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
if err != nil || len(nodes) != 2 || nodes[0] != "novox" || nodes[1] != "shanks" {
t.Errorf("nodes = %v, err %v, want [novox shanks]: the nodes with room, ace at capacity left out", nodes, err)
}
}
modules/mesh-delivery-builder/internal/build/claim.go
+3 -2
View File
@@ -4,6 +4,7 @@ import (
"context"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -70,7 +71,7 @@ type ClaimOptions struct {
// when a second node qualifies now (ClaimOptions.MoreNodes): that is taken as it stands, with the done value, and its
// builder writes the new revision when the second build ends (ADR 0478 clause 4);
// - a failed key is taken again as a new revision of the same key (ADR 0429 clause 18), except one failed for a kind no
// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (DigestsDifferPrefix, clause 4),
// builder builds yet (RefusedPrefix, ADR 0478 clause 2) or for digests that differ (buildrun.DigestsDifferPrefix, clause 4),
// which wait for another fingerprint;
// - a key left queued or building by this builder is taken as it stands (ADR 0478 clause 5: a stopped builder takes its
// own again when it starts), and so is one left by another builder when ClaimOptions.TakeOver (clause 6).
@@ -97,7 +98,7 @@ func (c Claimer) Claim(ctx context.Context, module, fingerprint, manifest string
return &Claim{Key: key, Rev: rev, Build: cur}, nil
}
return nil, nil
case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, DigestsDifferPrefix)):
case cur.State == delivery.BuildFailed && (strings.HasPrefix(cur.Error, RefusedPrefix) || strings.HasPrefix(cur.Error, buildrun.DigestsDifferPrefix)):
return nil, nil
case cur.State == delivery.BuildFailed:
return c.write(ctx, key, building, func(data []byte) (uint64, error) { return c.Store.Update(ctx, key, data, rev) })
modules/mesh-delivery-builder/internal/build/fakes_test.go
+94
View File
@@ -0,0 +1,94 @@
package build
// The fakes of the build modules the builder tests run against: a caller answering a long tool at once with the step and its key,
// and the steps watch. The caller, the steps watch and the runner themselves are the SDK's (mesh-sdk go/delivery/buildrun, issue 1671).
import (
"context"
"encoding/json"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
type call struct {
node, key string
body any
}
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
type fakeCaller struct {
calls []call
answer string
err error
}
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
f.calls = append(f.calls, call{node, key, body})
if f.err != nil {
return nil, f.err
}
return json.RawMessage(f.answer), nil
}
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
type fakeSteps struct {
mu sync.Mutex
steps map[string]delivery.Step
// during, when set, is called as a wait starts and the function it answers as the wait ends.
during func(key string) func()
}
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
f.mu.Lock()
s, ok := f.steps[module+"/"+key]
during := f.during
f.mu.Unlock()
if ok {
if during != nil {
defer during(key)()
}
return s, nil
}
<-ctx.Done()
return delivery.Step{}, context.Cause(ctx)
}
func stepAt(node, id, state, reason string, result string) delivery.Step {
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
if result != "" {
s.Result = json.RawMessage(result)
}
return s
}
// presenceValue is a watchman presence value, as its writer encodes it (mesh-sdk go/delivery EncodePresence).
func presenceValue(t *testing.T, online bool) []byte {
t.Helper()
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
if err != nil {
t.Fatal(err)
}
return b
}
// tcValue is a toolchains value, as the build module's writer encodes it (mesh-sdk go/delivery EncodeToolchain).
func tcValue(t *testing.T, language, version string) []byte {
t.Helper()
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
return b
}
// tcs are the toolchain entries of a language on nodes, as the toolchains set gives them.
func tcs(language, version string, nodes ...string) []delivery.NodeToolchain {
var out []delivery.NodeToolchain
for _, n := range nodes {
out = append(out, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: language, Version: version, Recipe: "1"}})
}
return out
}
modules/mesh-delivery-builder/internal/build/health.go
-56
View File
@@ -1,56 +0,0 @@
package build
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HaveSet holds what each node's node-engine says of each module on it (node-engine_have, key `<node>.<module>`): the
// builder reads it to know whether a node's build module is running and healthy, since a build module whose installed
// toolchain is not the one its recipe builds with is unhealthy (mesh-build-go build_go_health) and its steps would
// refuse (ADR 0478 clause 3). A module the node-engine has written no key of is not healthy. Safe for use from the watch
// and from a build at once.
type HaveSet struct {
mu sync.Mutex
healthy map[string]bool // by `<node>.<module>`
}
// Put takes a change of the have bucket in and answers whether a module's healthy state changed. A node's own key
// (`<node>.node`) and a value that is not a have are left out.
func (s *HaveSet) Put(key string, value []byte, deleted bool) bool {
node, module, isNode, err := delivery.ParseHaveKey(key)
if err != nil || isNode {
return false
}
ok := false
if !deleted {
h, err := delivery.DecodeHave(value)
if err != nil {
return false
}
ok = h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK
}
k := node + "." + module
s.mu.Lock()
defer s.mu.Unlock()
if s.healthy[k] == ok {
return false
}
if s.healthy == nil {
s.healthy = map[string]bool{}
}
if ok {
s.healthy[k] = true
} else {
delete(s.healthy, k)
}
return true
}
// Healthy is whether the module on the node was last seen running and healthy.
func (s *HaveSet) Healthy(node, module string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.healthy[node+"."+module]
}
modules/mesh-delivery-builder/internal/build/places.go
-171
View File
@@ -1,171 +0,0 @@
package build
import (
"context"
"fmt"
"sort"
"strconv"
"strings"
"sync"
)
// DefaultCapacity is the build capacity of a node that has no setting: one build step at a time.
const DefaultCapacity = 1
// ParseCapacity reads the builder's capacity setting: `node=steps` pairs separated by commas, such as
// `shanks=3,ace=2`. A node without a pair has DefaultCapacity. A pair with no node, a count that is not a whole number of
// at least 0, and a node named twice are refused. A count of 0 keeps the builder off the node: it is never chosen.
func ParseCapacity(s string) (map[string]int, error) {
out := map[string]int{}
for _, pair := range strings.Split(s, ",") {
pair = strings.TrimSpace(pair)
if pair == "" {
continue
}
node, count, ok := strings.Cut(pair, "=")
node, count = strings.TrimSpace(node), strings.TrimSpace(count)
n, err := strconv.Atoi(count)
if !ok || node == "" || err != nil || n < 0 {
return nil, fmt.Errorf("the capacity %q is not <node>=<steps at once, 0 or more>", pair)
}
if _, dup := out[node]; dup {
return nil, fmt.Errorf("the capacity names %s twice", node)
}
out[node] = n
}
return out, nil
}
// Places counts the build steps the builder runs on each node and holds a lane back while its node is at its
// capacity (the operator's decision of 2026-10-21; ADR 0478 clause 3). A lane is the fetch and the builds of one build
// module on one node, run one step at a time, so a lane holds one place while it runs. A node is chosen for a lane by
// reserving a place on it at the moment of the choice (choose), so that builds choosing at once see each other's
// choices and none waits on a full node while another has room. A lane waits for its place by watching the ends of the
// other lanes, not by a timer: ending a lane wakes the waiting ones.
type Places struct {
// Capacity is the number of steps at once by node; a node not in it has DefaultCapacity.
Capacity map[string]int
mu sync.Mutex
running map[string]int // lanes holding a place
reserved map[string]int // lanes chosen for the node that do not hold a place yet
changed chan struct{} // closed, and replaced, each time a lane ends
}
// CapacityOf is how many steps the builder runs on the node at once: DefaultCapacity for a node the setting does not
// name, and 0 for one it names so, where the builder never builds.
func (p *Places) CapacityOf(node string) int {
if n, ok := p.Capacity[node]; ok && n >= 0 {
return n
}
return DefaultCapacity
}
// Running is the steps in flight now, by node.
func (p *Places) Running() map[string]int {
p.mu.Lock()
defer p.mu.Unlock()
out := make(map[string]int, len(p.running))
for n, c := range p.running {
if c > 0 {
out[n] = c
}
}
return out
}
// Reserved is the lanes chosen for a node that hold no place yet, by node.
func (p *Places) Reserved() map[string]int {
p.mu.Lock()
defer p.mu.Unlock()
out := make(map[string]int, len(p.reserved))
for n, c := range p.reserved {
if c > 0 {
out[n] = c
}
}
return out
}
func (p *Places) init() {
if p.running == nil {
p.running = map[string]int{}
}
if p.reserved == nil {
p.reserved = map[string]int{}
}
if p.changed == nil {
p.changed = make(chan struct{})
}
}
// Acquire takes a place on the node, waiting while the node is at its capacity, and answers the function that gives it
// back. A place reserved for the node by a choice is the one taken. It returns an error when ctx ends first, and then
// gives up the reservation too.
func (p *Places) Acquire(ctx context.Context, node string) (release func(), err error) {
for {
p.mu.Lock()
p.init()
if p.running[node] < p.CapacityOf(node) {
p.running[node]++
if p.reserved[node] > 0 {
p.reserved[node]--
}
p.mu.Unlock()
var once sync.Once
return func() {
once.Do(func() {
p.mu.Lock()
p.running[node]--
close(p.changed)
p.changed = make(chan struct{})
p.mu.Unlock()
})
}, nil
}
wait := p.changed
p.mu.Unlock()
select {
case <-wait:
case <-ctx.Done():
p.Cancel(node)
return nil, fmt.Errorf("waiting for a place on %s: %w", node, ctx.Err())
}
}
}
// Cancel gives up a place reserved for the node that will not be taken.
func (p *Places) Cancel(node string) {
p.mu.Lock()
defer p.mu.Unlock()
p.init()
if p.reserved[node] > 0 {
p.reserved[node]--
}
}
// choose orders the nodes (those with room first, counting the lanes running and the lanes reserved, then the fewest of
// them, then by name), keeps the first want and reserves a place on each, all at once.
func (p *Places) choose(nodes []string, want int) []string {
p.mu.Lock()
defer p.mu.Unlock()
p.init()
load := func(n string) int { return p.running[n] + p.reserved[n] }
sort.SliceStable(nodes, func(i, j int) bool {
ri, rj := load(nodes[i]) < p.CapacityOf(nodes[i]), load(nodes[j]) < p.CapacityOf(nodes[j])
if ri != rj {
return ri
}
if load(nodes[i]) != load(nodes[j]) {
return load(nodes[i]) < load(nodes[j])
}
return nodes[i] < nodes[j]
})
if len(nodes) > want {
nodes = nodes[:want]
}
for _, n := range nodes {
p.reserved[n]++
}
return nodes
}
modules/mesh-delivery-builder/internal/build/places_test.go
-186
View File
@@ -1,186 +0,0 @@
package build
import (
"context"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
func TestParseCapacityReadsNodeEqualsCountPairs(t *testing.T) {
c, err := ParseCapacity(" shanks=3, ace=2 ")
if err != nil || c["shanks"] != 3 || c["ace"] != 2 || len(c) != 2 {
t.Errorf("capacity = %v, %v", c, err)
}
if c, err := ParseCapacity(""); err != nil || len(c) != 0 {
t.Errorf("empty: %v, %v", c, err)
}
for _, bad := range []string{"shanks", "shanks=x", "=2", "a=1,a=2", "shanks=-1"} {
if _, err := ParseCapacity(bad); err == nil {
t.Errorf("%q accepted", bad)
}
}
}
func acquireAsync(p *Places, node string) chan func() {
got := make(chan func(), 1)
go func() {
release, err := p.Acquire(context.Background(), node)
if err != nil {
close(got)
return
}
got <- release
}()
return got
}
func TestThreeStepsOnANodeOfCapacityTwoRunTwoAtOnceAndTheThirdStartsWhenOneEnds(t *testing.T) { // round 3
p := &Places{Capacity: map[string]int{"ace": 2}}
a, b := acquireAsync(p, "ace"), acquireAsync(p, "ace")
ra, rb := <-a, <-b
third := acquireAsync(p, "ace")
select {
case <-third:
t.Fatal("a third step started on a node of capacity 2 with two running")
case <-time.After(50 * time.Millisecond):
}
if p.Running()["ace"] != 2 {
t.Errorf("running = %v, want 2 on ace", p.Running())
}
ra()
select {
case r := <-third:
r()
case <-time.After(2 * time.Second):
t.Fatal("the third step did not start when one ended")
}
rb()
}
func TestANodeWithNoSettingRunsOneStepAtATime(t *testing.T) { // round 3
p := &Places{Capacity: map[string]int{"shanks": 3}}
first := <-acquireAsync(p, "g14")
second := acquireAsync(p, "g14")
select {
case <-second:
t.Fatal("two steps on a node without a setting")
case <-time.After(50 * time.Millisecond):
}
first()
r := <-second
r()
if p.CapacityOf("g14") != 1 || p.CapacityOf("shanks") != 3 {
t.Errorf("capacities = %d, %d", p.CapacityOf("g14"), p.CapacityOf("shanks"))
}
}
func TestAWaitingStepStopsWhenItsContextEnds(t *testing.T) { // round 3
p := &Places{}
r := <-acquireAsync(p, "ace")
defer r()
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { _, err := p.Acquire(ctx, "ace"); done <- err }()
cancel()
select {
case err := <-done:
if err == nil {
t.Error("Acquire returned no error for a cancelled wait")
}
case <-time.After(2 * time.Second):
t.Fatal("a cancelled wait did not return")
}
}
func TestStepsOfSeveralBuildsNeverExceedANodesCapacity(t *testing.T) { // round 3, through the builder
r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"})
r.b.Places = &Places{Capacity: map[string]int{"ace": 2}} // novox has none: one at a time
var now, most atomic.Int32
var mu sync.Mutex
perNode := map[string]int{}
r.bm.steps.during = func(key string) func() {
node, _, _ := strings.Cut(key, ".")
mu.Lock()
perNode[node]++
if int32(perNode[node]) > most.Load() && node == "ace" {
most.Store(int32(perNode[node]))
}
if node == "novox" && perNode[node] > 1 {
t.Errorf("%d steps at once on novox, which has capacity 1", perNode[node])
}
mu.Unlock()
now.Add(1)
time.Sleep(15 * time.Millisecond)
return func() { mu.Lock(); perNode[node]--; mu.Unlock(); now.Add(-1) }
}
for _, m := range []string{"m1", "m2", "m3", "m4"} {
r.b.Module(context.Background(), m, moduleValue(t, "modules/"+m))
}
r.b.Wait()
if most.Load() > 2 {
t.Errorf("%d steps at once on ace, which has capacity 2", most.Load())
}
if most.Load() < 2 {
t.Errorf("at most %d steps at once on ace: the capacity was not used", most.Load())
}
}
func TestTwoNodeAgreementStillTakesTwoDifferentNodesWhenCapacitiesAreSet(t *testing.T) { // round 3, ADR 0478 clause 4
r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"})
r.b.Places = &Places{Capacity: map[string]int{"ace": 2, "novox": 3}}
v := r.consider(t, "modules/m")
if v.State != "done" || !strings.Contains(v.Step, "ace and novox") {
t.Errorf("build = %+v, want done on ace and novox", v)
}
}
// The operator's decision of 2026-10-21 13:40: the builder builds on shanks and ace only; a capacity of 0 keeps it off a node.
func TestACapacityOfZeroKeepsTheBuilderOffTheNodeAndANodeNotNamedKeepsOne(t *testing.T) { // issue 1661
c, err := ParseCapacity("shanks=3,ace=2,g14=0,novox=0")
if err != nil || c["g14"] != 0 || c["novox"] != 0 || c["shanks"] != 3 || len(c) != 4 {
t.Fatalf("capacity = %v, %v", c, err)
}
p := &Places{Capacity: c}
if p.CapacityOf("g14") != 0 || p.CapacityOf("novox") != 0 || p.CapacityOf("ace") != 2 || p.CapacityOf("other") != DefaultCapacity {
t.Errorf("capacities = %d %d %d %d, want 0 0 2 and the default for a node not named", p.CapacityOf("g14"), p.CapacityOf("novox"), p.CapacityOf("ace"), p.CapacityOf("other"))
}
if _, err := ParseCapacity("shanks=-1"); err == nil {
t.Error("a negative capacity accepted")
}
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks", "other")
for i := 0; i < 6; i++ {
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
if err != nil {
t.Fatal(err)
}
for _, n := range nodes {
if n == "g14" || n == "novox" {
t.Fatalf("choice %d: %v includes a node of capacity 0", i, nodes)
}
}
}
// Only nodes of capacity 0 report: nothing qualifies, and the build says why.
_, _, err = ChooseNodes("go", tcs("go", "go1.27.1", "g14", "novox"), p, nil, 2)
if err == nil || !strings.Contains(err.Error(), "capacity") {
t.Errorf("only capacity-0 nodes: error = %v, want it to name the capacity", err)
}
}
func TestABuildNeverRunsOnANodeOfCapacityZero(t *testing.T) { // issue 1661
r := newRig(t, catalogue.Artifact{Name: "tool", Kind: "bundle", Language: "go"}) // ace and novox report
r.b.Places = &Places{Capacity: map[string]int{"novox": 0}}
v := r.consider(t, "modules/m")
if v.State != "done" || !strings.Contains(v.Step, "on one node, ace") {
t.Fatalf("build = %+v, want done on ace alone", v)
}
for _, c := range r.bm.calls {
if c.node == "novox" {
t.Errorf("a step was called on novox, whose capacity is 0: %+v", c)
}
}
}
modules/mesh-delivery-builder/internal/build/plan.go
+4 -9
View File
@@ -5,17 +5,12 @@ package build
import (
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"sort"
"git.novox.be/novox/mesh-sdk/go/catalogue"
)
// The build modules a task is given to, and the language whose toolchain each reports (the indexer's ToolchainModules).
const (
GoBuildModule = "mesh-build-go"
NodeBuildModule = "mesh-build-node"
)
// RefusedPrefix opens the error of a build key that is failed for a kind no builder builds yet (ADR 0478 clause 2): such
// a key is not built again as a new revision.
const RefusedPrefix = "refused kind:"
@@ -79,11 +74,11 @@ func PlanOf(m catalogue.Manifest) (Plan, error) {
for _, a := range artifacts {
switch {
case a.Kind == catalogue.ArtifactArchive:
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: GoBuildModule, Language: "go"})
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"})
case a.Kind == catalogue.ArtifactBundle && a.Language == "go":
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: GoBuildModule, Language: "go"})
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.GoBuildModule, Language: "go"})
case a.Kind == catalogue.ArtifactBundle && a.Language == "typescript":
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: NodeBuildModule, Language: "typescript"})
p.Tasks = append(p.Tasks, Task{Artifact: a.Name, Kind: a.Kind, Module: buildrun.NodeBuildModule, Language: "typescript"})
case a.Kind == catalogue.ArtifactBundle:
return Plan{}, &Refused{Artifact: a.Name, What: fmt.Sprintf("bundle of language %q", a.Language)}
default:
modules/mesh-delivery-builder/internal/build/plan_test.go
+5 -4
View File
@@ -2,6 +2,7 @@ package build
import (
"errors"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strings"
"testing"
@@ -23,9 +24,9 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
t.Fatal(err)
}
want := []Task{
{Artifact: "files", Kind: "archive", Module: GoBuildModule, Language: "go"},
{Artifact: "tool", Kind: "bundle", Module: GoBuildModule, Language: "go"},
{Artifact: "web", Kind: "bundle", Module: NodeBuildModule, Language: "typescript"},
{Artifact: "files", Kind: "archive", Module: buildrun.GoBuildModule, Language: "go"},
{Artifact: "tool", Kind: "bundle", Module: buildrun.GoBuildModule, Language: "go"},
{Artifact: "web", Kind: "bundle", Module: buildrun.NodeBuildModule, Language: "typescript"},
}
if len(p.Tasks) != len(want) {
t.Fatalf("tasks = %+v, want %+v", p.Tasks, want)
@@ -35,7 +36,7 @@ func TestPlanRoutesEachKindToItsBuildModule(t *testing.T) { // 1607/AC2
t.Errorf("task %d = %+v, want %+v", i, p.Tasks[i], want[i])
}
}
if modules := p.Modules(); len(modules) != 2 || modules[0] != GoBuildModule || modules[1] != NodeBuildModule {
if modules := p.Modules(); len(modules) != 2 || modules[0] != buildrun.GoBuildModule || modules[1] != buildrun.NodeBuildModule {
t.Errorf("modules = %v", modules)
}
}
modules/mesh-delivery-builder/internal/build/presence.go
-53
View File
@@ -1,53 +0,0 @@
package build
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// PresenceSet holds which nodes are online, as the watchman writes it (mesh-delivery-watchman_presence, key `<node>`, the
// node-engine's connection): a node asleep or off the bus is offline and its build module cannot answer. A node the
// watchman has no key of is not online. Safe for use from the watch and from a build at once.
type PresenceSet struct {
mu sync.Mutex
online map[string]bool
}
// Put takes a change of the presence bucket in and answers whether a node's online state changed. A part's key and a
// value that is not a presence are left out.
func (s *PresenceSet) Put(key string, value []byte, deleted bool) bool {
node, part, _, err := delivery.ParsePresenceKey(key)
if err != nil || part != "" {
return false
}
on := false
if !deleted {
p, err := delivery.DecodePresence(value)
if err != nil {
return false
}
on = p.Online
}
s.mu.Lock()
defer s.mu.Unlock()
if s.online[node] == on {
return false
}
if s.online == nil {
s.online = map[string]bool{}
}
if on {
s.online[node] = true
} else {
delete(s.online, node)
}
return true
}
// Online is whether the watchman last saw the node online.
func (s *PresenceSet) Online(node string) bool {
s.mu.Lock()
defer s.mu.Unlock()
return s.online[node]
}
modules/mesh-delivery-builder/internal/build/presence_test.go
-43
View File
@@ -1,43 +0,0 @@
package build
import (
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func presenceValue(t *testing.T, online bool) []byte {
t.Helper()
b, err := delivery.EncodePresence(delivery.Presence{Online: online})
if err != nil {
t.Fatal(err)
}
return b
}
func TestPresenceSetKnowsOnlyNodesTheWatchmanSawOnline(t *testing.T) { // 1607 review 3
var s PresenceSet
if s.Online("ace") {
t.Error("a node the watchman never wrote is online")
}
if !s.Put("ace", presenceValue(t, true), false) || !s.Online("ace") {
t.Error("ace online was not taken in")
}
if s.Put("ace", presenceValue(t, true), false) {
t.Error("the same presence again counted as a change")
}
if !s.Put("ace", presenceValue(t, false), false) || s.Online("ace") {
t.Error("ace going offline was not taken in")
}
s.Put("g14", presenceValue(t, true), false)
if !s.Put("g14", nil, true) || s.Online("g14") {
t.Error("a deleted key is still online")
}
// A part's instance (part.<part>.<instance>) is not a node.
if s.Put("part.mesh-delivery-builder.novox", presenceValue(t, true), false) || s.Online("part") {
t.Error("a part's key was taken for a node")
}
if s.Put("novox", []byte("junk"), false) {
t.Error("junk counted")
}
}
modules/mesh-delivery-builder/internal/build/round4_test.go
+23 -22
View File
@@ -2,6 +2,7 @@ package build
import (
"context"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strings"
"testing"
@@ -40,14 +41,14 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes
func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1
r := newRig(t, archiveA)
r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait
r.b.Health = &HaveSet{} // no build module is healthy anywhere
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait
r.b.Health = &buildrun.HaveSet{} // no build module is healthy anywhere
r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil))
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") {
t.Errorf("build = %+v, want failed: no node's build module is healthy", v)
}
have := &HaveSet{}
have := &buildrun.HaveSet{}
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
r.b.Health = have
r.b.Reconsider(context.Background())
@@ -60,11 +61,11 @@ func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *te
func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1
p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}})
if err != nil || !p.NeedsToolchain(GoBuildModule) {
t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(GoBuildModule), err)
if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) {
t.Errorf("a plan with a Go bundle: %v, %v", p.NeedsToolchain(buildrun.GoBuildModule), err)
}
p, _ = PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{archiveA}}})
if p.NeedsToolchain(GoBuildModule) {
if p.NeedsToolchain(buildrun.GoBuildModule) {
t.Error("a plan of an archive needs a toolchain")
}
}
@@ -90,7 +91,7 @@ func TestDifferingDigestsFailTheKeyWithThePrefixAndNothingBuildsItAgain(t *testi
func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review round 4, 2
st := newMem()
seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"})
seed(t, st, delivery.BuilderBuild{State: delivery.BuildFailed, Error: buildrun.DigestsDifferPrefix + " the nodes built tool to different digests: ace x; novox y"})
writes := st.writes
got, err := (Claimer{Store: st, By: "novox.a"}).Claim(context.Background(), "m", fp1, "", ClaimOptions{TakeOver: true})
if err != nil || got != nil || st.writes != writes {
@@ -103,10 +104,10 @@ func TestADoneKeyOfOneNodeIsBuiltAgainOnASecondNodeAndTheDigestsCompared(t *test
r := newRig(t, goBundle)
r.pres.Put("ace", presenceValue(t, false), false) // only novox qualifies
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") {
if v.State != delivery.BuildDone || !strings.HasPrefix(v.Step, OneNodePrefix+buildrun.GoBuildModule+"): ") || !strings.Contains(v.Step, "on one node, novox") {
t.Fatalf("build = %+v, want done on one node, marked", v)
}
if got := OneNodeModules(v); len(got) != 1 || got[0] != GoBuildModule {
if got := OneNodeModules(v); len(got) != 1 || got[0] != buildrun.GoBuildModule {
t.Errorf("OneNodeModules = %v", got)
}
rev, calls := r.st.revs[r.key], len(r.bm.callsOf("build_go_build"))
@@ -141,7 +142,7 @@ func TestASecondNodeThatDisagreesFailsTheDoneKeyWithTheDigestsPrefix(t *testing.
r.pres.Put("ace", presenceValue(t, true), false)
r.b.Reconsider(context.Background())
r.b.Wait()
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, DigestsDifferPrefix) {
if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, buildrun.DigestsDifferPrefix) {
t.Errorf("build = %+v, want failed with digests differ", v)
}
}
@@ -161,7 +162,7 @@ func TestASecondBuildThatFailsForAnotherReasonLeavesTheDoneKeyStanding(t *testin
// 4. A step never waits on a full node while another qualifying node has room.
func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4
p := &Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}}
p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}}
entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks")
total := 2 + 1 + 1 + 3
var waiting int
@@ -170,7 +171,7 @@ func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoo
for _, n := range []string{"ace", "g14", "novox", "shanks"} {
room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n]
}
nodes, _, err := ChooseNodes("go", entries, p, nil, 2)
nodes, _, err := buildrun.ChooseNodes("go", entries, p, nil, 2)
if err != nil || len(nodes) != 2 {
t.Fatalf("choice %d: %v, %v", i, nodes, err)
}
@@ -199,8 +200,8 @@ func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoo
}
func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4
p := &Places{}
nodes, _, _ := ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
p := &buildrun.Places{}
nodes, _, _ := buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
if len(nodes) != 1 || p.Reserved()["ace"] != 1 {
t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved())
}
@@ -209,7 +210,7 @@ func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *t
t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running())
}
release()
ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1)
p.Cancel("ace")
if len(p.Reserved()) != 0 {
t.Errorf("reserved after Cancel = %v", p.Reserved())
@@ -218,25 +219,25 @@ func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *t
// 5. A node whose build module is not healthy is not chosen.
func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5
var s HaveSet
if s.Healthy("ace", GoBuildModule) {
var s buildrun.HaveSet
if s.Healthy("ace", buildrun.GoBuildModule) {
t.Error("a module with no have is healthy")
}
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", GoBuildModule) {
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", buildrun.GoBuildModule) {
t.Error("a running, healthy module is not")
}
if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) {
t.Error("the same state counted as a change")
}
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", GoBuildModule) {
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", buildrun.GoBuildModule) {
t.Error("a module whose health failed is healthy")
}
s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", GoBuildModule) {
if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", buildrun.GoBuildModule) {
t.Error("a module whose phase failed is healthy")
}
s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", GoBuildModule) {
if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", buildrun.GoBuildModule) {
t.Error("a deleted have is healthy")
}
if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) {
@@ -246,7 +247,7 @@ func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review
func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5
r := newRig(t, goBundle)
have := &HaveSet{}
have := &buildrun.HaveSet{}
have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's
have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false)
r.b.Health = have
modules/mesh-delivery-builder/internal/build/run.go
+63 -20
View File
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"sort"
"strings"
"sync"
@@ -12,13 +13,13 @@ import (
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// ToolchainEntries are the nodes' toolchain entries of a language (ToolchainSet).
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
type ToolchainEntries interface {
Entries(language string) []delivery.NodeToolchain
}
// toolPrefix is the start of each build module's tool names: <prefix>fetch and <prefix>build.
var toolPrefix = map[string]string{GoBuildModule: "build_go_", NodeBuildModule: "build_node_"}
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}
// Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is
// handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an
@@ -27,7 +28,7 @@ var toolPrefix = map[string]string{GoBuildModule: "build_go_", NodeBuildModule:
// change of the toolchains.
type Builder struct {
Claimer Claimer
Runner Runner
Runner buildrun.Runner
Toolchains ToolchainEntries
// Presence says which nodes are online (the watchman's); a node that is not is not chosen.
Presence interface{ Online(node string) bool }
@@ -37,8 +38,8 @@ type Builder struct {
Health interface {
Healthy(node, module string) bool
}
// Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node.
Places *Places
// buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node.
Places *buildrun.Places
// Nodes is how many build nodes build each fingerprint: two until release R4e (ADR 0478 clause 4).
Nodes int
// Limit is how many builds run at once; four when 0.
@@ -49,11 +50,14 @@ type Builder struct {
sem chan struct{}
wg sync.WaitGroup
mu sync.Mutex
known map[string]delivery.Module // by modules key
inflight map[string]bool // by build key
again map[string]delivery.Module // by build key: the newest value that arrived while the key was being built
plans map[string]planned // by manifest digest
tried map[string]bool // by build key: looked at since this process started
known map[string]delivery.Module // by modules key
inflight map[string]bool // by build key
again map[string]delivery.Module // by build key: the newest value that arrived while the key was being built
plans map[string]planned // by manifest digest
cands map[string]delivery.Candidate // by candidates key (candidates.go)
via map[string]via // by build key: a candidate's fetch (candidates.go)
trunkFP map[string]bool // by build key: a trunk module value carries this fingerprint
tried map[string]bool // by build key: looked at since this process started
// epoch counts the changes of the nodes (online, healthy, a toolchain) seen since this process started; a done key
// whose second-node build failed is held back until it moves (issue 1664).
epoch uint64
@@ -72,10 +76,11 @@ func (b *Builder) init() {
limit = 4
}
b.sem = make(chan struct{}, limit)
b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{}
b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{}
b.secondAt = map[string]uint64{}
if b.Places == nil {
b.Places = &Places{}
b.Places = &buildrun.Places{}
}
})
}
@@ -100,6 +105,11 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) {
}
b.mu.Lock()
b.known[key] = m
if bk, err := delivery.BuilderBuildKey(key, m.FP2); err == nil && m.FP2 != "" {
// A fingerprint the trunk carries is built from the trunk's commit, whoever asked for it first.
b.trunkFP[bk] = true
delete(b.via, bk)
}
b.mu.Unlock()
b.consider(ctx, key, m)
}
@@ -181,6 +191,16 @@ func (b *Builder) Reconsider(ctx context.Context) {
for _, k := range sortedKeys(known) {
b.consider(ctx, k, known[k])
}
b.mu.Lock()
cands := make([]delivery.Candidate, 0, len(b.cands))
for _, c := range b.cands {
cands = append(cands, c)
}
b.mu.Unlock()
sort.Slice(cands, func(i, j int) bool { return cands[i].Group < cands[j].Group })
for _, c := range cands {
b.considerCandidate(ctx, c)
}
}
func sortedKeys(m map[string]delivery.Module) []string {
@@ -293,7 +313,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) {
return
}
result.By, result.Manifest = b.Claimer.By, m.Manifest
if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, DigestsDifferPrefix) {
if cl.Build.State == delivery.BuildDone && result.State == delivery.BuildFailed && !strings.HasPrefix(result.Error, buildrun.DigestsDifferPrefix) {
// A done key built again on a second node keeps its done value when the new build fails for another reason than
// the digests: the first build stands, and a later change of the nodes looks again.
b.logf("%s: built again on a second node and failed, the done build stands: %s", cl.Key, result.Error)
@@ -318,7 +338,7 @@ func (b *Builder) stale(m delivery.Module, plan Plan) string {
if !plan.NeedsToolchain(bm) {
continue // an archive is packed with no toolchain
}
language := LanguageOf[bm]
language := buildrun.LanguageOf[bm]
tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language))
if none != "" {
return fmt.Sprintf("the %s toolchain is not agreed: %s", language, none)
@@ -367,12 +387,12 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
}
}
for _, bm := range plan.Modules() {
language := LanguageOf[bm]
language := buildrun.LanguageOf[bm]
entries := b.Toolchains.Entries(language)
qualifies := func(node string) bool { return b.qualifies(node, bm) }
var nodes []string
if plan.NeedsToolchain(bm) {
ns, tc, err := ChooseNodes(language, entries, b.Places, qualifies, b.Nodes)
ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, b.Nodes)
if err != nil {
giveUpLanes()
return failed("%s", err)
@@ -386,7 +406,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
nodes = ns
} else {
// An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies.
ns, err := ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes)
ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, b.Nodes)
if err != nil {
giveUpLanes()
return failed("%s", err)
@@ -434,7 +454,7 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p
byNode[l.node] = d
}
}
d, err := Agree(t.Artifact, byNode)
d, err := buildrun.Agree(t.Artifact, byNode)
if err != nil {
return failed("%s", err)
}
@@ -463,7 +483,7 @@ func (b *Builder) moreNodes(modules []string) bool {
for _, bm := range modules {
seen := map[string]bool{}
n := 0
for _, e := range b.Toolchains.Entries(LanguageOf[bm]) {
for _, e := range b.Toolchains.Entries(buildrun.LanguageOf[bm]) {
if !seen[e.Node] && b.qualifies(e.Node, bm) {
seen[e.Node] = true
n++
@@ -489,18 +509,41 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del
return
}
defer release()
raw, err := b.step(ctx, l.module, l.node, "fetch", map[string]any{"repository": repository, "commit": m.Commit})
fetchArgs := map[string]any{"repository": repository, "commit": m.Commit}
if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil {
b.mu.Lock()
v, candidate := b.via[key]
b.mu.Unlock()
if candidate {
fetchArgs["ref"], fetchArgs["base"] = v.Ref, v.Base
}
}
raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs)
if err != nil {
l.err = err
return
}
var fetched struct {
Workspace string `json:"workspace"`
Workspace string `json:"workspace"`
Merge string `json:"merge"`
Conflicts []string `json:"conflicts"`
}
if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" {
l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw)
return
}
// A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at
// the trunk, and building that would store the trunk's build under the candidate's fingerprint.
if _, candidate := fetchArgs["base"]; candidate {
if len(fetched.Conflicts) > 0 {
l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", "))
return
}
if fetched.Merge == "" {
l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw)
return
}
}
for _, t := range l.tasks {
raw, err := b.step(ctx, l.module, l.node, "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact})
if err != nil {
modules/mesh-delivery-builder/internal/build/run_test.go
+24 -13
View File
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"strings"
"sync"
"testing"
@@ -23,8 +24,11 @@ type buildModules struct {
digests map[string]string // `<node>/<artifact>` -> digest; the default is `digest`
fails map[string]string // `<node>/<tool>` -> the reason the step fails with
// hold, when set before the builds start, blocks every call until it is closed; entered is sent to as a call arrives.
hold chan struct{}
entered chan struct{}
// conflicts, when set, is a file a fetch with a base reports in conflict; noMerge makes it answer neither a merge nor a conflict.
conflicts string
noMerge bool
hold chan struct{}
entered chan struct{}
}
func newBuildModules() *buildModules {
@@ -51,6 +55,13 @@ func (b *buildModules) CallOn(node, key string, body any) (json.RawMessage, erro
step.State, step.Reason = delivery.StepFailed, reason
} else if strings.HasSuffix(tool, "_fetch") {
step.Result = json.RawMessage(`{"workspace":"w-` + node + `"}`)
if args["base"] != nil { // a fetch with a base answers the merge, or the files in conflict (mesh-build-go FetchResult)
if b.conflicts != "" {
step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","conflicts":["` + b.conflicts + `"]}`)
} else if !b.noMerge {
step.Result = json.RawMessage(`{"workspace":"w-` + node + `","base":"` + fmt.Sprint(args["base"]) + `","merge":"` + strings.Repeat("9", 40) + `"}`)
}
}
} else {
d := digest
if v, ok := b.digests[node+"/"+fmt.Sprint(args["artifact"])]; ok {
@@ -94,22 +105,22 @@ type rig struct {
b *Builder
st *memStore
bm *buildModules
tools *ToolchainSet
pres *PresenceSet
tools *buildrun.ToolchainSet
pres *buildrun.PresenceSet
mf *fakeManifests
key string
}
func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig {
t.Helper()
r := &rig{st: newMem(), bm: newBuildModules(), tools: &ToolchainSet{}, pres: &PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, pres: &buildrun.PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}}
for _, n := range []string{"ace", "novox"} {
r.tools.Put(GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
r.tools.Put(NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false)
r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false)
r.pres.Put(n, presenceValue(t, true), false)
}
r.b = &Builder{
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second},
Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second},
Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Nodes: 2, Logf: t.Logf,
}
var err error
@@ -202,7 +213,7 @@ func TestDifferingDigestsFailTheBuildNamingBothNodes(t *testing.T) { // 1607/AC4
func TestOneQualifyingNodeBuildsOnceAndTheKeySaysSo(t *testing.T) { // 1607/AC4, ADR 0478 clause 4
r := newRig(t, goBundle)
r.tools.Put(GoBuildModule, "ace.go", nil, true)
r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true)
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone || !strings.Contains(v.Step, "one node") || !strings.Contains(v.Step, "novox") {
t.Errorf("build = %+v, want done with its step saying one node, novox", v)
@@ -340,8 +351,8 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
t.Errorf("keys %v and %d calls for a recipe that differs or a value that names no toolchain", r.st.vals, len(r.bm.calls))
}
// The toolchain moves under a key that is not built yet: the old fingerprint stays unbuilt, the new one builds.
r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false)
r.tools.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false)
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.2"), false)
r.tools.Put(buildrun.GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.2"), false)
r.b.Reconsider(context.Background())
r.b.Wait()
if len(r.st.vals) != 0 {
@@ -356,7 +367,7 @@ func TestAKeyIsBuiltOnlyWithTheToolchainItsFingerprintNamesAndNothingIsWrittenOt
func TestNodesTheyDisagreeOnLeaveTheKeyUnwritten(t *testing.T) { // 1615/AC4
r := newRig(t, goBundle)
r.tools.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false)
r.b.Module(context.Background(), "m", moduleValue(t, "modules/m"))
r.b.Wait()
if len(r.st.vals) != 0 || len(r.bm.calls) != 0 {
@@ -376,7 +387,7 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/
func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3
r := newRig(t, goBundle)
r.tools.Put(GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps
r.tools.Put(buildrun.GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps
v := r.consider(t, "modules/m")
if v.State != delivery.BuildDone {
t.Fatalf("build = %+v", v)
modules/mesh-delivery-builder/internal/build/steps.go
-67
View File
@@ -1,67 +0,0 @@
package build
import (
"context"
"encoding/json"
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Caller calls a tool of a module on one named machine (mesh-sdk process.Conn.CallOn; ADR 0469 clause 3).
type Caller interface {
CallOn(node, key string, body any) (json.RawMessage, error)
}
// Steps waits for the end of a build module's long step: the value of its key `<node>.<id>` in the module's steps state
// (ADR 0469 clause 2) once its state is not running. It is a watch of that key, not a poll; it returns when the step
// ends or ctx does.
type Steps interface {
Wait(ctx context.Context, module, key string) (delivery.Step, error)
}
// Runner runs a build module's long steps: the call answers at once with the step's key, and the result is read from that
// key when the step ends (ADR 0469 clause 2).
type Runner struct {
Caller Caller
Steps Steps
// Bound is how long a step may run before the builder stops waiting for it.
Bound time.Duration
}
// Run calls tool of the build module on node with args, waits for the step's end and answers its result. A step that
// failed or was refused is an error naming the node, the state and the reason.
func (r Runner) Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
raw, err := r.Caller.CallOn(node, module+"."+tool, args)
if err != nil {
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
}
var answer struct {
Step string `json:"step"`
Key string `json:"key"`
State string `json:"state"`
}
if err := json.Unmarshal(raw, &answer); err != nil {
return nil, fmt.Errorf("%s on %s answered what is not a step: %w", tool, node, err)
}
if answer.Key == "" {
return nil, fmt.Errorf("%s on %s answered no step key: %s", tool, node, raw)
}
if n, _, err := delivery.ParseStepKey(answer.Key); err != nil || n != node {
return nil, fmt.Errorf("%s on %s answered the step key %q, which is not a key of that node", tool, node, answer.Key)
}
wait, cancel := context.WithTimeoutCause(ctx, r.Bound, fmt.Errorf("the step %s did not end within %s", answer.Key, r.Bound))
defer cancel()
step, err := r.Steps.Wait(wait, module, answer.Key)
if err != nil {
if cause := context.Cause(wait); cause != nil && ctx.Err() == nil {
err = cause
}
return nil, fmt.Errorf("%s on %s: %w", tool, node, err)
}
if step.State != delivery.StepPassed {
return nil, fmt.Errorf("%s on %s: the step %s %s: %s", tool, node, answer.Key, step.State, step.Reason)
}
return step.Result, nil
}
modules/mesh-delivery-builder/internal/build/steps_test.go
-116
View File
@@ -1,116 +0,0 @@
package build
import (
"context"
"encoding/json"
"errors"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
type call struct {
node, key string
body any
}
// fakeCaller answers a long tool as the build modules do: at once, with the step and its key.
type fakeCaller struct {
calls []call
answer string
err error
}
func (f *fakeCaller) CallOn(node, key string, body any) (json.RawMessage, error) {
f.calls = append(f.calls, call{node, key, body})
if f.err != nil {
return nil, f.err
}
return json.RawMessage(f.answer), nil
}
// fakeSteps holds the steps values by `<module>/<key>`; Wait on a key it does not hold blocks until the context ends.
type fakeSteps struct {
mu sync.Mutex
steps map[string]delivery.Step
// during, when set, is called as a wait starts and the function it answers as the wait ends.
during func(key string) func()
}
func (f *fakeSteps) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
f.mu.Lock()
s, ok := f.steps[module+"/"+key]
during := f.during
f.mu.Unlock()
if ok {
if during != nil {
defer during(key)()
}
return s, nil
}
<-ctx.Done()
return delivery.Step{}, context.Cause(ctx)
}
func stepAt(node, id, state, reason string, result string) delivery.Step {
s := delivery.Step{ID: id, Tool: "build_go_fetch", Node: node, State: state, Reason: reason}
if result != "" {
s.Result = json.RawMessage(result)
}
return s
}
func TestRunCallsTheToolOnTheNodeAndReadsTheStepsKey(t *testing.T) { // 1607/AC5
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", delivery.StepPassed, "", `{"workspace":"w1","commit":"c"}`)}}
res, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", map[string]any{"repository": "novox/r", "commit": "c"})
if err != nil {
t.Fatal(err)
}
if len(caller.calls) != 1 || caller.calls[0].node != "ace" || caller.calls[0].key != "mesh-build-go.build_go_fetch" {
t.Errorf("calls = %+v, want one on ace for mesh-build-go.build_go_fetch", caller.calls)
}
var fr struct{ Workspace string }
if err := json.Unmarshal(res, &fr); err != nil || fr.Workspace != "w1" {
t.Errorf("result = %s (%v), want the step's result", res, err)
}
}
func TestRunFailsOnAFailedOrRefusedStepNamingItsReason(t *testing.T) { // 1607/AC5
for _, state := range []string{delivery.StepFailed, delivery.StepRefused} {
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
steps := &fakeSteps{steps: map[string]delivery.Step{"mesh-build-go/ace.abc1": stepAt("ace", "abc1", state, "go is go1.26.0, the recipe builds with go1.27.1", "")}}
_, err := Runner{Caller: caller, Steps: steps, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
if err == nil || !strings.Contains(err.Error(), "go1.27.1") || !strings.Contains(err.Error(), state) || !strings.Contains(err.Error(), "ace") {
t.Errorf("%s: error = %v, want the node, the state and the reason", state, err)
}
}
}
func TestRunStopsWaitingAtTheBound(t *testing.T) { // 1607/AC5
caller := &fakeCaller{answer: `{"step":"abc1","key":"ace.abc1","state":"running"}`}
_, err := Runner{Caller: caller, Steps: &fakeSteps{}, Bound: 20 * time.Millisecond}.Run(context.Background(), "mesh-build-go", "ace", "build_go_build", nil)
if err == nil || !strings.Contains(err.Error(), "ace.abc1") || !strings.Contains(err.Error(), "did not end") {
t.Errorf("error = %v, want the step key and that it did not end within its bound", err)
}
}
func TestRunRefusesAnAnswerWithNoKeyAndAKeyOfAnotherNode(t *testing.T) { // 1607/AC5
for _, answer := range []string{`{"state":"running"}`, `{"step":"abc1","key":"novox.abc1","state":"running"}`, `not json`} {
_, err := Runner{Caller: &fakeCaller{answer: answer}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
if err == nil {
t.Errorf("answer %s: no error", answer)
}
}
}
func TestRunReturnsTheCallsOwnError(t *testing.T) { // 1607/AC5
boom := errors.New("no responders")
_, err := Runner{Caller: &fakeCaller{err: boom}, Steps: &fakeSteps{}, Bound: time.Second}.Run(context.Background(), "mesh-build-go", "ace", "build_go_fetch", nil)
if !errors.Is(err, boom) || !strings.Contains(err.Error(), "ace") {
t.Errorf("error = %v, want the call's error naming the node", err)
}
}
modules/mesh-delivery-builder/internal/build/toolchains.go
-64
View File
@@ -1,64 +0,0 @@
package build
import (
"sync"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// LanguageOf is the language each build module's toolchains state holds (the indexer's ToolchainModules, reversed).
var LanguageOf = map[string]string{GoBuildModule: "go", NodeBuildModule: "typescript"}
// ToolchainSet holds what the build modules' toolchains buckets hold: each node's entry of a language, as the watches
// put them in. Safe for use from the watches and from a build at once.
type ToolchainSet struct {
mu sync.Mutex
by map[string]map[string]delivery.Toolchain // language -> node -> entry
}
// Put takes a change of a build module's toolchains bucket in: a put of a node's entry (key <node>.<language>) or its
// delete. It answers whether the language's entries changed (a node's version or recipe, or a node appearing or going).
// A key or a value that is not an entry of the module's language is left out.
func (s *ToolchainSet) Put(module, key string, value []byte, deleted bool) bool {
language, ok := LanguageOf[module]
if !ok {
return false
}
node, keyLanguage, err := delivery.ParseToolchainKey(key)
if err != nil || keyLanguage != language {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
if deleted {
if _, had := s.by[language][node]; !had {
return false
}
delete(s.by[language], node)
return true
}
tc, err := delivery.DecodeToolchain(value)
if err != nil || tc.Language != language {
return false
}
if s.by == nil {
s.by = map[string]map[string]delivery.Toolchain{}
}
if s.by[language] == nil {
s.by[language] = map[string]delivery.Toolchain{}
}
cur, had := s.by[language][node]
s.by[language][node] = tc
return !had || cur.Version != tc.Version || cur.Recipe != tc.Recipe
}
// Entries are the nodes' entries of a language, in no particular order.
func (s *ToolchainSet) Entries(language string) []delivery.NodeToolchain {
s.mu.Lock()
defer s.mu.Unlock()
var out []delivery.NodeToolchain
for node, tc := range s.by[language] {
out = append(out, delivery.NodeToolchain{Node: node, Toolchain: tc})
}
return out
}
modules/mesh-delivery-builder/internal/build/toolchains_test.go
-51
View File
@@ -1,51 +0,0 @@
package build
import (
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func tcValue(t *testing.T, language, version string) []byte {
t.Helper()
b, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: "1", At: time.Now()})
if err != nil {
t.Fatal(err)
}
return b
}
func TestToolchainSetKeepsEachNodesEntryByLanguageAndSaysWhenOneChanged(t *testing.T) { // 1607/AC6
var s ToolchainSet
if !s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
t.Error("a first entry did not count as a change")
}
if s.Put(GoBuildModule, "novox.go", tcValue(t, "go", "go1.27.1"), false) {
t.Error("the same entry again counted as a change")
}
s.Put(GoBuildModule, "ace.go", tcValue(t, "go", "go1.27.1"), false)
s.Put(NodeBuildModule, "ace.typescript", tcValue(t, "typescript", "tsc5"), false)
if got := s.Entries("go"); len(got) != 2 {
t.Errorf("go entries = %+v, want two", got)
}
if got := s.Entries("typescript"); len(got) != 1 || got[0].Node != "ace" || got[0].Version != "tsc5" {
t.Errorf("typescript entries = %+v", got)
}
if !s.Put(GoBuildModule, "ace.go", nil, true) || len(s.Entries("go")) != 1 {
t.Error("a delete did not remove the node's entry")
}
}
func TestToolchainSetLeavesOutWhatIsNotAToolchainOfTheModulesLanguage(t *testing.T) { // 1607/AC6
var s ToolchainSet
if s.Put(GoBuildModule, "novox.typescript", tcValue(t, "typescript", "x"), false) {
t.Error("a typescript toolchain in the go build module's bucket counted")
}
if s.Put(GoBuildModule, "novox.go", []byte("not json"), false) || s.Put("other-module", "novox.go", tcValue(t, "go", "x"), false) {
t.Error("garbage or an unknown module counted")
}
if len(s.Entries("go")) != 0 {
t.Error("an entry was kept")
}
}
modules/mesh-delivery-builder/manifest_test.go
+1 -1
View File
@@ -87,7 +87,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
"mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
modules/mesh-delivery-builder/module.json
+1
View File
@@ -14,6 +14,7 @@
"mesh-build-go.toolchains",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-watchman.presence",
modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go
+71
View File
@@ -0,0 +1,71 @@
// Command mesh-delivery-checker-tools serves the checker's read tools (novox/hq ADR 0484 clause 13, issue 1686): the runtime
// launches it for a call, it reads the checker's verdicts and kept states and the indexer's candidates through the checker's
// state and reads, and writes nothing. The checker is the other program: it serves no tool (ADR 0437 §4).
package main
import (
"encoding/json"
"fmt"
"os"
stdio "git.novox.be/novox/mesh-sdk/go"
"mesh-delivery-checker/internal/check"
)
func read(s stdio.KeptState) check.StateRead {
return check.StateRead{Keys: s.Keys, Get: func(key string) (json.RawMessage, error) {
e, err := s.Get(key)
if err != nil || e == nil {
return nil, err
}
return e.Value, nil
}}
}
// Tools are the read tools over the checker's states.
func Tools(r check.Reader) []stdio.Tool {
return []stdio.Tool{
{Name: check.VerdictsReadTool,
Description: "The verdicts the checker holds, each as stored under verdicts.<group>.<heads>: its state, what failed first, the tests run, kept and skipped per level, the time of each part, the trunk heads it judged against and the composition, with the full " +
"report of the status each pull request carries (the line the forge shows is cut at 140 characters, this is not). With a group, that group's only. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"group": map[string]any{"type": "string", "description": "the delivery group's branch name"}}},
Run: func(args map[string]any) (any, error) {
group, _ := args["group"].(string)
vs, err := r.Verdicts(group)
return map[string]any{"verdicts": vs}, err
}},
{Name: check.PullReadTool,
Description: "The verdict that judges one pull request and the full report of its status mesh/check; for an hq pull request, that hq is judged by " +
"merge-check.sh until issue 1296. Read-only.",
Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{
"repository": map[string]any{"type": "string", "description": "<owner>/<name>"},
"number": map[string]any{"type": "integer", "description": "the pull request's number"}}},
Run: func(args map[string]any) (any, error) {
repository, _ := args["repository"].(string)
n, _ := args["number"].(float64)
return r.Pull(repository, int(n))
}},
{Name: check.KeptReadTool,
Description: "The kept verdicts of passing packages, each as stored under kept.<level>.<digest>: the package, its tests, the time they took, the toolchain " +
"and when it was kept. With a level, that level's; with a level and a digest, that one. Read-only.",
Input: map[string]any{"type": "object", "properties": map[string]any{
"level": map[string]any{"type": "string", "enum": []string{"unit", "integration"}, "description": "the level"},
"digest": map[string]any{"type": "string", "description": "the package's digest, with a level"}}},
Run: func(args map[string]any) (any, error) {
level, _ := args["level"].(string)
digest, _ := args["digest"].(string)
kept, err := r.Kept(level, digest)
return map[string]any{"kept": kept}, err
}},
}
}
func main() {
r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates"))}
if err := stdio.Serve("", Tools(r)); err != nil {
fmt.Fprintf(os.Stderr, "[mesh-delivery-checker-tools] %v\n", err)
os.Exit(1)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main_test.go
+27
View File
@@ -0,0 +1,27 @@
package main
import (
"testing"
"mesh-delivery-checker/internal/check"
)
func TestTheToolsServedAreTheThreeReadTools(t *testing.T) {
var names []string
for _, tl := range Tools(check.Reader{}) {
names = append(names, tl.Name)
if tl.Description == "" || tl.Run == nil {
t.Errorf("%s has no description or no run", tl.Name)
}
}
want := []string{check.VerdictsReadTool, check.PullReadTool, check.KeptReadTool}
if len(names) != 3 || names[0] != want[0] || names[1] != want[1] || names[2] != want[2] {
t.Errorf("tools = %v, want %v", names, want)
}
// With nothing to read from a tool says so and changes nothing.
for _, tl := range Tools(check.Reader{}) {
if _, err := tl.Run(map[string]any{"repository": "novox/x", "number": float64(1)}); err == nil {
t.Errorf("%s answered with no state to read", tl.Name)
}
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go
+159
View File
@@ -0,0 +1,159 @@
package main
import (
"context"
"errors"
"fmt"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-checker/internal/check"
)
// bucketOpener is the part of the bus the watches use.
type bucketOpener interface {
KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error)
}
// kvStore is the builds bucket as the builder's store: create and compare-and-set.
type kvStore struct{ kv jetstream.KeyValue }
func conflict(err error) error {
if errors.Is(err, jetstream.ErrKeyExists) || errors.Is(err, jetstream.ErrKeyRevisionMismatch) {
return check.ErrConflict
}
return err
}
func (s kvStore) Get(ctx context.Context, key string) ([]byte, uint64, bool, error) {
e, err := s.kv.Get(ctx, key)
if errors.Is(err, jetstream.ErrKeyNotFound) {
return nil, 0, false, nil
}
if err != nil {
return nil, 0, false, err
}
return e.Value(), e.Revision(), true, nil
}
func (s kvStore) Create(ctx context.Context, key string, value []byte) (uint64, error) {
rev, err := s.kv.Create(ctx, key, value)
return rev, conflict(err)
}
func (s kvStore) Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error) {
rev, err := s.kv.Update(ctx, key, value, revision)
return rev, conflict(err)
}
func (s kvStore) Keys(ctx context.Context) ([]string, error) {
keys, err := s.kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return keys, err
}
func opOf(e jetstream.KeyValueEntry) bool { return e.Operation() == jetstream.KeyValuePut }
// settle reads the entries a watcher holds now, up to its mark of the last values, and returns the watcher for the
// changes that follow.
func settle(ctx context.Context, kv jetstream.KeyValue, each func(jetstream.KeyValueEntry)) (jetstream.KeyWatcher, error) {
w, err := kv.WatchAll(ctx)
if err != nil {
return nil, err
}
for {
select {
case <-ctx.Done():
w.Stop()
return nil, ctx.Err()
case e, ok := <-w.Updates():
if !ok {
return nil, errors.New("the watch ended")
}
if e == nil {
return w, nil
}
each(e)
}
}
}
// follow hands each change of a watcher to each until ctx ends; a watch that ends is an error, and the process manager
// starts the program again.
func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.KeyValueEntry)) error {
defer w.Stop()
for {
select {
case <-ctx.Done():
return nil
case e, ok := <-w.Updates():
if !ok {
return errors.New("a watch ended")
}
if e != nil {
each(e)
}
}
}
}
// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the
// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every
// change after. No timer: the caller bounds the wait with its context.
type stepsWaiter struct{ js bucketOpener }
func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) {
kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module))
if err != nil {
return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err)
}
watch, err := kv.Watch(ctx, key)
if err != nil {
return delivery.Step{}, err
}
defer watch.Stop()
for {
select {
case <-ctx.Done():
return delivery.Step{}, ctx.Err()
case e, ok := <-watch.Updates():
if !ok {
return delivery.Step{}, errors.New("the watch of the step ended")
}
if e == nil || !opOf(e) {
continue
}
step, err := delivery.DecodeStep(e.Value())
if err != nil {
return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err)
}
if step.State != delivery.StepRunning {
return step, nil
}
}
}
}
// manifestReader reads manifests.<digest> of the indexer and parses the manifest it holds.
type manifestReader struct{ kv jetstream.KeyValue }
func (m manifestReader) Manifest(ctx context.Context, digest string) (catalogue.Manifest, error) {
key, err := delivery.ManifestsKey(digest)
if err != nil {
return catalogue.Manifest{}, err
}
e, err := m.kv.Get(ctx, key)
if err != nil {
return catalogue.Manifest{}, fmt.Errorf("manifests.%s: %w", key, err)
}
kept, err := delivery.DecodeManifest(e.Value())
if err != nil {
return catalogue.Manifest{}, err
}
return catalogue.ParseManifest([]byte(kept.JSON))
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge.go
+80
View File
@@ -0,0 +1,80 @@
package main
import (
"context"
"encoding/json"
"fmt"
"strings"
)
// The forge's tools the checker reads with (mesh-catalog modules/gitea): gitea_get_file answers {file: {content}}, the text
// of a file at a ref, and gitea_compare answers {total_commits, commits}, what head holds that base does not. The checker
// writes nothing to the forge: the forge bridge sets the statuses (ADR 0484 clause 2).
const (
getFileTool = "gitea.gitea_get_file"
compareTool = "gitea.gitea_compare"
commitsTool = "gitea.gitea_commits"
)
// forgeReads reads the forge through call (mesh-sdk process.Conn.Call as the checker's own account).
type forgeReads struct {
call func(key string, body any) (json.RawMessage, error)
}
func ownerRepo(repository string) (string, string) {
owner, repo, _ := strings.Cut(repository, "/")
return owner, repo
}
// File is the text of a file of a repository at a ref.
func (f forgeReads) File(_ context.Context, repository, path, ref string) (string, error) {
owner, repo := ownerRepo(repository)
raw, err := f.call(getFileTool, map[string]string{"owner": owner, "repo": repo, "path": path, "ref": ref})
if err != nil {
return "", err
}
var a struct {
File *struct {
Content *string `json:"content"`
} `json:"file"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.File == nil || a.File.Content == nil {
return "", fmt.Errorf("the answer of %s is not {file: {content}}: %v", getFileTool, err)
}
return *a.File.Content, nil
}
// OnTrunk says whether the commit is already on the repository's main: main holds it when the commit holds nothing main does
// not.
func (f forgeReads) OnTrunk(_ context.Context, repository, commit string) (bool, error) {
owner, repo := ownerRepo(repository)
raw, err := f.call(compareTool, map[string]string{"owner": owner, "repo": repo, "base": "main", "head": commit})
if err != nil {
return false, err
}
var a struct {
Total *int `json:"total_commits"`
}
if err := json.Unmarshal(raw, &a); err != nil || a.Total == nil {
return false, fmt.Errorf("the answer of %s is not {total_commits}: %v", compareTool, err)
}
return *a.Total == 0, nil
}
// FullSHA is the whole sha of a commit named by its first digits: the newest commit of the history at that ref.
func (f forgeReads) FullSHA(_ context.Context, repository, commit string) (string, error) {
owner, repo := ownerRepo(repository)
raw, err := f.call(commitsTool, map[string]any{"owner": owner, "repo": repo, "ref": commit, "limit": 1})
if err != nil {
return "", err
}
var a struct {
Commits []struct {
SHA string `json:"sha"`
} `json:"commits"`
}
if err := json.Unmarshal(raw, &a); err != nil || len(a.Commits) == 0 || len(a.Commits[0].SHA) != 40 || !strings.HasPrefix(a.Commits[0].SHA, commit) {
return "", fmt.Errorf("the answer of %s for %s is not a commit that starts with it: %v", commitsTool, commit, err)
}
return a.Commits[0].SHA, nil
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/forge_test.go
+70
View File
@@ -0,0 +1,70 @@
package main
import (
"context"
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
// capturedAnswer is what the forge's tool answered (internal/check/testdata/captured/README).
func capturedAnswer(t *testing.T, name string) json.RawMessage {
t.Helper()
raw, err := os.ReadFile(filepath.Join("..", "..", "internal", "check", "testdata", "captured", name+".json"))
if err != nil {
t.Fatal(err)
}
return raw
}
func TestTheForgeReadsParseWhatTheForgeAnswered(t *testing.T) { // 1624/AC4
var asked []string
f := forgeReads{call: func(key string, body any) (json.RawMessage, error) {
b, _ := json.Marshal(body)
asked = append(asked, key+" "+string(b))
switch {
case key == getFileTool:
return capturedAnswer(t, "forge-get-file-go-mod"), nil
case strings.Contains(string(b), "cb54756c3865"):
return capturedAnswer(t, "forge-compare-trunk"), nil
}
return capturedAnswer(t, "forge-compare-branch"), nil
}}
text, err := f.File(context.Background(), "novox/mesh-delivery", "modules/mesh-delivery-builder/go.mod", "main")
if err != nil || !strings.Contains(text, "git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865") {
t.Errorf("file = %q, %v", text, err)
}
if on, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "cb54756c3865"); err != nil || !on {
t.Errorf("a commit on the trunk: %v, %v", on, err)
}
if on, err := f.OnTrunk(context.Background(), "novox/mesh-sdk", "9aaa5c80ac5c"); err != nil || on {
t.Errorf("a commit of the branch: %v, %v", on, err)
}
if len(asked) != 3 || !strings.Contains(asked[0], `"owner":"novox"`) || !strings.Contains(asked[2], `"base":"main"`) {
t.Errorf("asked %v", asked)
}
bad := forgeReads{call: func(string, any) (json.RawMessage, error) { return json.RawMessage(`{}`), nil }}
if _, err := bad.File(context.Background(), "novox/x", "go.mod", "main"); err == nil {
t.Error("an answer with no file was taken")
}
if _, err := bad.OnTrunk(context.Background(), "novox/x", "abc"); err == nil {
t.Error("an answer with no total was taken")
}
}
func TestTheForgeNamesTheWholeShaOfAShortCommit(t *testing.T) { // review D2
f := forgeReads{call: func(key string, body any) (json.RawMessage, error) {
if key != commitsTool {
t.Errorf("called %s", key)
}
return capturedAnswer(t, "forge-commits"), nil
}}
if sha, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "cb54756c3865"); err != nil || sha != "cb54756c3865df5ac9c0dc0481ad9ebe2bf013a1" {
t.Errorf("sha = %q, %v", sha, err)
}
if _, err := f.FullSHA(context.Background(), "novox/mesh-sdk", "deadbeefdead"); err == nil {
t.Error("a commit that does not start with the pin was taken")
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go
+101
View File
@@ -0,0 +1,101 @@
package main
import (
"context"
"errors"
"log"
"sync"
"github.com/nats-io/nats.go/jetstream"
"mesh-delivery-checker/internal/check"
)
// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): whether one does is the list of the keys of
// the controller's bucket of the delivery plans in their build phase, read each time a check is about to start. Its keys age
// out (a time to live, and no delete marker), which a watch never hears, so a watch cannot decide it (mesh-controller
// internal/link/holds.go lists the keys too); the watch is only the wake-up, for a plan that ended by a delete. The bucket
// is read as empty where it is absent (issue 1652 makes it) or cannot be read, saying so once. It may come into being after
// the checker started and the checker holds no timer (ADR 0437 §8), so each time a check is about to start and the bucket is
// not known yet, the checker looks for it.
type holdWatch struct {
ctx context.Context
js bucketOpener
keys func(ctx context.Context) ([]string, error)
changed func(ctx context.Context) // a plan key changed: the checks that waited may start
logger *log.Logger
errs chan error
mu sync.Mutex
watched bool
said bool
}
func newHoldWatch(ctx context.Context, js bucketOpener, changed func(context.Context), logger *log.Logger) *holdWatch {
return &holdWatch{ctx: ctx, js: js, changed: changed, logger: logger, errs: make(chan error, 1)}
}
// Held says whether a delivery builds now.
func (w *holdWatch) Held() bool {
w.ensure()
w.mu.Lock()
keys := w.keys
w.mu.Unlock()
if keys == nil {
return false
}
list, err := keys(w.ctx)
if err != nil {
w.say("the keys of %s cannot be listed, so it is read as empty: %v", check.PlansBuildingBucket, err)
return false
}
return len(list) > 0
}
func (w *holdWatch) say(format string, args ...any) {
w.mu.Lock()
defer w.mu.Unlock()
if !w.said {
w.said = true
w.logger.Printf(format, args...)
}
}
// ensure finds the bucket if it is not known and exists, and starts the wake-up watch.
func (w *holdWatch) ensure() {
w.mu.Lock()
if w.watched || w.js == nil {
w.mu.Unlock()
return
}
w.mu.Unlock()
kv, err := w.js.KeyValue(w.ctx, check.PlansBuildingBucket)
switch {
case errors.Is(err, jetstream.ErrBucketNotFound):
w.say("the bucket %s does not exist yet: it is read as empty, so no check is held back", check.PlansBuildingBucket)
return
case err != nil:
w.say("the bucket %s cannot be read, so it is read as empty: %v", check.PlansBuildingBucket, err)
return
}
w.mu.Lock()
defer w.mu.Unlock()
if w.watched {
return
}
w.watched, w.said = true, false
w.keys = func(ctx context.Context) ([]string, error) {
list, err := kv.Keys(ctx)
if errors.Is(err, jetstream.ErrNoKeysFound) {
return nil, nil
}
return list, err
}
go func() {
wake, err := kv.WatchAll(w.ctx)
if err != nil {
w.errs <- err
return
}
w.errs <- follow(w.ctx, wake, func(jetstream.KeyValueEntry) { w.changed(w.ctx) })
}()
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go
+35
View File
@@ -0,0 +1,35 @@
package main
import (
"context"
"errors"
"io"
"log"
"testing"
)
// The controller's plans-building keys age out of the bucket (a time to live, no delete marker), which a watch never hears:
// whether a delivery builds is the list of the keys now, and the watch is only a wake-up (review D3).
func TestADeliveryHoldsWhileItsKeyIsListedAndNotWhenItAgedOutWithNoEvent(t *testing.T) {
listed := []string{"plan-1"}
w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return listed, nil }}
if !w.Held() {
t.Fatal("a plan key is listed and nothing holds")
}
listed = nil // the key aged out: no delete reached the watch
if w.Held() {
t.Error("the key is gone from the bucket and the checker is still held")
}
}
func TestABucketThatCannotBeListedIsReadAsEmpty(t *testing.T) {
w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return []string{"plan-1"}, errors.New("the bus is down") }}
if w.Held() {
t.Error("an unreadable bucket held the checker")
}
var absent holdWatch
absent.logger = log.New(io.Discard, "", 0)
if absent.Held() {
t.Error("a watch with no bucket held the checker")
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go
+592
View File
@@ -0,0 +1,592 @@
//go:build integration
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/nats-io/nats-server/v2/server"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// busConf is the shape of the composed accounts file (mesh-controller internal/broker): one account with JetStream, the
// controller's user and the checker's own principal <node>.<module>. It carries no per-principal permissions: the grants
// the controller composes from the manifest's state and reads (only the checker writes its buckets) are the controller's,
// and are proved by its own module check, not here.
const busConf = `
listen: 127.0.0.1:-1
jetstream { store_dir: %q }
accounts {
MESH {
jetstream: enabled
users = [
{ user: "ctl", password: "pw" }
{ user: "n1.mesh-delivery-checker", password: "pw" }
]
}
}
`
const (
repo = "novox/mesh-delivery"
head1 = "1111111111111111111111111111111111111111"
head2 = "2222222222222222222222222222222222222222"
trunk = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
merged = "fedcba9876543210fedcba9876543210fedcba98" // the merge commit of the captured fetch answer
digU = "aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11"
digI = "bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22"
)
var captured = filepath.Join("..", "..", "internal", "check", "testdata", "captured")
func readCaptured(t *testing.T, name string) map[string]any {
t.Helper()
raw, err := os.ReadFile(filepath.Join(captured, name+".json"))
if err != nil {
t.Fatal(err)
}
var v map[string]any
if err := json.Unmarshal(raw, &v); err != nil {
t.Fatal(err)
}
return v
}
// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on
// mesh.mod.<module>.call.<tool>.<node>.<caller> answers at once with the step and its key, and the step's end is written to
// the key <node>.<id> of the module's steps bucket. What the steps end with are the results the build modules' own code
// marshalled (internal/check/testdata/captured, README).
type fakeBuildModules struct {
js jetstream.JetStream
ctx context.Context
n atomic.Int64
conflict atomic.Bool
mu sync.Mutex
calls map[string]int // by `<module>.<tool>`
testTool *testing.T
}
func (f *fakeBuildModules) count(tool string) int {
f.mu.Lock()
defer f.mu.Unlock()
return f.calls[tool]
}
func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any) json.RawMessage {
level, _ := args["level"].(string)
var v any
switch {
case strings.HasSuffix(tool, "_fetch"):
name := "go-fetch-clean"
if f.conflict.Load() {
name = "go-fetch-conflict"
}
fetched := readCaptured(t, name)
fetched["commit"], fetched["base"] = args["commit"], args["base"]
v = fetched
case strings.HasSuffix(tool, "_digest"):
d := map[string]string{"unit": digU, "integration": digI}[level]
v = map[string]any{"level": level, "packages": map[string]string{"fixture/slow": d}}
case strings.HasSuffix(tool, "_test"):
r := readCaptured(t, "go-test-slow")
r["level"] = level
v = r
}
data, _ := json.Marshal(v)
return data
}
func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) {
steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1})
if err != nil {
t.Fatal(err)
}
sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) {
tokens := strings.Split(m.Subject, ".") // mesh mod <module> call <tool> <node> <caller>
tool, node := tokens[4], tokens[5]
f.mu.Lock()
f.calls[module+"."+tool]++
f.mu.Unlock()
var args map[string]any
_ = json.Unmarshal(m.Data, &args)
id := fmt.Sprintf("s%d", f.n.Add(1))
key, _ := delivery.StepKey(node, id)
put := func(s delivery.Step) {
data, err := delivery.EncodeStep(s)
if err == nil {
_, err = steps.Put(f.ctx, key, data)
}
if err != nil {
t.Errorf("writing step %s: %v", key, err)
}
}
step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()}
put(step)
answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node})
_ = m.Respond(answer)
go func() {
time.Sleep(20 * time.Millisecond)
step.Ended = time.Now().UTC()
step.State, step.Result = delivery.StepPassed, f.result(t, tool, args)
put(step)
}()
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
// serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured):
// the go.mod of the captured file, whose mesh-sdk pin is cb54756c3865, and gitea_compare for the pin: on the trunk, unless the
// pin is the one the test set as stale.
func (b *bed) serveForge(t *testing.T, nc *nats.Conn) {
reply := func(m *nats.Msg, name string) {
var result map[string]any
_ = json.Unmarshal(mustRead(t, name), &result)
answer, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": "n0"})
_ = m.Respond(answer)
}
for _, tool := range []string{"gitea_get_file", "gitea_compare"} {
tool := tool
sub, err := nc.Subscribe("mesh.mod.gitea.tool."+tool, func(m *nats.Msg) {
var a map[string]string
_ = json.Unmarshal(m.Data, &a)
switch {
case tool == "gitea_get_file" && b.stalePin.Load():
content := strings.Replace(string(mustRead(t, "forge-get-file-go-mod")), "cb54756c3865", "9aaa5c80ac5c", 1)
var result map[string]any
_ = json.Unmarshal([]byte(content), &result)
answer, _ := json.Marshal(map[string]any{"result": result, "error": "", "node": "n0"})
_ = m.Respond(answer)
case tool == "gitea_get_file":
reply(m, "forge-get-file-go-mod")
case strings.HasPrefix(a["head"], "9aaa5c80ac5c"):
reply(m, "forge-compare-branch")
default:
reply(m, "forge-compare-trunk")
}
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
}
}
func mustRead(t *testing.T, name string) []byte {
t.Helper()
raw, err := os.ReadFile(filepath.Join(captured, name+".json"))
if err != nil {
t.Fatal(err)
}
return raw
}
type bed struct {
stalePin atomic.Bool
ctx context.Context
js jetstream.JetStream
buckets map[string]jetstream.KeyValue
tools map[string]jetstream.KeyValue
fake *fakeBuildModules
srvURL string
announce []string
mu sync.Mutex
}
// startBed starts the bus, the buckets the controller makes from the checker's and the other parts' manifests, the build
// modules' streams and the checker's durable consumers on them, one node's toolchains, the gate's source naming the trunk, and
// the fake build modules. It runs no checker.
func startBed(t *testing.T) *bed {
dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "c")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
conf := filepath.Join(dir, "bus.conf")
if err := os.WriteFile(conf, []byte(fmt.Sprintf(busConf, filepath.Join(dir, "js"))), 0o600); err != nil {
t.Fatal(err)
}
opts, err := server.ProcessConfigFile(conf)
if err != nil {
t.Fatal(err)
}
opts.NoLog, opts.NoSigs = true, true
srv, err := server.NewServer(opts)
if err != nil {
t.Fatal(err)
}
go srv.Start()
if !srv.ReadyForConnections(10 * time.Second) {
t.Fatal("the bus did not come up")
}
t.Cleanup(srv.Shutdown)
ctl, err := nats.Connect(srv.ClientURL(), nats.UserInfo("ctl", "pw"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(ctl.Close)
js, err := jetstream.New(ctl)
if err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
t.Cleanup(cancel)
b := &bed{ctx: ctx, js: js, buckets: map[string]jetstream.KeyValue{}, tools: map[string]jetstream.KeyValue{}, srvURL: srv.ClientURL()}
for module, stream := range toolchainStreams {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: stream, Subjects: []string{"mesh.stream." + module + ".event.toolchain.reported"}}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, stream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-checker", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ToolchainsBucket(module), History: 1})
if err != nil {
t.Fatal(err)
}
b.tools[module] = kv
}
// The checker's stream of events, which the indexer and the forge bridge hear.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
t.Fatal(err)
}
for name, history := range map[string]int{delivery.CheckerVerdictsBucket: delivery.CheckerVerdictsHistory, delivery.CheckerKeptBucket: delivery.CheckerKeptHistory,
delivery.CheckerOwedBucket: delivery.CheckerOwedHistory, delivery.PreviewsBucket: delivery.PreviewsHistory, delivery.CandidatesBucket: delivery.CandidatesHistory,
delivery.ModulesBucket: delivery.ModulesHistory, delivery.ManifestsBucket: delivery.ManifestsHistory, delivery.SourceBucket: delivery.SourceHistory,
delivery.PresenceBucket: delivery.PresenceHistory, delivery.HaveBucket: delivery.HaveHistory} {
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: uint8(history)})
if err != nil {
t.Fatal(err)
}
b.buckets[name] = kv
}
b.setOnline(t, "n1", true)
for _, module := range []string{"mesh-build-go", "mesh-build-node"} {
b.setHealth(t, "n1", module)
}
b.serveForge(t, ctl)
b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}}
b.fake.serve(t, ctl, "mesh-build-go")
b.fake.serve(t, ctl, "mesh-build-node")
b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1")
b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1")
b.setTrunk(t, map[string]string{repo: trunk})
return b
}
func (b *bed) setTrunk(t *testing.T, heads map[string]string) {
t.Helper()
data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads})
if err != nil {
t.Fatal(err)
}
if _, err := b.buckets[delivery.SourceBucket].Put(b.ctx, delivery.SourceMainKey, data); err != nil {
t.Fatal(err)
}
}
func (b *bed) setOnline(t *testing.T, node string, online bool) {
t.Helper()
data, _ := delivery.EncodePresence(delivery.Presence{Online: online})
if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, node, data); err != nil {
t.Fatal(err)
}
}
func (b *bed) setHealth(t *testing.T, node, module string) {
t.Helper()
data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.HaveKey(node, module)
if _, err := b.buckets[delivery.HaveBucket].Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
}
func (b *bed) report(t *testing.T, module, node, language, version, recipe string) {
t.Helper()
data, err := delivery.EncodeToolchain(delivery.Toolchain{Language: language, Version: version, Recipe: recipe, At: time.Now().UTC()})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.ToolchainKey(node, language)
if _, err := b.tools[module].Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
if _, err := b.js.Publish(b.ctx, "mesh.stream."+module+".event.toolchain.reported", []byte(`{}`)); err != nil {
t.Fatal(err)
}
}
// candidate writes candidates.<group> as the indexer does (the SDK's EncodeCandidate is the indexer's own encoder).
func (b *bed) candidate(t *testing.T, group string, pulls ...delivery.CandidatePull) string {
t.Helper()
heads := make([]delivery.Head, len(pulls))
for i, p := range pulls {
heads[i] = p.Head
}
hash, err := delivery.HeadsHash(heads)
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeCandidate(delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)})
if err != nil {
t.Fatal(err)
}
key, _ := delivery.CandidatesKey(group)
if _, err := b.buckets[delivery.CandidatesBucket].Put(b.ctx, key, data); err != nil {
t.Fatal(err)
}
vk, _ := delivery.CheckerVerdictKey(group, hash)
return vk
}
func pull(repository string, number int, head string) delivery.CandidatePull {
return delivery.CandidatePull{Head: delivery.Head{Repository: repository, Number: number, Head: head}, Modules: []string{}}
}
// start runs the program as the checker of a node.
func (b *bed) start(t *testing.T, env map[string]string) {
t.Helper()
broker := filepath.Join(t.TempDir(), "broker")
cred, _ := json.Marshal(credential{URL: b.srvURL, Node: "n1", Module: "mesh-delivery-checker", User: "n1.mesh-delivery-checker", Password: "pw"})
if err := os.WriteFile(broker, cred, 0o600); err != nil {
t.Fatal(err)
}
e := map[string]string{"MESH_BROKER_FILE": broker}
for k, v := range env {
e[k] = v
}
runCtx, stop := context.WithCancel(b.ctx)
done := make(chan error, 1)
go func() { done <- run(runCtx, func(k string) string { return e[k] }, io.Discard) }()
t.Cleanup(func() { stop(); <-done })
}
// verdict reads the verdict key until pred holds of it, within 20 s.
func (b *bed) verdict(t *testing.T, key string, pred func(delivery.Verdict) bool) delivery.Verdict {
t.Helper()
var last delivery.Verdict
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key)
if err != nil {
continue
}
if last, err = delivery.DecodeVerdict(e.Value()); err == nil && pred(last) {
return last
}
}
t.Fatalf("verdicts.%s = %+v within 20 s", key, last)
return last
}
func ended(v delivery.Verdict) bool { return v.State != delivery.VerdictRunning }
// owed reads the status the checker owes on a head.
func (b *bed) owed(t *testing.T, number int, head, state string) delivery.Owed {
t.Helper()
key, err := delivery.CheckerOwedKey(repo, number, head)
if err != nil {
t.Fatal(err)
}
var last delivery.Owed
for deadline := time.Now().Add(20 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key)
if err != nil {
continue
}
if last, err = delivery.DecodeCheckerOwed(e.Value()); err == nil && last.State == state {
return last
}
}
t.Fatalf("owed.%s = %+v within 20 s, want the state %s", key, last, state)
return last
}
// TestCheckerJudgesACandidateAndOwesItsStatus: a candidate the indexer wrote, one pull request of mesh-delivery, is fetched
// merged onto the trunk the gate names, digested and tested at both levels on the node, kept, previewed; the verdict passes
// with the counts of both levels and the composition not run, and the status mesh/check is owed on the head as success.
func TestCheckerJudgesACandidateAndOwesItsStatus(t *testing.T) { // N6/AC5, 1300/AC5, 1445/AC1
b := startBed(t)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
v := b.verdict(t, key, ended)
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 || v.Trunk[repo] != trunk {
t.Fatalf("verdict = %+v", v)
}
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || len(v.Machines) != 1 || v.Machines[0] != "n1" {
t.Errorf("composition %+v machines %v", v.Composition, v.Machines)
}
o := b.owed(t, 7, head1, "success")
if o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Text, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(o.Text, "unit 2 run") {
t.Errorf("owed = %+v", o)
}
for tool, want := range map[string]int{"mesh-build-go.build_go_fetch": 1, "mesh-build-go.build_go_digest": 2, "mesh-build-go.build_go_test": 2} {
if got := b.fake.count(tool); got != want {
t.Errorf("%s called %d times, want %d", tool, got, want)
}
}
// What passed is kept once under each level's digest.
for _, k := range []string{"unit." + digU, "integration." + digI} {
if _, err := b.buckets[delivery.CheckerKeptBucket].Get(b.ctx, k); err != nil {
t.Errorf("kept.%s: %v", k, err)
}
}
// The preview of the head merged onto the trunk, with the tree and the manifest digest the fetch named.
pk, _ := delivery.PreviewKey(repo, head1, trunk)
e, err := b.buckets[delivery.PreviewsBucket].Get(b.ctx, pk)
if err != nil {
t.Fatalf("previews.%s: %v", pk, err)
}
pv, err := delivery.DecodePreview(e.Value())
if err != nil || pv.Merge != merged || pv.Modules["modules/m"].Tree != "1111111111111111111111111111111111111111" {
t.Errorf("preview = %+v, %v", pv, err)
}
}
// TestACheckOfEqualInputsRunsNothing: a second candidate (another head, the same digests) is checked with every package
// taken from the first's kept verdicts: no test step runs and no second kept key appears.
func TestACheckOfEqualInputsRunsNothing(t *testing.T) { // 1287/AC5, 1288/AC6
b := startBed(t)
first := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
b.verdict(t, first, ended)
tests, kept := b.fake.count("mesh-build-go.build_go_test"), keptCount(t, b)
second := b.candidate(t, "feat/y", pull(repo, 8, head2))
v := b.verdict(t, second, ended)
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 0 || v.Levels[delivery.LevelUnit].Kept != 2 || v.Levels[delivery.LevelIntegration].Run != 0 {
t.Fatalf("verdict = %+v, want everything taken from the kept verdicts", v)
}
if got := b.fake.count("mesh-build-go.build_go_test"); got != tests {
t.Errorf("build_go_test called %d times, was %d: a test ran for equal inputs", got, tests)
}
if got := keptCount(t, b); got != kept {
t.Errorf("%d kept keys, was %d", got, kept)
}
}
func keptCount(t *testing.T, b *bed) int {
t.Helper()
keys, err := b.buckets[delivery.CheckerKeptBucket].Keys(b.ctx)
if err != nil {
return 0
}
return len(keys)
}
// TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest: the fetch answers the files in conflict; the verdict fails naming
// them, no test or digest step is run, a preview of the conflicts alone is written and the status is owed as failure.
func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4
b := startBed(t)
b.fake.conflict.Store(true)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
v := b.verdict(t, key, ended)
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "files in conflict: a.go, sub/b.go") {
t.Fatalf("verdict = %+v", v)
}
if o := b.owed(t, 7, head1, "failure"); !strings.HasPrefix(o.Text, "failed: n1: ") || !strings.Contains(o.Text, "files in conflict: a.go, sub/b.go") {
t.Errorf("owed = %+v", o)
}
if n := b.fake.count("mesh-build-go.build_go_digest") + b.fake.count("mesh-build-go.build_go_test"); n != 0 {
t.Errorf("%d digest and test steps ran after a conflict", n)
}
pk, _ := delivery.PreviewKey(repo, head1, trunk)
if e, err := b.buckets[delivery.PreviewsBucket].Get(b.ctx, pk); err != nil {
t.Errorf("previews.%s: %v", pk, err)
} else if pv, err := delivery.DecodePreview(e.Value()); err != nil || strings.Join(pv.Conflicts, ",") != "a.go,sub/b.go" || pv.Merge != "" {
t.Errorf("preview = %+v, %v, want the files in conflict alone", pv, err)
}
}
// TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in
// the same run is.
func TestAnHqPullRequestGetsNoVerdictAndAnotherDoes(t *testing.T) { // ADR 0484 clause 8
b := startBed(t)
hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1))
other := b.candidate(t, "feat/x", pull(repo, 7, head2))
b.start(t, nil)
b.verdict(t, other, ended)
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, hq); err == nil {
t.Error("hq's pull request has a verdict")
}
}
// TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts: the verdict key holds running, left by a checker that stopped; the
// started checker judges it again and ends it (ADR 0478 clause 6, as the builder's).
func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/AC2
b := startBed(t)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
group, heads, _ := delivery.ParseCheckerVerdictKey(key)
left, err := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, Trunk: map[string]string{repo: trunk}, State: delivery.VerdictRunning})
if err != nil {
t.Fatal(err)
}
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Create(b.ctx, key, left); err != nil {
t.Fatal(err)
}
b.start(t, nil)
if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
t.Errorf("verdict = %+v", v)
}
}
// TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds: the controller's bucket of the plans in their build phase
// holds a plan's key; the candidate waits, no step is called; the key deleted, the check starts and passes (ADR 0484 clause 12).
func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1
b := startBed(t)
plans, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_plans-building", History: 1})
if err != nil {
t.Fatal(err)
}
if _, err := plans.Put(b.ctx, "plan-1", []byte(`{"plan":"plan-1","since":"2026-10-21T19:00:00Z","seen":1}`)); err != nil {
t.Fatal(err)
}
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
time.Sleep(1500 * time.Millisecond) // a wait for nothing to happen: there is no event to wait on
if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key); err == nil {
t.Fatal("a check started while a delivery builds")
}
if n := b.fake.count("mesh-build-go.build_go_fetch"); n != 0 {
t.Fatalf("%d fetches while a delivery builds", n)
}
if err := plans.Delete(b.ctx, "plan-1"); err != nil {
t.Fatal(err)
}
if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass {
t.Errorf("verdict = %+v", v)
}
}
// TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not
// hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9).
func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2
b := startBed(t)
b.stalePin.Store(true)
key := b.candidate(t, "feat/x", pull(repo, 7, head1))
b.start(t, nil)
v := b.verdict(t, key, ended)
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "9aaa5c80ac5c") || !strings.Contains(v.Failed.Words, repo+"#7") {
t.Fatalf("verdict = %+v", v)
}
if n := b.fake.count("mesh-build-go.build_go_test"); n != 0 {
t.Errorf("%d test steps ran after a stale pin", n)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go
+140
View File
@@ -0,0 +1,140 @@
package main
import (
"context"
"sort"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"mesh-delivery-checker/internal/check"
)
// judgeDeps are the parts the judge of a check is made of.
type judgeDeps struct {
runner buildrun.Runner
tools *buildrun.ToolchainSet
presence *buildrun.PresenceSet
have *buildrun.HaveSet
places *buildrun.Places
trunks *check.Trunks
kept check.KV
previews check.KV
repos *repoModules
forge check.Forge
announce check.Announcer
builds check.BuildStates
logf func(format string, args ...any)
}
func newJudge(d judgeDeps) check.Judge {
return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places,
Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf}
}
// manifestKV reads a manifest by its digest.
type manifestKV interface {
Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error)
}
// repoModules knows which build modules judge a repository, from the indexer's modules and their manifests: the Go build
// module always (a repository's go.mod files are its own), the Node one when a module of the repository has a TypeScript
// artifact (ADR 0484 clause 10). Safe for use from the watch and from a check at once.
type repoModules struct {
manifests manifestKV
logf func(format string, args ...any)
mu sync.Mutex
langs map[string]map[string]bool // by manifest digest: the languages of its artifacts
repoOf map[string]string // by module: its repository
commit map[string]string // by module: the commit its record was computed at (the commit the mesh runs)
typed map[string]string // by module: its manifest digest
}
// Put takes a change of the indexer's modules bucket in: a module's repository and the languages its manifest names.
func (r *repoModules) Put(ctx context.Context, key string, value []byte, deleted bool) {
r.mu.Lock()
if r.repoOf == nil {
r.repoOf, r.typed, r.langs, r.commit = map[string]string{}, map[string]string{}, map[string]map[string]bool{}, map[string]string{}
}
if deleted {
delete(r.repoOf, key)
delete(r.typed, key)
delete(r.commit, key)
r.mu.Unlock()
return
}
r.mu.Unlock()
m, err := delivery.DecodeModule(value)
if err != nil {
return
}
r.mu.Lock()
r.repoOf[key] = m.Repository
r.typed[key] = m.Manifest
r.commit[key] = m.Commit
_, known := r.langs[m.Manifest]
r.mu.Unlock()
if m.Manifest == "" || known {
return
}
set := map[string]bool{}
if mk, err := delivery.ManifestsKey(m.Manifest); err == nil {
if raw, _, found, err := r.manifests.Get(ctx, mk); err == nil && found {
if kept, err := delivery.DecodeManifest(raw); err == nil {
if man, err := catalogue.ParseManifest([]byte(kept.JSON)); err == nil && man.Build != nil {
for _, a := range man.Build.Artifacts {
if a.Language != "" {
set[a.Language] = true
}
}
} else if err != nil && r.logf != nil {
r.logf("the manifest %s cannot be read, so its module counts as no TypeScript: %v", m.Manifest, err)
}
}
}
}
r.mu.Lock()
r.langs[m.Manifest] = set
r.mu.Unlock()
}
// BuildModules are the build modules that judge a repository, sorted.
func (r *repoModules) BuildModules(repository string) []string {
out := []string{buildrun.GoBuildModule}
r.mu.Lock()
defer r.mu.Unlock()
for module, repo := range r.repoOf {
if repo == repository && r.langs[r.typed[module]]["typescript"] {
out = append(out, buildrun.NodeBuildModule)
break
}
}
sort.Strings(out)
return out
}
// buildReader reads the builder's build of a module at a fingerprint (mesh-delivery-builder_builds).
type buildReader struct{ kv kvStore }
func (r buildReader) Build(ctx context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error) {
key, err := delivery.BuilderBuildKey(module, fp2)
if err != nil {
return delivery.BuilderBuild{}, false, err
}
value, _, found, err := r.kv.Get(ctx, key)
if err != nil || !found {
return delivery.BuilderBuild{}, false, err
}
b, err := delivery.DecodeBuilderBuild(value)
return b, err == nil, err
}
// CommitOf is the commit the indexer's record of the module was computed at: the commit of its repository the mesh runs.
func (r *repoModules) CommitOf(module string) string {
r.mu.Lock()
defer r.mu.Unlock()
return r.commit[module]
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go
+75
View File
@@ -0,0 +1,75 @@
package main
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
type memManifests map[string][]byte
func (m memManifests) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
v, ok := m[key]
return v, 1, ok, nil
}
func putModule(t *testing.T, r *repoModules, name, repository, manifest string) {
t.Helper()
data, err := delivery.EncodeModule(delivery.Module{Repository: repository, Commit: strings.Repeat("a", 40), FP: "src1:" + strings.Repeat("a", 64), Manifest: delivery.ManifestDigest([]byte(manifest))})
if err != nil {
t.Fatal(err)
}
r.Put(context.Background(), name, data, false)
}
func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *testing.T) { // 1292/AC2
own, err := os.ReadFile("../../module.json") // a manifest with a Go bundle: this module's own
if err != nil {
t.Fatal(err)
}
goManifest := string(own)
// A TypeScript bundle names neither a system nor a directory it is built from (the catalogue's parser says so).
var doc map[string]any
if err := json.Unmarshal(own, &doc); err != nil {
t.Fatal(err)
}
artifacts := doc["build"].(map[string]any)["artifacts"].([]any)
program := artifacts[0].(map[string]any)
program["language"] = "typescript"
delete(program, "system")
delete(program, "from")
delete(program, "binary")
tsRaw, err := json.Marshal(doc)
if err != nil {
t.Fatal(err)
}
tsManifest := string(tsRaw)
store := memManifests{}
for _, m := range []string{goManifest, tsManifest} {
data, err := delivery.EncodeManifest(delivery.Manifest{Module: "demo", JSON: m})
if err != nil {
t.Fatal(err)
}
store[delivery.ManifestDigest([]byte(m))] = data
}
r := &repoModules{manifests: store}
putModule(t, r, "mesh-delivery-checker", "novox/mesh-delivery", goManifest)
putModule(t, r, "mesh-ui", "novox/mesh-catalog", tsManifest)
if got := r.BuildModules("novox/mesh-delivery"); strings.Join(got, ",") != buildrun.GoBuildModule {
t.Errorf("mesh-delivery: %v, want the Go build module alone", got)
}
if got := r.BuildModules("novox/mesh-catalog"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule {
t.Errorf("mesh-catalog: %v, want both", got)
}
if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" {
t.Errorf("the commit the mesh runs = %q", got)
}
if got := r.BuildModules("novox/mesh-sdk"); strings.Join(got, ",") != buildrun.GoBuildModule {
t.Errorf("a repository with no module: %v, want the Go build module", got)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go
+352
View File
@@ -0,0 +1,352 @@
// Command mesh-delivery-checker is delivery's checker (novox/hq ADR 0453 clause 4, ADR 0447, ADR 0484, issues 1678 to 1686): for
// each candidate the indexer writes it judges every pull request's head merged onto its trunk, through the build modules'
// fetch, digest and test steps, keeps the passing packages' verdicts, writes the verdict, and owes the forge the status
// `mesh/check`, which the forge bridge sets. It runs dark: nothing reads its answers to merge or deliver (ADR 0453 clause 4).
// It serves no request subject and holds no timer: every reaction is a watch or a bus event (ADR 0437 §8).
package main
import (
"context"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
"git.novox.be/novox/mesh-sdk/go/process"
"mesh-delivery-checker/internal/check"
)
// eventSubject is the subject of an event of the checker's stream checks (the bus names the stream CHECKS): the indexer's
// watch of the previews bucket and the forge bridge's watch of the owed bucket are woken by them.
func eventSubject(event string) string {
return "mesh.stream." + delivery.CheckerModule + ".event." + event
}
const (
// defaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a
// test level runs a repository's tests).
defaultStepBound = 30 * time.Minute
// defaultLink is the base of the address a status points at; the verdict key follows.
defaultLink = "mesh-delivery-view/checks"
)
func main() {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
if err := run(ctx, os.Getenv, os.Stderr); err != nil {
fmt.Fprintln(os.Stderr, "mesh-delivery-checker:", err)
os.Exit(1)
}
}
// credential is the broker file the controller issues to the module: JSON of the bus address, the certificate
// fingerprint to pin, the principal and its password (as the indexer reads it; issue 1242 moves this into the SDK).
type credential struct {
URL string `json:"url"`
Fingerprint string `json:"fingerprint"`
Node string `json:"node"`
Module string `json:"module"`
User string `json:"user"`
Password string `json:"password"`
}
func readCredential(path string) (credential, error) {
raw, err := os.ReadFile(path)
if err != nil {
return credential{}, err
}
var c credential
if err := json.Unmarshal(raw, &c); err != nil {
return credential{}, fmt.Errorf("the broker credential is not one: %w", err)
}
if c.URL == "" || c.User == "" {
return credential{}, errors.New("the broker credential names no bus address or user")
}
return c, nil
}
// pinned accepts exactly the certificate with this SHA-256 and no other.
func pinned(want string) *tls.Config {
return &tls.Config{
InsecureSkipVerify: true, //nolint:gosec // replaced by the pin below, which is stricter
MinVersion: tls.VersionTLS12,
VerifyPeerCertificate: func(rawCerts [][]byte, _ [][]*x509.Certificate) error {
if len(rawCerts) == 0 {
return errors.New("the bus presented no certificate")
}
sum := sha256.Sum256(rawCerts[0])
if got := "sha256:" + hex.EncodeToString(sum[:]); got != want {
return fmt.Errorf("the bus presented a certificate this mesh does not know (%.23s…)", got)
}
return nil
},
}
}
// durationEnv reads a duration setting from the environment, def when it is not set.
func durationEnv(getenv func(string) string, name string, def time.Duration) (time.Duration, error) {
v := strings.TrimSpace(getenv(name))
if v == "" {
return def, nil
}
d, err := time.ParseDuration(v)
if err != nil || d <= 0 {
return 0, fmt.Errorf("%s: %q is not a duration of more than 0", name, v)
}
return d, nil
}
// run is the program: it returns when ctx ends or the bus cannot be used.
func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error {
logger := log.New(logOut, "mesh-delivery-checker: ", log.LstdFlags)
path := strings.TrimSpace(getenv("MESH_BROKER_FILE"))
if path == "" {
return errors.New("MESH_BROKER_FILE is not set, so there is no credential to hear the bus with")
}
cred, err := readCredential(path)
if err != nil {
return err
}
opts := []nats.Option{
nats.Name(cred.Node + "." + cred.Module),
nats.UserInfo(cred.User, cred.Password),
nats.CustomInboxPrefix("_INBOX." + cred.User),
nats.MaxReconnects(-1),
nats.RetryOnFailedConnect(true),
}
if strings.TrimSpace(cred.Fingerprint) != "" {
opts = append(opts, nats.Secure(pinned(cred.Fingerprint)))
}
conn, err := nats.Connect(cred.URL, opts...)
if err != nil {
return fmt.Errorf("connecting to the bus: %w", err)
}
defer conn.Close()
js, err := jetstream.New(conn)
if err != nil {
return err
}
bucket := func(name string) (jetstream.KeyValue, error) {
kv, err := js.KeyValue(ctx, name)
if err != nil {
return nil, fmt.Errorf("the bucket %s: %w", name, err)
}
return kv, nil
}
buckets := map[string]jetstream.KeyValue{}
for _, name := range []string{delivery.CheckerVerdictsBucket, delivery.CheckerKeptBucket, delivery.CheckerOwedBucket, delivery.PreviewsBucket,
delivery.CandidatesBucket, delivery.ModulesBucket, delivery.ManifestsBucket} {
if buckets[name], err = bucket(name); err != nil {
return err
}
}
// Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the
// buckets are read and written on the connection above.
account, err := process.ReadAccount(path)
if err != nil {
return err
}
caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true))
if err != nil {
return err
}
defer caller.Close()
stepBound, err := durationEnv(getenv, "MESH_CHECKER_STEP_BOUND", defaultStepBound)
if err != nil {
return err
}
checkBound, err := durationEnv(getenv, "MESH_CHECKER_BOUND", check.DefaultBound)
if err != nil {
return err
}
capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_CHECKER_SETTINGS")), getenv("MESH_CHECKER_CAPACITY"))
if err != nil {
return fmt.Errorf("the capacity setting: %w", err)
}
link := strings.TrimSpace(getenv("MESH_CHECKER_LINK"))
if link == "" {
link = defaultLink
}
tools := &buildrun.ToolchainSet{}
presence := &buildrun.PresenceSet{}
have := &buildrun.HaveSet{}
trunks := &check.Trunks{}
places := &buildrun.Places{Capacity: capacity}
runner := buildrun.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: stepBound}
ch := &check.Checker{
Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]},
Owed: kvStore{buckets[delivery.CheckerOwedBucket]},
Trunks: trunks,
Bound: checkBound,
Link: link,
Logf: logger.Printf,
}
// The builder's builds say how the candidates' fingerprints stand (issue 1684); where its bucket is not there yet the line
// says nothing of the builds, until the checker starts again.
var builds check.BuildStates
switch kv, err := js.KeyValue(ctx, delivery.BuilderBuildsBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: the line says nothing of the builds until the checker starts again", delivery.BuilderBuildsBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.BuilderBuildsBucket, err)
default:
builds = buildReader{kvStore{kv}}
}
holds := newHoldWatch(ctx, js, ch.Reconsider, logger)
ch.Hold = holds.Held
announce := func(ctx context.Context, event, key string) {
body, _ := json.Marshal(map[string]string{"key": key})
if _, err := js.Publish(ctx, eventSubject(event), body); err != nil {
logger.Printf("%s of %s was written and the part that reads it could not be told: %v", event, key, err)
}
}
repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf}
ch.Judge = newJudge(judgeDeps{
runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks,
kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]},
repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf,
announce: announce,
})
ch.Announce = announce
// The toolchains first, so a check that starts from the first candidate finds the nodes.
watch := newToolchainWatch(ctx, js, tools, ch.NodesChanged, logger)
if err := watch.start(); err != nil {
return err
}
// A build module says each time it wrote its toolchains key: its stream, and the checker's durable consumer on it, are
// made by the controller. Where the stream is not there yet the checker says so and is started again with the build
// module.
for _, module := range []string{buildrun.GoBuildModule, buildrun.NodeBuildModule} {
stream := toolchainStreams[module]
cons, err := js.Consumer(ctx, stream, cred.Node+"_"+cred.Module)
if err != nil {
logger.Printf("the consumer %s_%s on %s does not exist yet, so %s's reports are not heard until the checker starts again: %v", cred.Node, cred.Module, stream, module, err)
continue
}
cc, err := cons.Consume(func(m jetstream.Msg) { watch.Reported(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
defer cc.Stop()
}
errs := make(chan error, 8)
// The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing
// is checked; the checker says so and is started again with the watchman.
switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err)
default:
pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) {
if presence.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx) // a node came online or went: a check that waited for one may go on
}
})
}()
}
// The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not
// chosen. Where its bucket is not there yet no build module counts as healthy, so nothing is checked.
switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the checker starts again", delivery.HaveBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err)
default:
hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) {
if have.Put(e.Key(), e.Value(), !opOf(e)) {
ch.NodesChanged(ctx)
}
})
}()
}
// The gate's source says the trunk head of each repository, which a pull request is merged onto. Where it is not there
// yet no trunk is known, so nothing is checked.
switch kv, err := js.KeyValue(ctx, delivery.SourceBucket); {
case errors.Is(err, jetstream.ErrBucketNotFound):
logger.Printf("the bucket %s does not exist yet: no trunk is known until the checker starts again", delivery.SourceBucket)
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.SourceBucket, err)
default:
sw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey {
trunks.Put(e.Value(), !opOf(e))
}
})
if err != nil {
return err
}
go func() {
errs <- follow(ctx, sw, func(e jetstream.KeyValueEntry) {
if e.Key() == delivery.SourceMainKey && trunks.Put(e.Value(), !opOf(e)) {
ch.Reconsider(ctx) // the trunk moved: a verdict for the old trunk is judged again
}
})
}()
}
// The modules say which repository holds a TypeScript module, so the Node build module judges it too.
mw, err := settle(ctx, buckets[delivery.ModulesBucket], func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
go func() {
errs <- follow(ctx, mw, func(e jetstream.KeyValueEntry) { repos.Put(ctx, e.Key(), e.Value(), !opOf(e)) })
}()
each := func(e jetstream.KeyValueEntry) {
if opOf(e) {
ch.Candidate(ctx, e.Key(), e.Value())
} else {
ch.Forget(e.Key())
}
}
w, err := settle(ctx, buckets[delivery.CandidatesBucket], each)
if err != nil {
return err
}
// A key left running for heads no candidate holds is closed; the candidates are all known now.
if err := ch.Sweep(ctx); err != nil {
return err
}
logger.Printf("started on %s", cred.Node)
go func() { errs <- follow(ctx, w, each) }()
select {
case <-ctx.Done():
ch.Wait()
return nil
case err := <-errs:
return err
case err := <-watch.errs:
return err
case err := <-holds.errs:
return err
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go
+29
View File
@@ -0,0 +1,29 @@
package main
import (
"encoding/json"
"fmt"
"os"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `<node>=<steps at
// once>` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a
// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch.
func capacityFrom(path, fallback string) (map[string]int, error) {
if path == "" {
return buildrun.ParseCapacity(fallback)
}
raw, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("the settings file: %w", err)
}
var s struct {
Capacity string `json:"capacity"`
}
if err := json.Unmarshal(raw, &s); err != nil {
return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err)
}
return buildrun.ParseCapacity(s.Capacity)
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go
+68
View File
@@ -0,0 +1,68 @@
package main
import (
"os"
"path/filepath"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
func settingsFile(t *testing.T, body string) string {
t.Helper()
p := filepath.Join(t.TempDir(), "settings.json")
if err := os.WriteFile(p, []byte(body), 0o600); err != nil {
t.Fatal(err)
}
return p
}
// ADR 0484 clause 12: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere.
func TestCapacityIsReadFromTheSettingsFile(t *testing.T) {
for body, want := range map[string]map[string]int{
`{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2},
`{"capacity": ""}`: {},
`{}`: {},
} {
got, err := capacityFrom(settingsFile(t, body), "")
if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] {
t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want)
}
}
p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))}
if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 {
t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace"))
}
}
func mustCapacity(t *testing.T, path string) map[string]int {
t.Helper()
c, err := capacityFrom(path, "")
if err != nil {
t.Fatal(err)
}
return c
}
func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) {
if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil {
t.Error("shanks=-1 accepted")
}
if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 {
t.Errorf("the starting value: %v, %v", c, err)
}
if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil {
t.Error("a file that is not JSON accepted")
}
if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil {
t.Error("a settings file that is not there accepted")
}
}
// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's.
func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) {
got, err := capacityFrom("", "ace=2")
if err != nil || got["ace"] != 2 {
t.Errorf("capacity %v, %v", got, err)
}
}
modules/mesh-delivery-checker/cmd/mesh-delivery-checker/toolchains.go
+104
View File
@@ -0,0 +1,104 @@
package main
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// toolchainStreams are the stream each build module carries its toolchain.reported event on, as the bus names it
// (declared by the build modules, heard by this module through consumes).
var toolchainStreams = map[string]string{buildrun.GoBuildModule: "TOOLCHAINS_GO", buildrun.NodeBuildModule: "TOOLCHAINS_NODE"}
// toolchainWatch keeps the build modules' toolchains in a buildrun.ToolchainSet and tells the builder when one changed. A
// build module's bucket is made when it is assigned, after the builder started, and the builder holds no timer (ADR 0437
// §8): it watches the buckets that exist at start, and a build module says each time it wrote its toolchains key
// (toolchain.reported, handled by Reported); a bucket not watched yet is then watched.
type toolchainWatch struct {
ctx context.Context
js bucketOpener
set *buildrun.ToolchainSet
changed func(ctx context.Context) // a language's entries changed: the failed builds are tried again
logger *log.Logger
errs chan error
mu sync.Mutex
watched map[string]bool
}
func newToolchainWatch(ctx context.Context, js bucketOpener, set *buildrun.ToolchainSet, changed func(context.Context), logger *log.Logger) *toolchainWatch {
return &toolchainWatch{ctx: ctx, js: js, set: set, changed: changed, logger: logger, errs: make(chan error, len(buildrun.LanguageOf)), watched: map[string]bool{}}
}
// start watches the buckets that exist now; a missing one is said and waits for its build module's event.
func (w *toolchainWatch) start() error {
for module := range buildrun.LanguageOf {
if err := w.ensure(module); err != nil {
return err
}
}
return nil
}
func (w *toolchainWatch) ensure(module string) error {
w.mu.Lock()
defer w.mu.Unlock()
if w.watched[module] {
return nil
}
name := delivery.ToolchainsBucket(module)
kv, err := w.js.KeyValue(w.ctx, name)
switch {
case errors.Is(err, jetstream.ErrBucketNotFound):
w.logger.Printf("the bucket %s does not exist yet: it is watched when %s reports a toolchain", name, module)
return nil
case err != nil:
return fmt.Errorf("the bucket %s: %w", name, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.set.Put(module, e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
w.watched[module] = true
go func() {
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
if w.set.Put(module, e.Key(), e.Value(), !opOf(e)) {
w.changed(w.ctx)
}
})
}()
w.changed(w.ctx) // what the settle read is new to the builder
return nil
}
// delivered is the part of a JetStream message the watch uses.
type delivered interface {
Subject() string
Ack() error
Term() error
}
// Reported takes one toolchain.reported event of a build module in: the module is read off the subject,
// mesh.stream.<module>.event.toolchain.reported, and its bucket is watched if it is not. A subject of no build module is
// terminated; a failure stays unacknowledged, and the server delivers it again.
func (w *toolchainWatch) Reported(m delivered) {
module, _, _ := strings.Cut(strings.TrimPrefix(m.Subject(), "mesh.stream."), ".event.")
if _, known := buildrun.LanguageOf[module]; !known || !strings.HasPrefix(m.Subject(), "mesh.stream.") {
w.logger.Printf("%s: not a build module's event, dropped", m.Subject())
_ = m.Term()
return
}
if err := w.ensure(module); err != nil {
w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err)
return
}
_ = m.Ack()
}
modules/mesh-delivery-checker/files/settings.json
+3
View File
@@ -0,0 +1,3 @@
{
"capacity": ""
}
modules/mesh-delivery-checker/go.mod
+22
View File
@@ -0,0 +1,22 @@
module mesh-delivery-checker
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/klauspost/compress v1.20.0 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.16 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
golang.org/x/crypto v0.57.0 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/time v0.15.0 // indirect
)
modules/mesh-delivery-checker/go.sum
+27
View File
@@ -0,0 +1,27 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I=
github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k=
github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg=
github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
modules/mesh-delivery-checker/internal/check/beside.go
+101
View File
@@ -0,0 +1,101 @@
package check
import (
"context"
"fmt"
"strconv"
)
// BesideRef is a sibling repository a repository's tests read, placed beside the fetched tree as a plain directory (ADR
// 0484 clause 4). Ref names the pull request's head when the commit is only there.
type BesideRef struct {
Repository string `json:"repository"`
Commit string `json:"commit"`
Ref string `json:"ref,omitempty"`
}
const (
catalogueRepository = "novox/mesh-catalog"
nodeEngineRepo = "novox/mesh-host"
nodeEngineModule = "mesh-host"
controllerRepo = "novox/mesh-controller"
)
// besideRules are the repositories each core repository's tests read, as the tests state it (mesh-sdk go/internal/beside
// beside.go): the catalogue, the node-engine, the SDK, and the delivery's pinned forge dependencies (mesh-catalog
// checks/deps). Which commit of each is besideCommit's rule.
var besideRules = map[string][]string{
controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository},
nodeEngineRepo: {catalogueRepository, SDKRepository},
SDKRepository: {catalogueRepository, nodeEngineRepo},
catalogueRepository: {SDKRepository, controllerRepo},
"novox/mesh-delivery": {SDKRepository, catalogueRepository},
}
// besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the catalogue at its trunk, always, even in a
// group holding a catalogue pull request; the node-engine at the commit the mesh runs (the commit of the indexer's module
// mesh-host, its trunk head when none is known); mesh-sdk, for the controller's tests, at the commit the controller's go.mod
// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk.
func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef, error) {
var heads map[string]string
if j.Trunks != nil {
heads = j.Trunks.Heads()
}
var out []BesideRef
for _, repo := range besideRules[p.Repository] {
switch {
case repo == catalogueRepository:
if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head})
}
case repo == nodeEngineRepo:
commit := ""
if j.Runs != nil {
commit = j.Runs.CommitOf(nodeEngineModule)
}
if commit == "" {
commit = heads[repo]
}
if commit != "" {
out = append(out, BesideRef{Repository: repo, Commit: commit})
}
case repo == SDKRepository && p.Repository == controllerRepo && j.Forge != nil:
text, err := j.Forge.File(ctx, p.Repository, "go.mod", p.Head)
if err != nil {
return nil, fmt.Errorf("go.mod of %s#%d cannot be read to place the mesh-sdk its tests read: %w", p.Repository, p.Number, err)
}
pin, ok := SDKPin(text)
if !ok {
return nil, fmt.Errorf("go.mod of %s#%d names no mesh-sdk commit, so the mesh-sdk its tests read cannot be placed", p.Repository, p.Number)
}
full, err := j.Forge.FullSHA(ctx, SDKRepository, pin)
if err != nil {
return nil, fmt.Errorf("mesh-sdk %s, which %s#%d pins, cannot be found: %w", pin, p.Repository, p.Number, err)
}
out = append(out, BesideRef{Repository: repo, Commit: full})
default:
if member := c.member(repo); member != nil {
out = append(out, BesideRef{Repository: repo, Commit: member.Head, Ref: refOf(member.Number)})
} else if head := heads[repo]; head != "" {
out = append(out, BesideRef{Repository: repo, Commit: head})
}
}
}
return out, nil
}
func (c *Case) member(repository string) *Pull {
for i := range c.Pulls {
if c.Pulls[i].Repository == repository {
return &c.Pulls[i]
}
}
return nil
}
func refOf(number int) string {
if number <= 0 {
return ""
}
return "refs/pull/" + strconv.Itoa(number) + "/head"
}
modules/mesh-delivery-checker/internal/check/builds.go
+66
View File
@@ -0,0 +1,66 @@
package check
import (
"context"
"sort"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The states of a build the status line says (ADR 0484 clause 7).
const (
BuildBuilt = "built"
BuildFailed = "failed"
BuildWaiting = "waiting"
)
// BuildStates reads the builder's build of a module at a fingerprint of the new form (builds.<module>.<fingerprint>): the
// value and whether the key exists.
type BuildStates interface {
Build(ctx context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error)
}
// NoteOf is the state of a build as the line says it: done is built, failed is failed, and one queued, building or not
// begun is waiting.
func NoteOf(b delivery.BuilderBuild, found bool) string {
switch {
case !found:
return BuildWaiting
case b.State == delivery.BuildDone:
return BuildBuilt
case b.State == delivery.BuildFailed:
return BuildFailed
}
return BuildWaiting
}
// buildNotes are the state of each build the case's modules carry a fingerprint of, by module, sorted. A module two pull
// requests give is said once.
func (j *Examiner) buildNotes(ctx context.Context, c *Case) []BuildNote {
if j.Builds == nil {
return nil
}
fps := map[string]string{}
for _, p := range c.Pulls {
for module, cm := range p.Fps {
if cm.FP2 != "" {
fps[module] = cm.FP2
}
}
}
modules := make([]string, 0, len(fps))
for m := range fps {
modules = append(modules, m)
}
sort.Strings(modules)
var notes []BuildNote
for _, m := range modules {
b, found, err := j.Builds.Build(ctx, m, fps[m])
if err != nil {
j.logf("the build of %s at %s cannot be read: %v", m, fps[m], err)
found = false
}
notes = append(notes, BuildNote{Module: m, State: NoteOf(b, found)})
}
return notes
}
modules/mesh-delivery-checker/internal/check/case.go
+135
View File
@@ -0,0 +1,135 @@
package check
import (
"fmt"
"maps"
"sort"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// HQRepository is the repository the dark checker writes no status for (ADR 0484 clause 8): hq builds no module and its
// check is a script no build module can run, so it is judged by its merge-check.sh until issue 1296 settles how.
const HQRepository = "novox/hq"
// HQNote is what the read tools say of an hq pull request (ADR 0484 clause 8).
const HQNote = "hq: judged by merge-check.sh until issue 1296"
// Pull is one pull request the checker judges: its head, the trunk head of its repository it is merged onto and the
// modules the indexer says it touches, with the fingerprints the indexer wrote for them.
type Pull struct {
Repository string
Number int
Head string
Trunk string
Modules []string
Fps map[string]delivery.CandidateModule
}
// Case is one check: a candidate's pull requests that are judged (those of hq and of a repository whose trunk is not known
// are left out), the trunk head of each repository and the key of the verdict.
type Case struct {
// Key is the verdict key, `<group>.<heads hash>`.
Key string
Group string
HeadsHash string
Pulls []Pull
// Trunk is the trunk head of each repository of Pulls.
Trunk map[string]string
// Left says why pull requests of the candidate are not judged (hq's, a repository with no known trunk); empty when all are.
Left []string
// Queued is when the checker took the case up, for the time of its queue.
Queued time.Time
// Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at.
Progress *Progress
}
// Progress is the step a check is at, written by the check and read when its bound passes.
type Progress struct {
mu sync.Mutex
stage, node string
}
// Set records the step now running and the node it runs on, empty for none.
func (p *Progress) Set(stage, node string) {
p.mu.Lock()
p.stage, p.node = stage, node
p.mu.Unlock()
}
// Now is the step recorded last and its node.
func (p *Progress) Now() (stage, node string) {
p.mu.Lock()
defer p.mu.Unlock()
return p.stage, p.node
}
// CaseOf is the check of a candidate against the trunk heads (the gate's source.main): the pull requests that are judged.
// The checker judges pull requests into main only (ADR 0484 clause 11): the pull message carries no base, a merge into
// another branch is a closed message, so every candidate is into the trunk; a repository whose trunk head is not known has
// none to merge onto. It answers nil when no pull request is left to judge.
func CaseOf(c delivery.Candidate, trunk map[string]string) *Case {
key, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return nil
}
cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Trunk: map[string]string{}, Progress: &Progress{}}
for _, p := range c.Pulls {
switch head, known := trunk[p.Repository]; {
case p.Repository == HQRepository:
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, HQNote))
case !known || head == "":
cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository))
default:
cs.Pulls = append(cs.Pulls, Pull{Repository: p.Repository, Number: p.Number, Head: p.Head.Head, Trunk: head, Modules: p.Modules, Fps: p.Fps})
cs.Trunk[p.Repository] = head
}
}
if len(cs.Pulls) == 0 {
return nil
}
sort.SliceStable(cs.Pulls, func(i, j int) bool {
if cs.Pulls[i].Repository != cs.Pulls[j].Repository {
return cs.Pulls[i].Repository < cs.Pulls[j].Repository
}
return cs.Pulls[i].Number < cs.Pulls[j].Number
})
return cs
}
// sameTrunk says whether two verdicts' trunk heads are the same.
func sameTrunk(a, b map[string]string) bool { return maps.Equal(a, b) }
// Trunks holds the trunk head of each repository, as the gate's source.main says (a watch puts it in).
type Trunks struct {
mu sync.Mutex
heads map[string]string
}
// Put takes a source.main value in and answers whether the heads changed.
func (t *Trunks) Put(value []byte, deleted bool) bool {
var heads map[string]string
if !deleted {
s, err := delivery.DecodeSource(value)
if err != nil {
return false
}
heads = s.Heads
}
t.mu.Lock()
defer t.mu.Unlock()
if maps.Equal(t.heads, heads) {
return false
}
t.heads = maps.Clone(heads)
return true
}
// Heads are the trunk heads now (a copy).
func (t *Trunks) Heads() map[string]string {
t.mu.Lock()
defer t.mu.Unlock()
return maps.Clone(t.heads)
}
modules/mesh-delivery-checker/internal/check/checker.go
+469
View File
@@ -0,0 +1,469 @@
package check
import (
"context"
"errors"
"fmt"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678).
const DefaultBound = 30 * time.Minute
// NodesWords is in the Words of a failure that is the build nodes' (none qualifies, the toolchains disagree): such a verdict
// is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8).
const NodesWords = "build node qualifies"
// The events the checker says on its stream checks (module.json): a preview or a status was written, so a part that reads
// the bucket, which is made when the checker is assigned and may not exist when that part starts, finds it (ADR 0437 §8: no
// timer). The body is {"key": ...}.
const (
EventPreviewWritten = "preview.written"
EventOwedWritten = "owed.written"
)
// Announcer says an event of the checker's stream, with the key written.
type Announcer func(ctx context.Context, event, key string)
// Outcome is what judging a case gives: the verdict (state, failure, levels, times, composition; the checker adds the
// group, the heads and the trunk) and, for the line, what the check knows of the builds of the modules it touches.
type Outcome struct {
Verdict delivery.Verdict
Builds []BuildNote
// Slowest names the slowest tests of a level that went over its bound ("unit pkg.Test 1.2s").
Slowest []string
}
// BuildNote is the state of a candidate's module build, by module: built, failed or waiting (ADR 0484 clause 7).
type BuildNote struct {
Module string
State string
}
// Judge runs the check of a case: the merge, the levels. It writes what it is doing to c.Progress. It answers an error
// only when it could not judge at all; a failure of the change is a Verdict.
type Judge interface {
Judge(ctx context.Context, c *Case) (Outcome, error)
}
// Checker judges the candidates the indexer writes, dark (ADR 0453 clause 4, ADR 0484): for each candidate it claims a
// verdict key, runs the check through its Judge, writes the verdict and, for each pull request judged, the owed status
// `mesh/check` the forge bridge performs. It holds no timer; a check starts from a candidate or from a change of the
// nodes, the trunk or the controller's plan, and ends when its judge does or its bound passes.
type Checker struct {
Verdicts KV
Owed KV
Judge Judge
Trunks interface{ Heads() map[string]string }
// Bound is how long one check may run; DefaultBound when 0.
Bound time.Duration
// Limit is how many checks run at once; two when 0.
Limit int
// Hold says whether a delivery builds, so no new check starts (ADR 0484 clause 12); nil never holds.
Hold func() bool
// Link is the base of the address a status points at; the verdict key follows.
Link string
// Announce says a status was written (EventOwedWritten); nil says nothing.
Announce Announcer
Now func() time.Time
Logf func(format string, args ...any)
once sync.Once
sem chan struct{}
wg sync.WaitGroup
mu sync.Mutex
known map[string]delivery.Candidate // by candidates key
inflight map[string]bool // by verdict key
again map[string]bool // by verdict key: asked again while it ran
tried map[string]bool // by verdict key: looked at since this process started
epoch uint64 // changes of the nodes seen since this process started
nodesFail map[string]uint64 // by verdict key: the epoch a failure of the nodes was written at
}
func (ch *Checker) init() {
ch.once.Do(func() {
limit := ch.Limit
if limit <= 0 {
limit = 2
}
ch.sem = make(chan struct{}, limit)
ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{}
ch.nodesFail = map[string]uint64{}
})
}
func (ch *Checker) logf(format string, args ...any) {
if ch.Logf != nil {
ch.Logf(format, args...)
}
}
func (ch *Checker) now() time.Time {
if ch.Now != nil {
return ch.Now()
}
return time.Now()
}
func (ch *Checker) bound() time.Duration {
if ch.Bound > 0 {
return ch.Bound
}
return DefaultBound
}
// Wait returns when every check started so far has ended.
func (ch *Checker) Wait() { ch.wg.Wait() }
// Candidate takes one value of the indexer's candidates bucket in (the key is the group) and checks it if it is to be
// checked. A value that is not a candidate is said and left.
func (ch *Checker) Candidate(ctx context.Context, key string, value []byte) {
ch.init()
c, err := delivery.DecodeCandidate(value)
if err != nil {
ch.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return
}
ch.mu.Lock()
ch.known[key] = c
ch.mu.Unlock()
ch.consider(ctx, key, c)
}
// Forget takes a candidate out (its key was deleted).
func (ch *Checker) Forget(key string) {
ch.init()
ch.mu.Lock()
delete(ch.known, key)
ch.mu.Unlock()
}
// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or
// stopped being, or a toolchain moved. A check that failed because no node qualified is run again only after such a change.
func (ch *Checker) NodesChanged(ctx context.Context) {
ch.init()
ch.mu.Lock()
ch.epoch++
ch.mu.Unlock()
ch.Reconsider(ctx)
}
// Reconsider looks at every candidate again: the trunk moved, the nodes changed or a delivery ended. A candidate whose
// verdict stands for the trunk it was judged against is left.
func (ch *Checker) Reconsider(ctx context.Context) {
ch.init()
ch.mu.Lock()
known := make(map[string]delivery.Candidate, len(ch.known))
for k, c := range ch.known {
known[k] = c
}
ch.mu.Unlock()
for k, c := range known {
ch.consider(ctx, k, c)
}
}
func (ch *Checker) heads() map[string]string {
if ch.Trunks == nil {
return nil
}
return ch.Trunks.Heads()
}
func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) {
cs := CaseOf(c, ch.heads())
if cs == nil {
return // nothing to judge: hq's pull requests, or no trunk known
}
if ch.Hold != nil && ch.Hold() {
ch.logf("%s: not started, a delivery builds", cs.Key)
return // started when the plan bucket empties (Reconsider)
}
cs.Queued = ch.now()
ch.mu.Lock()
if ch.inflight[cs.Key] {
ch.again[cs.Key] = true // looked at once more when the check ends, so a change of the trunk is never lost
ch.mu.Unlock()
return
}
ch.inflight[cs.Key] = true
ch.mu.Unlock()
ch.wg.Add(1)
go func() {
defer ch.wg.Done()
ch.sem <- struct{}{}
defer func() { <-ch.sem }()
for {
ch.try(ctx, key, cs)
ch.mu.Lock()
more := ch.again[cs.Key]
delete(ch.again, cs.Key)
if !more || ctx.Err() != nil {
delete(ch.inflight, cs.Key)
ch.mu.Unlock()
return
}
ch.mu.Unlock()
if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil {
cs = next
}
}
}()
}
func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate {
ch.mu.Lock()
defer ch.mu.Unlock()
if c, ok := ch.known[key]; ok {
return c
}
return fallback
}
// claimed is a verdict key the checker holds: the revision the next write is made at.
type claimed struct{ rev uint64 }
// claim takes the check of the case, or answers nil when it is not this checker's to run now:
// - no key: it is created running; a create that finds the key made meanwhile gets nothing;
// - a key left running is taken over as it stands when this process looks at it for the first time (it was left by the
// checker that stopped; there is one instance, as there is one builder per node, ADR 0478 clause 6), and left otherwise;
// - a pass or a failure for the trunk heads the case has stands; one for other trunk heads (the trunk moved) is taken
// again by compare-and-set on the same key, and so is a failure of the nodes after the nodes changed.
func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64) (*claimed, error) {
running := delivery.Verdict{Group: cs.Group, Heads: cs.HeadsHash, Trunk: cs.Trunk, State: delivery.VerdictRunning}
data, err := delivery.EncodeVerdict(running)
if err != nil {
return nil, err
}
value, rev, found, err := ch.Verdicts.Get(ctx, cs.Key)
if err != nil {
return nil, fmt.Errorf("%s: %w", cs.Key, err)
}
if !found {
rev, err := ch.Verdicts.Create(ctx, cs.Key, data)
if errors.Is(err, ErrConflict) {
return nil, nil // another checker made it first
}
if err != nil {
return nil, fmt.Errorf("%s: %w", cs.Key, err)
}
return &claimed{rev}, nil
}
cur, err := delivery.DecodeVerdict(value)
if err != nil {
return nil, fmt.Errorf("%s holds what is not a verdict: %w", cs.Key, err)
}
retake := false
switch {
case cur.State == delivery.VerdictRunning:
if first {
return &claimed{rev}, nil
}
return nil, nil
case !sameTrunk(cur.Trunk, cs.Trunk):
retake = true
case cur.State == delivery.VerdictFail && IsNodesFailure(cur):
ch.mu.Lock()
at, ok := ch.nodesFail[cs.Key]
ch.mu.Unlock()
retake = !ok || epoch > at
}
if !retake {
return nil, nil
}
rev, err = ch.Verdicts.Update(ctx, cs.Key, data, rev)
if errors.Is(err, ErrConflict) {
return nil, nil
}
if err != nil {
return nil, fmt.Errorf("%s: %w", cs.Key, err)
}
return &claimed{rev}, nil
}
// IsNodesFailure says whether a verdict failed because no build node could take the check.
func IsNodesFailure(v delivery.Verdict) bool {
return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, NodesWords)
}
// try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends
// because ctx did is left running: the checker takes it again when it starts.
func (ch *Checker) try(ctx context.Context, key string, cs *Case) {
ch.mu.Lock()
first := !ch.tried[cs.Key]
ch.tried[cs.Key] = true
epoch := ch.epoch
ch.mu.Unlock()
cl, err := ch.claim(ctx, cs, first, epoch)
if err != nil {
ch.logf("%s: %v", cs.Key, err)
return
}
if cl == nil {
return
}
began := ch.now()
ch.writeOwed(ctx, cs, delivery.Verdict{State: delivery.VerdictRunning}, nil, nil)
out := ch.judge(ctx, cs)
if ctx.Err() != nil {
return
}
v := out.Verdict
v.Group, v.Heads, v.Trunk = cs.Group, cs.HeadsHash, cs.Trunk
if v.Took.Queue == 0 && !cs.Queued.IsZero() && began.After(cs.Queued) {
v.Took.Queue = began.Sub(cs.Queued).Milliseconds()
}
data, err := delivery.EncodeVerdict(v)
if err != nil {
ch.logf("%s: the verdict cannot be written: %v", cs.Key, err)
v = delivery.Verdict{Group: cs.Group, Heads: cs.HeadsHash, Trunk: cs.Trunk, State: delivery.VerdictFail,
Failed: delivery.FailedFirst{Machine: "checker", Words: "the verdict could not be written: " + err.Error()}}
if data, err = delivery.EncodeVerdict(v); err != nil {
return
}
}
if _, err := ch.Verdicts.Update(ctx, cs.Key, data, cl.rev); err != nil {
ch.logf("%s: the verdict could not be written: %v", cs.Key, err)
return
}
if IsNodesFailure(v) {
ch.mu.Lock()
ch.nodesFail[cs.Key] = epoch
ch.mu.Unlock()
}
ch.writeOwed(ctx, cs, v, out.Builds, out.Slowest)
ch.logf("%s: %s", cs.Key, v.State)
}
// judge runs the case within the bound. A judge that fails to judge, and a check that ran past its bound, are verdicts of
// failure naming the machine and the step.
func (ch *Checker) judge(ctx context.Context, cs *Case) Outcome {
bound := ch.bound()
run, cancel := context.WithTimeoutCause(ctx, bound, errBound)
defer cancel()
out, err := ch.Judge.Judge(run, cs)
stage, node := cs.Progress.Now()
if node == "" {
node = "checker"
}
switch {
case ctx.Err() != nil:
return Outcome{}
case context.Cause(run) == errBound:
if stage == "" {
stage = "starting"
}
return Outcome{Verdict: failure(node, fmt.Sprintf("the check ran past its bound of %s, stuck at: %s", bound, stage))}
case err != nil:
return Outcome{Verdict: failure(node, err.Error())}
}
return out
}
var errBound = errors.New("the check ran past its bound")
func failure(machine, words string) delivery.Verdict {
return delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: machine, Words: words}}
}
// writeOwed writes, for each pull request judged, the status the forge bridge performs: pending while the check runs, success
// or failure at its end (ADR 0484 clause 2). A key already holding the same value is not written again, so the bridge
// performs one act for each change; an older head's key is never touched.
func (ch *Checker) writeOwed(ctx context.Context, cs *Case, v delivery.Verdict, builds []BuildNote, slowest []string) {
for _, p := range cs.Pulls {
key, err := delivery.CheckerOwedKey(p.Repository, p.Number, p.Head)
if err != nil {
ch.logf("%s: %v", cs.Key, err)
continue
}
state := map[string]string{delivery.VerdictRunning: "pending", delivery.VerdictPass: "success", delivery.VerdictFail: "failure"}[v.State]
o := delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state,
Target: strings.TrimSuffix(ch.Link, "/") + "/" + cs.Key, Text: Line(v, cs, p, builds, slowest)}
data, err := delivery.EncodeCheckerOwed(o)
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err)
continue
}
wrote, err := putOnce(ctx, ch.Owed, key, data, func(cur []byte) bool {
held, err := delivery.DecodeCheckerOwed(cur)
o.V = delivery.Version
return err == nil && delivery.SameOwed(held, o)
})
if err != nil {
ch.logf("%s: the status of %s cannot be written: %v", cs.Key, key, err)
} else if wrote && ch.Announce != nil {
ch.Announce(ctx, EventOwedWritten, key)
}
}
}
// putOnce writes data under key by create, or by compare-and-set when the key exists, and writes nothing when same says the
// key holds the value already.
func putOnce(ctx context.Context, kv KV, key string, data []byte, same func(cur []byte) bool) (bool, error) {
for try := 0; try < 4; try++ {
cur, rev, found, err := kv.Get(ctx, key)
if err != nil {
return false, err
}
if found && same(cur) {
return false, nil
}
if !found {
_, err = kv.Create(ctx, key, data)
} else {
_, err = kv.Update(ctx, key, data, rev)
}
if errors.Is(err, ErrConflict) {
continue
}
return err == nil, err
}
return false, fmt.Errorf("%s changed under every one of 4 tries", key)
}
// Sweep closes, at start, every verdict left running for a candidate that no longer holds those heads: nothing will check
// them again, and the key would stay running for ever (the builder's Sweep, issue 1660). Call it once the candidates have been
// read. A key of heads a candidate holds now is not touched: the check takes it over.
func (ch *Checker) Sweep(ctx context.Context) error {
ch.init()
keys, err := ch.Verdicts.Keys(ctx)
if err != nil {
return err
}
ch.mu.Lock()
current := map[string]bool{}
for _, c := range ch.known {
if k, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash); err == nil {
current[k] = true
}
}
ch.mu.Unlock()
for _, key := range keys {
if current[key] {
continue
}
value, rev, found, err := ch.Verdicts.Get(ctx, key)
if err != nil || !found {
continue
}
cur, err := delivery.DecodeVerdict(value)
if err != nil || cur.State != delivery.VerdictRunning {
continue
}
cur.State = delivery.VerdictFail
cur.Failed = delivery.FailedFirst{Machine: "checker", Words: "closed at start: left running, and no candidate holds these heads any more, so nothing will check them"}
data, err := delivery.EncodeVerdict(cur)
if err != nil {
return err
}
if _, err := ch.Verdicts.Update(ctx, key, data, rev); err != nil && !errors.Is(err, ErrConflict) {
return fmt.Errorf("%s: %w", key, err)
}
ch.logf("%s: closed, left running for heads no candidate holds", key)
}
return nil
}
modules/mesh-delivery-checker/internal/check/checker_test.go
+387
View File
@@ -0,0 +1,387 @@
package check
import (
"context"
"encoding/json"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const (
repoA = "novox/mesh-delivery"
repoSDK = "novox/mesh-sdk"
head1 = "1111111111111111111111111111111111111111"
head2 = "2222222222222222222222222222222222222222"
trunkA = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
trunkA2 = "abababababababababababababababababababab"
trunkS = "5555555555555555555555555555555555555555"
)
// candidateOf is a candidate of a group with the given pull requests, as the indexer writes one.
func candidateOf(t testing.TB, group string, pulls ...delivery.CandidatePull) []byte {
t.Helper()
heads := make([]delivery.Head, len(pulls))
for i, p := range pulls {
heads[i] = p.Head
}
hash, err := delivery.HeadsHash(heads)
if err != nil {
t.Fatal(err)
}
c := delivery.Candidate{V: delivery.Version, Group: group, HeadsHash: hash, Pulls: pulls, FPSet: delivery.FingerprintSet(pulls)}
data, err := delivery.EncodeCandidate(c)
if err != nil {
t.Fatal(err)
}
return data
}
func pullOf(repo string, number int, head string) delivery.CandidatePull {
return delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: head}, Modules: []string{}}
}
// judgeFunc is a Judge written as a function; it counts its calls.
type judgeFunc struct {
mu sync.Mutex
calls int
f func(ctx context.Context, c *Case, call int) (Outcome, error)
}
func (j *judgeFunc) Judge(ctx context.Context, c *Case) (Outcome, error) {
j.mu.Lock()
j.calls++
n := j.calls
j.mu.Unlock()
return j.f(ctx, c, n)
}
func (j *judgeFunc) count() int { j.mu.Lock(); defer j.mu.Unlock(); return j.calls }
func passing() delivery.Verdict {
return delivery.Verdict{State: delivery.VerdictPass, Levels: map[string]delivery.LevelResult{
delivery.LevelUnit: {Run: 2, Kept: 3}, delivery.LevelIntegration: {Run: 1}}}
}
func passJudge() *judgeFunc {
return &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { return Outcome{Verdict: passing()}, nil }}
}
type rig struct {
ch *Checker
verdicts *memKV
owed *memKV
trunks *Trunks
}
func newRig(t testing.TB, j Judge) *rig {
t.Helper()
r := &rig{verdicts: newMemKV(), owed: newMemKV(), trunks: &Trunks{}}
r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS})
r.ch = &Checker{Verdicts: r.verdicts, Owed: r.owed, Judge: j, Trunks: r.trunks, Link: "https://view.invalid/checks", Now: time.Now}
return r
}
func (r *rig) setTrunks(t testing.TB, heads map[string]string) bool {
t.Helper()
data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads})
if err != nil {
t.Fatal(err)
}
return r.trunks.Put(data, false)
}
func (r *rig) verdict(t testing.TB, key string) delivery.Verdict {
t.Helper()
raw, _, found, _ := r.verdicts.Get(context.Background(), key)
if !found {
t.Fatalf("no verdict %s; the bucket holds %v", key, r.verdicts.vals)
}
v, err := delivery.DecodeVerdict(raw)
if err != nil {
t.Fatal(err)
}
return v
}
func keyOf(t testing.TB, group string, value []byte) string {
t.Helper()
c, err := delivery.DecodeCandidate(value)
if err != nil {
t.Fatal(err)
}
k, err := delivery.CheckerVerdictKey(group, c.HeadsHash)
if err != nil {
t.Fatal(err)
}
return k
}
func TestACandidateNotYetCheckedGetsItsVerdictKeyCreatedOnceAndIsJudged(t *testing.T) { // N6/AC1, 1678
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
key := keyOf(t, "feat/x", cand)
if v := r.verdict(t, key); v.State != delivery.VerdictPass || v.Group != "feat/x" || v.Trunk[repoA] != trunkA {
t.Fatalf("verdict = %+v", v)
}
if r.verdicts.creates[key] != 1 || j.count() != 1 {
t.Errorf("creates = %d, judged = %d, want one each", r.verdicts.creates[key], j.count())
}
// The same candidate heard again, with nothing changed, is not judged again.
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 1 {
t.Errorf("judged %d times after the same candidate again, want once", j.count())
}
}
// racyKV lets another instance create the key just before this one's create.
type racyKV struct {
*memKV
other []byte
}
func (r racyKV) Create(ctx context.Context, key string, value []byte) (uint64, error) {
if r.other != nil {
_, _ = r.memKV.Create(ctx, key, r.other)
}
return r.memKV.Create(ctx, key, value)
}
func TestASecondInstancesCreateIsRefusedAndItDoesNothing(t *testing.T) { // N6/AC1
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
key := keyOf(t, "feat/x", cand)
other, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], State: delivery.VerdictRunning})
r.ch.Verdicts = racyKV{r.verdicts, other}
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 0 {
t.Errorf("the instance that lost the create judged %d times, want none", j.count())
}
if v := r.verdict(t, key); v.State != delivery.VerdictRunning {
t.Errorf("the other instance's key was changed: %+v", v)
}
}
func TestARestartWithARunningKeyJudgesItAgain(t *testing.T) { // N6/AC2
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
key := keyOf(t, "feat/x", cand)
left, _ := delivery.EncodeVerdict(delivery.Verdict{Group: "feat/x", Heads: strings.SplitN(key, ".", 2)[1], Trunk: map[string]string{repoA: trunkA}, State: delivery.VerdictRunning})
if _, err := r.verdicts.Create(ctx, key, left); err != nil {
t.Fatal(err)
}
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if j.count() != 1 || r.verdict(t, key).State != delivery.VerdictPass {
t.Fatalf("judged %d times, verdict %+v, want the left check judged once and passed", j.count(), r.verdict(t, key))
}
}
func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3
ctx := context.Background()
j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
c.Progress.Set("test the unit level of mesh-delivery", "shanks")
<-ctx.Done()
return Outcome{}, ctx.Err()
}}
r := newRig(t, j)
r.ch.Bound = 30 * time.Millisecond
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
v := r.verdict(t, keyOf(t, "feat/x", cand))
if v.State != delivery.VerdictFail || v.Failed.Machine != "shanks" || !strings.Contains(v.Failed.Words, "stuck at: test the unit level of mesh-delivery") {
t.Fatalf("verdict = %+v, want a failure naming shanks and the stuck step", v)
}
}
func TestNoQualifyingNodeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4
ctx := context.Background()
j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) {
if n == 1 {
return Outcome{Verdict: failure("shanks", "no go "+NodesWords+": none is online")}, nil
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
key := keyOf(t, "feat/x", cand)
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
if v := r.verdict(t, key); !IsNodesFailure(v) {
t.Fatalf("verdict = %+v, want a failure of the nodes", v)
}
r.ch.Reconsider(ctx) // nothing changed: not judged again
r.ch.Wait()
if j.count() != 1 {
t.Fatalf("judged %d times with the nodes unchanged, want once", j.count())
}
r.ch.NodesChanged(ctx)
r.ch.Wait()
if j.count() != 2 || r.verdict(t, key).State != delivery.VerdictPass {
t.Fatalf("after the nodes changed: judged %d times, verdict %+v", j.count(), r.verdict(t, key))
}
}
func TestHqAndARepositoryWithNoTrunkGetNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clauses 8 and 11
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(HQRepository, 3, head1)))
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1)))
r.ch.Wait()
if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 {
t.Errorf("judged %d times, %d verdicts, %d statuses, want none", j.count(), len(r.verdicts.vals), len(r.owed.vals))
}
}
func TestAGroupWithHqJudgesItsOtherMembersAndWritesNoStatusForHq(t *testing.T) { // ADR 0484 clause 8
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(HQRepository, 3, head1), pullOf(repoA, 7, head2)))
r.ch.Wait()
if j.count() != 1 {
t.Fatalf("judged %d times, want once", j.count())
}
keys, _ := r.owed.Keys(ctx)
if len(keys) != 1 || !strings.HasPrefix(keys[0], "novox/mesh-delivery.7.") && !strings.Contains(keys[0], "mesh-delivery") {
t.Errorf("statuses %v, want one for mesh-delivery#7 only", keys)
}
}
func TestATrunkMoveRewritesTheSameVerdictKeyAndTheHeadsStatus(t *testing.T) { // 1300/AC7
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
key := keyOf(t, "feat/x", cand)
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
_, rev1, _, _ := r.verdicts.Get(ctx, key)
owedKey, _ := delivery.CheckerOwedKey(repoA, 7, head1)
_, orev1, _, _ := r.owed.Get(ctx, owedKey)
if !r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) {
t.Fatal("the trunks did not change")
}
r.ch.Reconsider(ctx)
r.ch.Wait()
_, rev2, _, _ := r.verdicts.Get(ctx, key)
if j.count() != 2 || rev2 <= rev1 || r.verdict(t, key).Trunk[repoA] != trunkA2 {
t.Fatalf("judged %d times, revisions %d then %d, trunk %v", j.count(), rev1, rev2, r.verdict(t, key).Trunk)
}
if r.verdicts.creates[key] != 1 {
t.Errorf("the key was created %d times, want the same key rewritten", r.verdicts.creates[key])
}
if _, orev2, _, _ := r.owed.Get(ctx, owedKey); orev2 <= orev1 {
// the line is the same when nothing differs; the status is rewritten when the words change, here the trunk is in them
t.Logf("the status keeps revision %d (its words did not change)", orev2)
}
// A new head is another key, and the older head's status is not touched.
older, _, _, _ := r.owed.Get(ctx, owedKey)
cand2 := candidateOf(t, "feat/x", pullOf(repoA, 7, head2))
r.ch.Candidate(ctx, "feat%2Fx", cand2)
r.ch.Wait()
if again, _, _, _ := r.owed.Get(ctx, owedKey); string(again) != string(older) {
t.Errorf("the older head's status was rewritten")
}
if n := len(r.owed.vals); n != 2 {
t.Errorf("%d statuses, want one for each head", n)
}
}
func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
held := true
r.ch.Hold = func() bool { return held }
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
r.ch.Wait()
if j.count() != 0 {
t.Fatalf("judged %d times while held", j.count())
}
held = false
r.ch.Reconsider(ctx)
r.ch.Wait()
if j.count() != 1 {
t.Fatalf("judged %d times after the hold ended, want once", j.count())
}
}
func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part)
ctx := context.Background()
var seen []string
var r *rig
j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) {
key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
raw, _, _, _ := r.owed.Get(ctx, key)
o, _ := delivery.DecodeCheckerOwed(raw)
seen = append(seen, o.State)
return Outcome{Verdict: delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "m/a.TestX"}}}, nil
}}
r = newRig(t, j)
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
r.ch.Wait()
key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
raw, _, _, _ := r.owed.Get(ctx, key)
o, err := delivery.DecodeCheckerOwed(raw)
if err != nil {
t.Fatal(err)
}
if len(seen) != 1 || seen[0] != "pending" || o.State != "failure" || o.Context != delivery.StatusContextCheck || !strings.HasPrefix(o.Target, "https://view.invalid/checks/") {
t.Errorf("while running %v, at the end %+v", seen, o)
}
}
func TestSweepClosesARunningKeyNoCandidateHoldsAndLeavesOneItDoes(t *testing.T) {
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
live := keyOf(t, "feat/x", cand)
dead, _ := delivery.CheckerVerdictKey("feat/gone", "0123456789ab")
for _, k := range []string{live, dead} {
group, heads, _ := delivery.ParseCheckerVerdictKey(k)
raw, _ := delivery.EncodeVerdict(delivery.Verdict{Group: group, Heads: heads, State: delivery.VerdictRunning})
_, _ = r.verdicts.Create(ctx, k, raw)
}
r.ch.init()
var c delivery.Candidate
_ = json.Unmarshal(cand, &c)
r.ch.known["feat%2Fx"] = c
if err := r.ch.Sweep(ctx); err != nil {
t.Fatal(err)
}
if v := r.verdict(t, dead); v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "closed at start") {
t.Errorf("the dead key = %+v", v)
}
if v := r.verdict(t, live); v.State != delivery.VerdictRunning {
t.Errorf("the live key = %+v, want it left for the check to take over", v)
}
}
func TestTheForgeBridgeIsToldOnceForEachChangeOfAStatus(t *testing.T) { // 1683 (the checker's half of AC5)
ctx, j := context.Background(), passJudge()
r := newRig(t, j)
var told []string
r.ch.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) }
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1))
r.ch.Candidate(ctx, "feat%2Fx", cand)
r.ch.Wait()
key, _ := delivery.CheckerOwedKey(repoA, 7, head1)
want := []string{EventOwedWritten + " " + key, EventOwedWritten + " " + key} // pending, then success
if len(told) != 2 || told[0] != want[0] || told[1] != want[1] {
t.Fatalf("told %v, want the status said when pending and when success", told)
}
r.ch.Candidate(ctx, "feat%2Fx", cand) // nothing changed: nothing written, nothing said
r.ch.Wait()
if len(told) != 2 {
t.Errorf("told %v after a check that changed nothing", told)
}
}
modules/mesh-delivery-checker/internal/check/doc.go
+6
View File
@@ -0,0 +1,6 @@
// Package check is the checker's logic (novox/hq ADR 0484, ADR 0453 clause 4, ADR 0447, ADR 0444), apart from the bus: the
// claim of a verdict key, the judging of a pull request's head merged onto its trunk through the build modules' fetch,
// digest and test steps, the kept verdicts, the line of the status and the act the forge bridge performs. The bus is
// reached through the small interfaces here, so each part is tested with fakes. The checker runs dark: nothing reads its
// answers to merge or deliver (ADR 0453 clause 4).
package check
modules/mesh-delivery-checker/internal/check/examiner_test.go
+587
View File
@@ -0,0 +1,587 @@
package check
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// captured reads a result the build module's own code marshalled (testdata/captured/README).
func captured[T any](t testing.TB, name string) T {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "captured", name+".json"))
if err != nil {
t.Fatal(err)
}
var v T
if err := json.Unmarshal(raw, &v); err != nil {
t.Fatal(err)
}
return v
}
type stepCall struct {
module, node, tool string
args map[string]any
}
// fakeSteps stands for the build modules: a handler for each tool answers what the module's code answers.
type fakeSteps struct {
mu sync.Mutex
calls []stepCall
handlers map[string]func(c stepCall) (any, error)
}
func (f *fakeSteps) Run(_ context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) {
c := stepCall{module, node, tool, args}
f.mu.Lock()
f.calls = append(f.calls, c)
h := f.handlers[tool]
f.mu.Unlock()
if h == nil {
return nil, fmt.Errorf("the fake build module has no tool %s", tool)
}
v, err := h(c)
if err != nil {
return nil, err
}
return json.Marshal(v)
}
func (f *fakeSteps) tool(name string) []stepCall {
f.mu.Lock()
defer f.mu.Unlock()
var out []stepCall
for _, c := range f.calls {
if strings.HasSuffix(c.tool, "_"+name) {
out = append(out, c)
}
}
return out
}
type entries []delivery.NodeToolchain
func (e entries) Entries(language string) []delivery.NodeToolchain {
var out []delivery.NodeToolchain
for _, x := range e {
if x.Language == language {
out = append(out, x)
}
}
return out
}
type langs map[string][]string
func (l langs) BuildModules(repository string) []string {
if m, ok := l[repository]; ok {
return m
}
return []string{buildrun.GoBuildModule}
}
func goEntries(nodes ...string) entries {
var e entries
for _, n := range nodes {
e = append(e, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: "go", Version: "go1.27.1", Recipe: "7"}})
e = append(e, delivery.NodeToolchain{Node: n, Toolchain: delivery.Toolchain{Language: "typescript", Version: "node24", Recipe: "3"}})
}
return e
}
const digC = "cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33cc33"
const digD = "dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44"
type bench struct {
ex *Examiner
steps *fakeSteps
kept *memKV
prev *memKV
fetch Fetched
digest map[string]Digested // by level
test func(c stepCall) (any, error)
}
// newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass
// the captured slow fixture's result at both levels, on shanks.
func newBench(t testing.TB) *bench {
b := &bench{steps: &fakeSteps{}, kept: newMemKV(), prev: newMemKV()}
b.fetch = captured[Fetched](t, "go-fetch-clean")
b.fetch.Modules, b.fetch.Changed, b.fetch.Trees = []string{"."}, []string{"README.md"}, nil
one := captured[Digested](t, "go-digest")
b.digest = map[string]Digested{
delivery.LevelUnit: {Level: "unit", Packages: map[string]string{"fixture/slow": digA}},
delivery.LevelIntegration: {Level: "integration", Packages: map[string]string{"fixture/slow": digB}},
}
_ = one
slow := captured[Tested](t, "go-test-slow")
b.test = func(c stepCall) (any, error) {
r := slow
r.Level, _ = c.args["level"].(string)
return r, nil
}
b.steps.handlers = map[string]func(stepCall) (any, error){
"build_go_fetch": func(stepCall) (any, error) { return b.fetch, nil },
"build_node_fetch": func(stepCall) (any, error) { return b.fetch, nil },
"build_go_digest": func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil },
"build_go_test": func(c stepCall) (any, error) { return b.test(c) },
"build_node_test": func(c stepCall) (any, error) { return b.test(c) },
}
now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
b.ex = &Examiner{Steps: b.steps, Toolchains: goEntries("shanks"), Places: &buildrun.Places{}, Kept: Kept{Store: b.kept}, Previews: b.prev,
Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }}
return b
}
func caseOf(t testing.TB, pulls ...delivery.CandidatePull) *Case {
t.Helper()
raw := candidateOf(t, "feat/x", pulls...)
c, err := delivery.DecodeCandidate(raw)
if err != nil {
t.Fatal(err)
}
cs := CaseOf(c, map[string]string{repoA: trunkA, repoSDK: trunkS, "novox/mesh-controller": trunkA, "novox/mesh-catalog": trunkA})
if cs == nil {
t.Fatal("no case")
}
return cs
}
func (b *bench) judge(t testing.TB, cs *Case) Outcome {
t.Helper()
out, err := b.ex.Judge(context.Background(), cs)
if err != nil {
t.Fatal(err)
}
return out
}
func testedWith(t testing.TB, name string, edit func(*Tested)) Tested {
r := captured[Tested](t, name)
if edit != nil {
edit(&r)
}
return r
}
func packagesOf(c stepCall) []string {
var out []string
if l, ok := c.args["packages"].([]string); ok {
out = l
}
return out
}
func TestACleanMergeIsFetchedAtItsTrunkAndBothLevelsPass(t *testing.T) { // 1679/AC1
b := newBench(t)
cs := caseOf(t, pullOf(repoA, 7, head1))
out := b.judge(t, cs)
v := out.Verdict
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Run != 2 || v.Levels[delivery.LevelIntegration].Run != 2 {
t.Fatalf("verdict = %+v", v)
}
if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun {
t.Errorf("composition = %+v, want not run", v.Composition)
}
if len(v.Machines) != 1 || v.Machines[0] != "shanks" {
t.Errorf("machines = %v", v.Machines)
}
fetch := b.steps.tool("fetch")
if len(fetch) != 1 || fetch[0].args["repository"] != repoA || fetch[0].args["commit"] != head1 || fetch[0].args["base"] != trunkA || fetch[0].args["ref"] != "refs/pull/7/head" {
t.Fatalf("fetch calls = %+v", fetch)
}
if v.Took.Clone <= 0 || v.Took.Unit <= 0 {
t.Errorf("took = %+v, want the clone and the unit level timed", v.Took)
}
// What passed is kept under each level's digest.
for _, k := range []string{"unit." + digA, "integration." + digB} {
if _, _, found, _ := b.kept.Get(context.Background(), k); !found {
t.Errorf("%s was not kept", k)
}
}
}
func TestOnlyThePackageWithoutAKeptVerdictRuns(t *testing.T) { // 1288/AC1
b := newBench(t)
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/slow": digA, "fixture/other": digC}}
if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digC, keptValue("fixture/other", 40)); err != nil {
t.Fatal(err)
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests := b.steps.tool("test")
if len(tests) != 2 || strings.Join(packagesOf(tests[0]), ",") != "fixture/slow" || tests[0].args["level"] != "unit" {
t.Fatalf("test calls = %+v, want the unit level of fixture/slow alone, then the integration level", tests)
}
if u := out.Verdict.Levels[delivery.LevelUnit]; u.Run != 2 || u.Kept != 40 {
t.Errorf("unit = %+v, want 2 run and 40 kept", u)
}
}
func TestAReAskRunsOnlyWhatFailedAndWhatChanged(t *testing.T) { // 1288/AC2, AC3
b := newBench(t)
failing := captured[Tested](t, "go-test-failing")
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digB}}
b.test = func(c stepCall) (any, error) {
if c.args["level"] == "unit" {
return failing, nil
}
return captured[Tested](t, "go-test-slow"), nil
}
cs := caseOf(t, pullOf(repoA, 7, head1))
first := b.judge(t, cs).Verdict
if first.State != delivery.VerdictFail || first.Failed.Test != "unit: fixture/failing/bad.TestThreeIsFour" {
t.Fatalf("first = %+v", first)
}
if _, _, found, _ := b.kept.Get(context.Background(), "unit."+digB); !found {
t.Fatal("the package that passed in the failing run was not kept")
}
if _, _, found, _ := b.kept.Get(context.Background(), "unit."+digA); found {
t.Fatal("the package that failed was kept")
}
// Asked again with nothing changed: only the failed package runs; the good one is taken.
b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests := b.steps.tool("test")
if len(tests) == 0 || strings.Join(packagesOf(tests[0]), ",") != "fixture/failing/bad" {
t.Fatalf("the re-ask ran %+v, want only fixture/failing/bad", tests)
}
// The trunk moved and one digest changed: that package runs, the unchanged one is taken.
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"fixture/failing/bad": digA, "fixture/failing/good": digD}}
b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
tests = b.steps.tool("test")
if got := strings.Join(packagesOf(tests[0]), ","); got != "fixture/failing/bad,fixture/failing/good" && got != "fixture/failing/good,fixture/failing/bad" {
t.Errorf("after a digest changed the run was %q, want the failed package and the changed one", got)
}
}
func TestANodePackageIsNeverKeptAndRunsEveryCheck(t *testing.T) { // 1288/AC4, 1292/AC1
b := newBench(t)
b.ex.Modules = langs{repoA: {buildrun.NodeBuildModule}}
b.fetch = captured[Fetched](t, "node-fetch-clean")
b.test = func(c stepCall) (any, error) {
return Tested{Level: c.args["level"].(string), Module: "modules/m", Passed: true, Elapsed: 0.4, Bound: 5}, nil
}
cs := caseOf(t, pullOf(repoA, 7, head1))
for i := 0; i < 2; i++ {
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass || v.Levels[delivery.LevelUnit].Kept != 0 || v.Levels[delivery.LevelUnit].Run != 1 {
t.Fatalf("round %d: verdict = %+v", i, v)
}
}
if n := len(b.steps.tool("test")); n != 4 {
t.Errorf("%d node test calls in two checks, want 4 (both levels each time)", n)
}
if n := len(b.steps.tool("digest")); n != 0 {
t.Errorf("the node build module was asked for a digest %d times", n)
}
if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
t.Errorf("kept %v for a node package", keys)
}
}
func TestTheChangesOwnFailingTestRunsFirstAndNothingElseRuns(t *testing.T) { // 1289/AC1, AC2
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
failing := captured[Tested](t, "go-test-failing")
b.test = func(c stepCall) (any, error) {
if got := packagesOf(c); len(got) == 1 && got[0] == "m/internal/b" {
return failing, nil
}
return captured[Tested](t, "go-test-slow"), nil
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictFail || !strings.Contains(out.Verdict.Failed.Test, "TestThreeIsFour") {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 1 || strings.Join(packagesOf(tests[0]), ",") != "m/internal/b" {
t.Fatalf("test calls = %+v, want only m/internal/b, once", tests)
}
if n := len(b.steps.tool("digest")); n != 1 {
t.Errorf("%d digest calls, want the unit level's alone (the check stopped)", n)
}
if out.Verdict.Composition == nil || out.Verdict.Composition.State != delivery.CompositionNotRun {
t.Errorf("composition = %+v", out.Verdict.Composition)
}
}
func TestTheChangesOwnPassingTestRunsFirstAndTheRestFollow(t *testing.T) { // 1289/AC3
b := newBench(t)
b.fetch.Changed = []string{"modules/m/internal/b/b_test.go", "modules/m/internal/a/a.go"}
b.digest[delivery.LevelUnit] = Digested{Level: "unit", Packages: map[string]string{"m/internal/a": digA, "m/internal/b": digB}}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", out.Verdict)
}
var unit []string
for _, c := range b.steps.tool("test") {
if c.args["level"] == "unit" {
unit = append(unit, strings.Join(packagesOf(c), ","))
}
}
if strings.Join(unit, " | ") != "m/internal/b | m/internal/a" {
t.Errorf("the unit level ran %v, want the changed test's package first, then the rest", unit)
}
}
func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T) { // 1289/AC4
b := newBench(t)
b.fetch = captured[Fetched](t, "go-fetch-conflict")
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
f := out.Verdict.Failed
if out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "files in conflict: a.go, sub/b.go") {
t.Fatalf("verdict = %+v", out.Verdict)
}
if n := len(b.steps.tool("digest")) + len(b.steps.tool("test")); n != 0 {
t.Errorf("%d digest and test steps ran after a conflict", n)
}
}
func TestNoQualifyingNodeIsAFailureOfTheNodes(t *testing.T) { // N6/AC4
b := newBench(t)
b.ex.Toolchains = entries{}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if !IsNodesFailure(out.Verdict) {
t.Fatalf("verdict = %+v, want a failure of the nodes", out.Verdict)
}
b.ex.Toolchains = goEntries("shanks")
b.ex.Presence = offline{}
if out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1))); !IsNodesFailure(out.Verdict) {
t.Fatalf("an offline node: verdict = %+v", out.Verdict)
}
}
type offline struct{}
func (offline) Online(string) bool { return false }
func TestAStepThatFailsIsAVerdictNamingTheNode(t *testing.T) {
b := newBench(t)
b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { return nil, fmt.Errorf("the step failed: ssh refused") }
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "ssh refused") {
t.Fatalf("verdict = %+v", out.Verdict)
}
}
func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4
b := newBench(t)
b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, fmt.Errorf("go list failed") }
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if out.Verdict.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", out.Verdict)
}
tests := b.steps.tool("test")
if len(tests) != 2 || tests[0].args["packages"] != nil {
t.Errorf("test calls = %+v, want each level once, whole", tests)
}
if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
t.Errorf("kept %v with no digest", keys)
}
}
func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndKeepsItsVerdict(t *testing.T) { // 1300/AC3
b := newBench(t)
b.test = func(c stepCall) (any, error) {
return testedWith(t, "go-test-slow", func(r *Tested) {
r.Level = c.args["level"].(string)
r.OverBound = true
r.OverWhich = "run"
r.Elapsed = 7
}), nil
}
out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
v := out.Verdict
if v.State != delivery.VerdictPass || strings.Join(v.OverBound, ",") != "unit,integration" {
t.Fatalf("verdict = %+v", v)
}
if len(out.Slowest) == 0 || !strings.Contains(out.Slowest[0], "TestTakesAWhile") {
t.Errorf("slowest = %v", out.Slowest)
}
}
func TestSkippedTestsAreCountedPerLevel(t *testing.T) { // 1300/AC6, 1251
b := newBench(t)
b.test = func(c stepCall) (any, error) {
return testedWith(t, "go-test-slow", func(r *Tested) { r.Level = c.args["level"].(string); r.Skipped = 3 }), nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.Levels[delivery.LevelUnit].Skipped != 3 || v.Levels[delivery.LevelIntegration].Skipped != 3 {
t.Errorf("levels = %+v", v.Levels)
}
}
func TestAGroupIsOneVerdictAndEachMembersTestsReadTheOthersAtTheirHeads(t *testing.T) { // 1624/AC1
b := newBench(t)
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
v := b.judge(t, cs).Verdict
if v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
fetch := b.steps.tool("fetch")
if len(fetch) != 2 {
t.Fatalf("%d fetches, want one for each member", len(fetch))
}
var forA []BesideRef
for _, f := range fetch {
if f.args["repository"] == repoA {
forA, _ = f.args["beside"].([]BesideRef)
}
}
found := false
for _, r := range forA {
if r.Repository == repoSDK && r.Commit == head2 && r.Ref == "refs/pull/12/head" {
found = true
}
}
if !found {
t.Errorf("mesh-delivery's tests were not given the SDK member at its head: %+v", forA)
}
}
func TestAnExaminerIsAJudgeOfTheChecker(t *testing.T) {
var _ Judge = (*Examiner)(nil)
}
const (
manA = "0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a0a"
)
func TestACleanMergeWritesItsPreviewWithTheTreesManifestsAndLocks(t *testing.T) { // 1445/AC1
b := newBench(t)
b.fetch.Trees = map[string]MergeTree{
"modules/avahi": {Tree: "7777777777777777777777777777777777777777", Manifest: manA, Locks: map[string]string{"go.sum": "ff00"}},
"modules/bare": {Tree: "8888888888888888888888888888888888888888"}, // no manifest digest: left out
}
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
key, _ := delivery.PreviewKey(repoA, head1, trunkA)
raw, _, found, _ := b.prev.Get(context.Background(), key)
if !found {
t.Fatalf("no preview %s; the bucket holds %v", key, b.prev.vals)
}
pv, err := delivery.DecodePreview(raw)
if err != nil {
t.Fatal(err)
}
m := pv.Modules["modules/avahi"]
if pv.Merge != b.fetch.Merge || pv.Trunk != trunkA || pv.Head != head1 || len(pv.Modules) != 1 || m.Tree != "7777777777777777777777777777777777777777" || m.Manifest != manA || m.Locks["go.sum"] != "ff00" {
t.Errorf("preview = %+v", pv)
}
}
func TestAHeadThatDoesNotMergeWritesAPreviewOfItsConflictsAlone(t *testing.T) { // 1445/AC5
b := newBench(t)
b.fetch = captured[Fetched](t, "go-fetch-conflict")
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
key, _ := delivery.PreviewKey(repoA, head1, trunkA)
raw, _, found, _ := b.prev.Get(context.Background(), key)
if !found {
t.Fatalf("no preview for a head that does not merge; the bucket holds %v", b.prev.vals)
}
pv, err := delivery.DecodePreview(raw)
if err != nil || strings.Join(pv.Conflicts, ",") != "a.go,sub/b.go" || pv.Merge != "" || len(pv.Modules) != 0 {
t.Errorf("preview = %+v, %v, want the files in conflict and no merge", pv, err)
}
}
func TestAPreviewIsWrittenOnceAndAnExistingKeyStands(t *testing.T) {
b := newBench(t)
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
b.fetch.Merge = "5656565656565656565656565656565656565656"
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
key, _ := delivery.PreviewKey(repoA, head1, trunkA)
if b.prev.creates[key] != 1 {
t.Errorf("the preview was created %d times, want once", b.prev.creates[key])
}
raw, _, _, _ := b.prev.Get(context.Background(), key)
if pv, _ := delivery.DecodePreview(raw); pv.Merge == b.fetch.Merge {
t.Error("the second merge commit replaced the first")
}
}
func TestTheIndexerIsToldOfAPreviewOnceWhenItIsCreated(t *testing.T) { // 1445/AC4 (the checker's half)
b := newBench(t)
var told []string
b.ex.Announce = func(_ context.Context, event, key string) { told = append(told, event+" "+key) }
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
key, _ := delivery.PreviewKey(repoA, head1, trunkA)
if len(told) != 1 || told[0] != EventPreviewWritten+" "+key {
t.Errorf("told %v, want once of %s", told, key)
}
}
type fakeBuilds map[string]delivery.BuilderBuild
func (f fakeBuilds) Build(_ context.Context, module, fp2 string) (delivery.BuilderBuild, bool, error) {
b, ok := f[module+" "+fp2]
return b, ok, nil
}
func TestTheOutcomeSaysIfEachCandidatesModuleIsBuiltFailedOrWaiting(t *testing.T) { // N7/AC1 (the checker reads the builder's keys)
b := newBench(t)
fpA := "src2:" + strings.Repeat("a", 64)
fpB := "src2:" + strings.Repeat("b", 64)
fpC := "src2:" + strings.Repeat("c", 64)
b.ex.Builds = fakeBuilds{
"avahi " + fpA: {State: delivery.BuildDone},
"mesh-tools " + fpB: {State: delivery.BuildFailed, Error: "boom"},
// netcheck has no key yet
}
p := pullOf(repoA, 7, head1)
p.Modules = []string{"avahi", "mesh-tools", "netcheck", "nofp"}
p.Fps = map[string]delivery.CandidateModule{"avahi": {FP2: fpA}, "mesh-tools": {FP2: fpB}, "netcheck": {FP2: fpC}, "nofp": {None2: "toolchain not yet reported"}}
out := b.judge(t, caseOf(t, p))
var got []string
for _, n := range out.Builds {
got = append(got, n.Module+" "+n.State)
}
if strings.Join(got, ", ") != "avahi built, mesh-tools failed, netcheck waiting" {
t.Errorf("builds = %v, want one note for each module that has a fingerprint, sorted", got)
}
}
// trunksOf are trunk heads of every repository, as the gate's source names them.
func trunksOf(t *testing.T, heads map[string]string) *Trunks {
t.Helper()
data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: heads})
if err != nil {
t.Fatal(err)
}
tr := &Trunks{}
tr.Put(data, false)
return tr
}
func TestNothingIsKeptForAWorkspaceGivenSiblingsBeside(t *testing.T) { // review D1, ADR 0444, issue 1287
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA})
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if got := len(b.steps.tool("fetch")); got != 1 || b.steps.tool("fetch")[0].args["beside"] == nil {
t.Fatalf("the fetch was not given siblings: %+v", b.steps.tool("fetch"))
}
if keys, _ := b.kept.Keys(context.Background()); len(keys) != 0 {
t.Errorf("kept %v for a package whose tests may read a sibling the digest does not cover", keys)
}
// And a kept verdict of the same digest from elsewhere is not taken for it: everything runs.
if _, err := (Kept{Store: b.kept}).Keep(context.Background(), "unit", digA, keptValue("fixture/slow", 2)); err != nil {
t.Fatal(err)
}
b.steps.calls = nil
b.judge(t, caseOf(t, pullOf(repoA, 7, head1)))
if n := len(b.steps.tool("test")); n != 2 {
t.Errorf("%d test steps, want both levels run whole", n)
}
}
modules/mesh-delivery-checker/internal/check/fakes_test.go
+62
View File
@@ -0,0 +1,62 @@
package check
import (
"context"
"sort"
"sync"
)
// memKV is a bucket in memory with the revisions a real one gives.
type memKV struct {
mu sync.Mutex
vals map[string][]byte
revs map[string]uint64
next uint64
// creates counts the creates that succeeded, by key.
creates map[string]int
}
func newMemKV() *memKV {
return &memKV{vals: map[string][]byte{}, revs: map[string]uint64{}, creates: map[string]int{}}
}
func (m *memKV) Get(_ context.Context, key string) ([]byte, uint64, bool, error) {
m.mu.Lock()
defer m.mu.Unlock()
v, ok := m.vals[key]
return v, m.revs[key], ok, nil
}
func (m *memKV) Create(_ context.Context, key string, value []byte) (uint64, error) {
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.vals[key]; ok {
return 0, ErrConflict
}
m.next++
m.vals[key], m.revs[key] = append([]byte(nil), value...), m.next
m.creates[key]++
return m.next, nil
}
func (m *memKV) Update(_ context.Context, key string, value []byte, revision uint64) (uint64, error) {
m.mu.Lock()
defer m.mu.Unlock()
if _, ok := m.vals[key]; !ok || m.revs[key] != revision {
return 0, ErrConflict
}
m.next++
m.vals[key], m.revs[key] = append([]byte(nil), value...), m.next
return m.next, nil
}
func (m *memKV) Keys(context.Context) ([]string, error) {
m.mu.Lock()
defer m.mu.Unlock()
keys := make([]string, 0, len(m.vals))
for k := range m.vals {
keys = append(keys, k)
}
sort.Strings(keys)
return keys, nil
}
modules/mesh-delivery-checker/internal/check/hold.go
+7
View File
@@ -0,0 +1,7 @@
package check
// PlansBuildingBucket is the controller's bucket of the delivery plans in their build phase, one key for each plan, written
// by the controller alone (issue 1652): a plan key there means a delivery builds on the build nodes, and no new check starts
// meanwhile (ADR 0484 clause 12, ADR 0483 clause 6). The keys age out with no delete marker, so the keys are listed, not watched. The bucket may be absent: the controller's issue 1652 is delivered
// before R4c-5b, and if it is not, the bucket is read as empty.
const PlansBuildingBucket = "mesh-controller_plans-building"
modules/mesh-delivery-checker/internal/check/hold_test.go
+38
View File
@@ -0,0 +1,38 @@
package check
import (
"context"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func TestARunningCheckIsNotStoppedWhenADeliveryStartsToBuild(t *testing.T) { // ADR 0484 clause 12, ADR 0483 clause 4
ctx := context.Background()
held := false
j := &judgeFunc{f: func(_ context.Context, _ *Case, call int) (Outcome, error) {
if call == 1 {
held = true // a delivery starts while this check runs
}
return Outcome{Verdict: passing()}, nil
}}
r := newRig(t, j)
r.ch.Hold = func() bool { return held }
r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))
r.ch.Wait()
if v := r.verdict(t, keyOf(t, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))); v.State != delivery.VerdictPass {
t.Errorf("verdict = %+v, want the running check to end as it was", v)
}
// A new candidate waits while the plan builds.
r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf(repoA, 8, head2)))
r.ch.Wait()
if j.count() != 1 {
t.Errorf("judged %d times while a delivery builds, want only the one already running", j.count())
}
held = false
r.ch.Reconsider(ctx)
r.ch.Wait()
if j.count() != 2 {
t.Errorf("judged %d times after the delivery ended, want the waiting candidate too", j.count())
}
}
modules/mesh-delivery-checker/internal/check/judge.go
+500
View File
@@ -0,0 +1,500 @@
package check
import (
"context"
"encoding/json"
"fmt"
"path"
"sort"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"git.novox.be/novox/mesh-sdk/go/delivery/buildrun"
)
// goModule is the build module the Go repositories are judged by.
const goModule = buildrun.GoBuildModule
// verdictPtr is a verdict of failure a step of the check ends with.
type verdictPtr = *delivery.Verdict
func failed(machine, words string) verdictPtr { return fail(machine, words) }
// Steps runs a build module's long step on a node and answers its result (buildrun.Runner).
type Steps interface {
Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error)
}
// ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet).
type ToolchainEntries interface {
Entries(language string) []delivery.NodeToolchain
}
// BuildModules says which build modules judge a repository: the Go one always, the Node one when the repository holds a
// TypeScript module.
type BuildModules interface {
BuildModules(repository string) []string
}
// toolPrefix is the start of each build module's tool names.
var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"}
// Examiner judges a case through the build modules (ADR 0484 clauses 3, 4, 10, 13): for each pull request it fetches the head
// merged onto its trunk head on a build node, then runs the unit level and the integration level, taking the kept verdicts
// of unchanged packages, running the change's own tests first and stopping at the first failure.
type Examiner struct {
Steps Steps
Toolchains ToolchainEntries
// Presence says which nodes are online; Health whether a node's build module is running and healthy. A node that is not
// is not chosen; nil means every node is.
Presence interface{ Online(node string) bool }
Health interface {
Healthy(node, module string) bool
}
// Places holds a step back while its node is at the checker's capacity (ADR 0484 clause 12).
Places *buildrun.Places
Kept Kept
// Previews is the previews bucket; nil writes none.
Previews KV
Modules BuildModules
// Runs says which commit the mesh runs of a module (the indexer's modules); nil knows none.
Runs interface{ CommitOf(module string) string }
// Forge reads the go.mod files of the Go members and says whether a mesh-sdk commit is on its trunk (ADR 0484 clause 9);
// nil checks no pin.
Forge Forge
// Builds says how the build of a fingerprint the candidate carries stands (the builder's builds bucket); nil says nothing
// of the builds.
Builds BuildStates
// Announce says a preview was written under the key (EventPreviewWritten); nil says nothing.
Announce Announcer
// Trunks are the trunk heads of every repository, for the siblings a repository's tests read.
Trunks interface{ Heads() map[string]string }
Now func() time.Time
Logf func(format string, args ...any)
}
func (j *Examiner) now() time.Time {
if j.Now != nil {
return j.Now()
}
return time.Now()
}
func (j *Examiner) logf(format string, args ...any) {
if j.Logf != nil {
j.Logf(format, args...)
}
}
// target is a pull request on one build module: the node chosen, the workspace its fetch made and what it said.
type target struct {
pull Pull
module string
node string
toolchain string
fetched Fetched
// priority are the Go packages (import paths) or the Node module directories whose test files the merge changes.
changed []string
// beside says the workspace was given siblings: its tests may read what the digest does not cover, so nothing is kept for it.
beside bool
plans map[string]*goPlan // by level
}
// goPlan is what a Go target has to run at one level: the digests, the packages without a kept verdict and the tests taken.
type goPlan struct {
digests map[string]string
todo []string
// all says the digest step gave nothing to key a kept verdict by, so the level runs whole and keeps nothing.
all bool
}
// acc collects a level's counts across targets.
type acc struct {
run, kept, skipped int
elapsed, setup float64
over bool
slowest []TestTime
}
// state is one judging.
type state struct {
j *Examiner
c *Case
targets []*target
levels map[string]*acc
nodes map[string]bool
clone time.Duration
slow []string
}
// Judge runs the case (it is the Judge of a Checker). A failure of the change, of a merge or of a step is a verdict; an error is only a case that cannot
// be judged at all.
func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) {
st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}}
if v := st.fetchAll(ctx); v != nil {
return st.finish(ctx, *v), nil
}
if v := st.checkPins(ctx); v != nil {
return st.finish(ctx, *v), nil
}
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
for _, priority := range []bool{true, false} {
for _, t := range st.targets {
if v := st.runTarget(ctx, t, level, priority); v != nil {
return st.finish(ctx, *v), nil
}
}
}
}
return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil
}
// finish is the outcome of the verdict with the state of the builds the candidate's modules carry.
func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome {
out := st.outcome(v)
out.Builds = st.j.buildNotes(ctx, st.c)
return out
}
// outcome fills the verdict's levels, times, machines and composition from what ran.
func (st *state) outcome(v delivery.Verdict) Outcome {
if v.State == delivery.VerdictPass || len(v.Levels) == 0 {
v.Levels = map[string]delivery.LevelResult{}
for level, a := range st.levels {
v.Levels[level] = delivery.LevelResult{Run: a.run, Kept: a.kept, Skipped: a.skipped}
}
}
ms := func(s float64) int64 { return int64(s * 1000) }
u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration]
v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)}
for node := range st.nodes {
v.Machines = append(v.Machines, node)
}
sort.Strings(v.Machines)
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
if st.levels[level].over {
v.OverBound = append(v.OverBound, level)
}
}
// The composition is the assigner's check (issue 1297): until it delivers a verdict says it was not run (ADR 0484 clause 6).
v.Composition = &delivery.Composition{State: delivery.CompositionNotRun}
var slow []string
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
a := st.levels[level]
if !a.over {
continue
}
sort.SliceStable(a.slowest, func(x, y int) bool { return a.slowest[x].Elapsed > a.slowest[y].Elapsed })
for n, s := range a.slowest {
if n == 3 {
break
}
slow = append(slow, fmt.Sprintf("%s %s %.1fs", level, path.Base(s.Package)+"."+s.Test, s.Elapsed))
}
}
return Outcome{Verdict: v, Slowest: slow}
}
// fail is a verdict of failure at a machine.
func fail(machine, words string) *delivery.Verdict {
v := failure(machine, words)
return &v
}
// step runs one step of a build module on the node the target is on, holding a place there while it runs.
func (st *state) step(ctx context.Context, t *target, tool, label string, args map[string]any) (json.RawMessage, error) {
st.c.Progress.Set(label, t.node)
release, err := st.j.Places.Acquire(ctx, t.node)
if err != nil {
return nil, err
}
defer release()
return st.j.Steps.Run(ctx, t.module, t.node, toolPrefix[t.module]+tool, args)
}
// choose picks the build node for a pull request's repository on a build module.
func (st *state) choose(bm string) (node, toolchain string, err error) {
language := buildrun.LanguageOf[bm]
qualifies := func(node string) bool {
return (st.j.Presence == nil || st.j.Presence.Online(node)) && (st.j.Health == nil || st.j.Health.Healthy(node, bm))
}
nodes, tc, err := buildrun.ChooseNodes(language, st.j.Toolchains.Entries(language), st.j.Places, qualifies, 1)
if err != nil {
return "", "", err
}
return nodes[0], tc.Version, nil
}
// fetchAll fetches every pull request's head merged onto its trunk on a build node of each build module the repository
// needs. A head that does not merge, or a step that fails, ends the check at once, running no test (ADR 0484 clause 13).
func (st *state) fetchAll(ctx context.Context) *delivery.Verdict {
began := st.j.now()
defer func() { st.clone = st.j.now().Sub(began) }()
for _, p := range st.c.Pulls {
var merged []Fetched
for _, bm := range st.j.Modules.BuildModules(p.Repository) {
node, tc, err := st.choose(bm)
if err != nil {
return fail(bm, err.Error())
}
t := &target{pull: p, module: bm, node: node, toolchain: tc, plans: map[string]*goPlan{}}
st.nodes[node] = true
args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk}
if bm == buildrun.GoBuildModule {
beside, err := st.j.besideFor(ctx, st.c, p)
if err != nil {
return fail(node, err.Error())
}
if len(beside) > 0 {
args["beside"] = beside
t.beside = true
}
}
raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args)
if err != nil {
return fail(node, fmt.Sprintf("fetch of %s#%d: %v%s", p.Repository, p.Number, err, pinHint(st.c, err.Error())))
}
if err := json.Unmarshal(raw, &t.fetched); err != nil || t.fetched.Workspace == "" {
return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw))
}
if len(t.fetched.Conflicts) > 0 {
st.j.writePreview(ctx, p, append(merged, t.fetched))
return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", ")))
}
t.changed = t.fetched.Changed
merged = append(merged, t.fetched)
st.targets = append(st.targets, t)
}
st.j.writePreview(ctx, p, merged)
}
return nil
}
// isTestFile says whether a repository-relative path is a test file of the languages the checker judges: Go's _test.go and
// the TypeScript module's .test. and .spec. files and test directories.
func isTestFile(p string) bool {
switch {
case strings.HasSuffix(p, "_test.go"):
return true
case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."):
return true
}
for _, seg := range strings.Split(path.Dir(p), "/") {
if seg == "__tests__" || seg == "test" || seg == "tests" {
return true
}
}
return false
}
// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the directory of the
// file and the import path agree on their last segments, whichever holds the module's directory and path in front.
func changesTestOf(changed []string, pkg string) bool {
for _, f := range changed {
if !strings.HasSuffix(f, "_test.go") {
continue
}
dir := path.Dir(f)
if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) {
return true
}
}
return false
}
// changesTestIn says whether the merge changes a test file inside the directory of a Node module.
func changesTestIn(changed []string, dir string) bool {
for _, f := range changed {
if !isTestFile(f) || strings.HasSuffix(f, "_test.go") {
continue
}
if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") {
return true
}
}
return false
}
// failedName names a failure: its package and test.
func failedName(f Failure) string {
if f.Test == "" {
return f.Package
}
return f.Package + "." + f.Test
}
// firstLine is the first non-empty line of a failure's output, shortened.
func firstLine(s string) string {
for _, l := range strings.Split(s, "\n") {
if l = strings.TrimSpace(l); l != "" {
if len(l) > 120 {
l = l[:120]
}
return l
}
}
return ""
}
// failure turns a level that did not pass into a verdict naming what failed first.
func levelFailure(t *target, level string, res Tested) *delivery.Verdict {
v := delivery.Verdict{State: delivery.VerdictFail}
switch {
case len(res.Failed) > 0:
f := res.Failed[0]
name := failedName(f)
if t.module == buildrun.NodeBuildModule && f.Package == "" {
name = res.Module
}
v.Failed = delivery.FailedFirst{Test: level + ": " + name, Words: firstLine(f.Output)}
default:
why := res.Why
if why == "" {
why = "the " + level + " level did not pass"
}
v.Failed = delivery.FailedFirst{Test: level + ": " + t.pull.Repository, Words: why}
}
return &v
}
// runTarget runs one level of a target: first the packages the change's own tests are in (priority), then the others.
func (st *state) runTarget(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict {
if t.module == buildrun.NodeBuildModule {
return st.runNode(ctx, t, level, priority)
}
return st.runGo(ctx, t, level, priority)
}
// A level's work is planned once for a target: the digest of each package, the kept verdicts taken, the packages left to run.
func (st *state) planGo(ctx context.Context, t *target, level string) *delivery.Verdict {
if t.plans[level] != nil {
return nil
}
plan := &goPlan{digests: map[string]string{}}
t.plans[level] = plan
if t.beside {
// The digest covers the package's own files, the toolchain and the header, not the siblings placed beside the tree: a
// package whose tests read one must never be kept, nor taken from a verdict kept for other inputs (ADR 0444).
plan.all = true
return nil
}
raw, err := st.step(ctx, t, "digest", fmt.Sprintf("digest the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node),
map[string]any{"workspace": t.fetched.Workspace, "level": level})
var d Digested
if err == nil {
err = json.Unmarshal(raw, &d)
}
if ctx.Err() != nil {
return fail(t.node, ctx.Err().Error())
}
if err != nil || len(d.Packages) == 0 {
// The build module named no digest (a level it cannot digest here): the level runs whole and keeps nothing.
if err != nil {
st.j.logf("%s#%d: no digest of the %s level, so it runs whole: %v", t.pull.Repository, t.pull.Number, level, err)
}
plan.all = true
return nil
}
plan.digests = d.Packages
a := st.levels[level]
pkgs := make([]string, 0, len(d.Packages))
for p := range d.Packages {
pkgs = append(pkgs, p)
}
sort.Strings(pkgs)
for _, p := range pkgs {
k, found, err := st.j.Kept.Lookup(ctx, level, d.Packages[p])
if err != nil {
st.j.logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err)
}
if found {
a.kept += k.Tests
continue
}
plan.todo = append(plan.todo, p)
}
return nil
}
func (st *state) runGo(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict {
if v := st.planGo(ctx, t, level); v != nil {
return v
}
plan := t.plans[level]
var run []string
switch {
case plan.all:
if priority {
return nil // nothing to put first: the level runs whole after
}
default:
for _, p := range plan.todo {
if changesTestOf(t.changed, p) == priority {
run = append(run, p)
}
}
if len(run) == 0 {
return nil
}
}
args := map[string]any{"workspace": t.fetched.Workspace, "level": level}
if run != nil {
args["packages"] = run
}
raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), args)
if err != nil {
return fail(t.node, fmt.Sprintf("the %s level of %s#%d: %v", level, t.pull.Repository, t.pull.Number, err))
}
var res Tested
if err := json.Unmarshal(raw, &res); err != nil {
return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err))
}
a := st.levels[level]
a.run += res.Tests
a.skipped += res.Skipped
a.elapsed += res.Elapsed
a.setup += res.Setup
a.over = a.over || res.OverBound
a.slowest = append(a.slowest, res.Slowest...)
if len(plan.digests) > 0 {
if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, plan.digests, st.j.now()); err != nil {
st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err)
}
}
if !res.Passed {
return levelFailure(t, level, res)
}
return nil
}
func (st *state) runNode(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict {
for _, dir := range t.fetched.Modules {
if changesTestIn(t.changed, dir) != priority {
continue
}
raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d (%s) on %s", level, t.pull.Repository, t.pull.Number, dir, t.node),
map[string]any{"workspace": t.fetched.Workspace, "module": dir, "level": level})
if err != nil {
return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s): %v", level, t.pull.Repository, t.pull.Number, dir, err))
}
var res Tested
if err := json.Unmarshal(raw, &res); err != nil {
return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s) answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, dir, err))
}
if res.Module == "" {
res.Module = dir
}
a := st.levels[level]
if !res.NoTests {
a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10)
}
a.elapsed += res.Elapsed
a.setup += res.Setup
a.over = a.over || res.OverBound
if !res.Passed {
return levelFailure(t, level, res)
}
}
return nil
}
modules/mesh-delivery-checker/internal/check/kept.go
+90
View File
@@ -0,0 +1,90 @@
package check
import (
"context"
"errors"
"fmt"
"sort"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Kept is the checker's bucket of kept verdicts (ADR 0444, ADR 0484 clause 13): a passing verdict per package and level,
// under the digest the language's build module named for it. A key is written once, by a create that fails when it
// exists, from the result the checker read from the test step and never from anything the code under test wrote; nothing
// changes it afterwards.
type Kept struct{ Store KV }
// Lookup is the kept verdict of a level under a digest, or none.
func (k Kept) Lookup(ctx context.Context, level, digest string) (delivery.Kept, bool, error) {
key, err := delivery.KeptKey(level, digest)
if err != nil {
return delivery.Kept{}, false, err
}
value, _, found, err := k.Store.Get(ctx, key)
if err != nil || !found {
return delivery.Kept{}, false, err
}
v, err := delivery.DecodeKept(value)
if err != nil {
return delivery.Kept{}, false, fmt.Errorf("%s holds what is not a kept verdict: %w", key, err)
}
return v, true, nil
}
// Keep writes v under level and digest by a create. It answers false, and writes nothing, when the key exists: the first
// value stands.
func (k Kept) Keep(ctx context.Context, level, digest string, v delivery.Kept) (bool, error) {
key, err := delivery.KeptKey(level, digest)
if err != nil {
return false, err
}
v.Level = level
data, err := delivery.EncodeKept(v)
if err != nil {
return false, err
}
if _, err := k.Store.Create(ctx, key, data); err != nil {
if errors.Is(err, ErrConflict) {
return false, nil
}
return false, fmt.Errorf("%s: %w", key, err)
}
return true, nil
}
// KeepPasses keeps, for the packages of a test result that passed with tests, a verdict under the digest the build module
// named for the package at the level. A package that failed, has no tests or has no digest is never kept (ADR 0444 clause
// 1): it runs every check. It answers the packages it kept now.
func (k Kept) KeepPasses(ctx context.Context, repository, level, toolchain string, res Tested, digests map[string]string, at time.Time) ([]string, error) {
var kept []string
for _, p := range res.Packages {
digest, ok := digests[p.Package]
if !ok || !p.Passed || p.NoTests || p.Tests == 0 {
continue
}
if failedIn(res, p.Package) {
continue
}
made, err := k.Keep(ctx, level, digest, delivery.Kept{State: delivery.VerdictPass, Repository: repository, Package: p.Package,
Level: level, Tests: p.Tests, Took: int64(p.Elapsed * 1000), Toolchain: toolchain, At: at})
if err != nil {
return kept, err
}
if made {
kept = append(kept, p.Package)
}
}
sort.Strings(kept)
return kept, nil
}
func failedIn(res Tested, pkg string) bool {
for _, f := range res.Failed {
if f.Package == pkg {
return true
}
}
return false
}
modules/mesh-delivery-checker/internal/check/kept_test.go
+102
View File
@@ -0,0 +1,102 @@
package check
import (
"context"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
var at0 = time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC)
const (
digA = "aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11aa11"
digB = "bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22bb22"
)
func keptValue(pkg string, tests int) delivery.Kept {
return delivery.Kept{State: delivery.VerdictPass, Repository: "novox/mesh-delivery", Package: pkg, Tests: tests, Took: 120, Toolchain: "go1.26.0", At: at0}
}
func TestAPassIsKeptByACreateUnderItsLevelAndDigest(t *testing.T) { // 1287/AC1
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
made, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/a", 3))
if err != nil || !made {
t.Fatalf("Keep = %v, %v, want made", made, err)
}
key, _ := delivery.KeptKey(delivery.LevelUnit, digA)
if st.creates[key] != 1 {
t.Fatalf("the key %s was created %d times, want once", key, st.creates[key])
}
got, found, err := k.Lookup(ctx, delivery.LevelUnit, digA)
if err != nil || !found || got.Package != "m/a" || got.Tests != 3 || got.Level != delivery.LevelUnit || got.Toolchain != "go1.26.0" {
t.Fatalf("Lookup = %+v, %v, %v", got, found, err)
}
}
func TestASecondKeepUnderTheSameDigestIsRefusedAndTheFirstStands(t *testing.T) { // 1287/AC2
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
if made, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/a", 3)); err != nil || !made {
t.Fatal(made, err)
}
made, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/other", 9))
if err != nil || made {
t.Fatalf("the second Keep = %v, %v, want refused without an error", made, err)
}
got, _, _ := k.Lookup(ctx, delivery.LevelUnit, digA)
if got.Package != "m/a" || got.Tests != 3 {
t.Errorf("the first value did not stand: %+v", got)
}
}
func TestOnlyPassingPackagesWithADigestAreKept(t *testing.T) { // 1287/AC3, AC4
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
res := Tested{Level: "unit", Passed: false, Packages: []GoPackage{
{Package: "m/a", Passed: true, Tests: 4, Elapsed: 0.2},
{Package: "m/b", Passed: false, Tests: 2},
{Package: "m/c", Passed: true, Tests: 1}, // no digest named
{Package: "m/d", Passed: true, NoTests: true}, // no tests
}, Failed: []Failure{{Package: "m/b", Test: "TestB"}}}
kept, err := k.KeepPasses(ctx, "novox/mesh-delivery", "unit", "go1.26.0", res, map[string]string{"m/a": digA, "m/b": digB, "m/d": digB}, at0)
if err != nil {
t.Fatal(err)
}
if strings.Join(kept, ",") != "m/a" {
t.Fatalf("kept %v, want only m/a", kept)
}
if keys, _ := st.Keys(ctx); len(keys) != 1 {
t.Errorf("the bucket holds %v, want one key", keys)
}
}
func TestAKeptValueOfAnotherDigestIsNotRead(t *testing.T) { // 1287/AC7
ctx, st := context.Background(), newMemKV()
k := Kept{Store: st}
if _, err := k.Keep(ctx, delivery.LevelUnit, digA, keptValue("m/a", 3)); err != nil {
t.Fatal(err)
}
// A toolchain change changes the digest, so the new digest finds nothing and the old key is not deleted.
if _, found, _ := k.Lookup(ctx, delivery.LevelUnit, digB); found {
t.Error("a digest that was not kept was found")
}
if _, found, _ := k.Lookup(ctx, delivery.LevelIntegration, digA); found {
t.Error("a kept unit verdict was found at the integration level")
}
if keys, _ := st.Keys(ctx); len(keys) != 1 {
t.Errorf("the old key was removed: %v", keys)
}
}
func TestAKeptVerdictThatIsNotAPassIsRefused(t *testing.T) {
k := Kept{Store: newMemKV()}
v := keptValue("m/a", 1)
v.State = delivery.VerdictFail
if _, err := k.Keep(context.Background(), delivery.LevelUnit, digA, v); err == nil {
t.Error("a failure was kept")
}
}
modules/mesh-delivery-checker/internal/check/line.go
+214
View File
@@ -0,0 +1,214 @@
package check
import (
"fmt"
"strings"
"unicode/utf8"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// MaxLine is the most characters a status's line has: the forge shows a status's description in a short space, and the full
// report is linked (ADR 0447 clause 3, ADR 0484 clause 7). A line that would be longer drops its lesser parts in a fixed
// order, and is cut last.
const MaxLine = 140
// seconds is a time in milliseconds in the unit a person reads: 350ms, 4.1s, 72s.
func seconds(ms int64) string {
switch {
case ms < 1000:
return fmt.Sprintf("%dms", ms)
case ms < 10000:
return fmt.Sprintf("%.1fs", float64(ms)/1000)
}
return fmt.Sprintf("%ds", (ms+500)/1000)
}
func short12(s string) string {
if len(s) > 12 {
return s[:12]
}
return s
}
// firstFailed is what failed first, in words: a conflict, a refused step or a test, named before anything else.
func firstFailed(v delivery.Verdict) string {
f := v.Failed
switch {
case f.Test != "":
if f.Words != "" {
return f.Test + ": " + f.Words
}
return f.Test
case f.Machine != "" && f.Words != "":
return f.Machine + ": " + f.Words
case f.Machine != "":
return f.Machine
}
return f.Words
}
// pieces are the parts of a line, from the most to the least needed.
type pieces struct {
head, commits, counts, shortCounts, times, shortTimes, over, builds, composition, group string
}
func (p pieces) join(drop int) string {
parts := []string{p.head, p.commits, p.counts}
switch {
case drop < 3 && p.times != "":
parts = append(parts, p.times)
case drop == 3 && p.shortTimes != "":
parts = []string{p.head, p.commits, p.shortCounts, p.shortTimes}
}
if p.over != "" {
parts = append(parts, p.over)
}
if drop < 1 && p.builds != "" {
parts = append(parts, p.builds)
}
if drop < 2 && p.group != "" {
parts = append(parts, p.group)
}
if p.composition != "" {
parts = append(parts, p.composition)
}
out := make([]string, 0, len(parts))
for _, s := range parts {
if s != "" {
out = append(out, s)
}
}
return strings.Join(out, " | ")
}
func levelWords(name string, r delivery.LevelResult) string {
s := fmt.Sprintf("%s %d run, %d kept", name, r.Run, r.Kept)
if r.Skipped > 0 {
s += fmt.Sprintf(", %d skipped", r.Skipped)
}
return s
}
func compositionWords(c *delivery.Composition) string {
switch {
case c == nil, c.State == delivery.CompositionNotRun:
return "composition not yet run"
case c.State == delivery.CompositionPass:
return "composition passed with " + short12(c.SDK)
}
return "composition failed: " + c.Words
}
func buildWords(builds []BuildNote) string {
if len(builds) == 0 {
return ""
}
parts := make([]string, len(builds))
for i, b := range builds {
parts[i] = b.Module + " " + b.State
}
return "builds: " + strings.Join(parts, ", ")
}
// parts makes the pieces of the line of a pull request of a case.
func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) pieces {
var pc pieces
commits := short12(p.Head) + " on " + short12(p.Trunk)
switch v.State {
case delivery.VerdictRunning:
pc.head = "checking " + commits
case delivery.VerdictPass:
pc.head = "pass " + commits
default:
pc.head = "failed: " + firstFailed(v)
pc.commits = commits
}
var counts []string
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
if r, ok := v.Levels[level]; ok {
counts = append(counts, levelWords(level, r))
}
}
pc.counts = strings.Join(counts, " | ")
var short []string
for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} {
if r, ok := v.Levels[level]; ok {
name := map[string]string{delivery.LevelUnit: "unit", delivery.LevelIntegration: "int"}[level]
w := fmt.Sprintf("%s %d run %d kept", name, r.Run, r.Kept)
if r.Skipped > 0 {
w += fmt.Sprintf(" %d skipped", r.Skipped)
}
short = append(short, w)
}
}
pc.shortCounts = strings.Join(short, " | ")
if v.State != delivery.VerdictRunning {
t := v.Took
var times []string
for _, x := range []struct {
name string
ms int64
}{{"queue", t.Queue}, {"clone", t.Clone}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}, {"composition", t.Composition}} {
if x.ms > 0 {
times = append(times, x.name+" "+seconds(x.ms))
}
}
if len(times) > 0 {
pc.times = strings.Join(times, ", ")
}
// The compact form for a line that is too long: the parts that dominate, the integration level's setup and run as one.
short = short[:0:0]
if t.Clone > 0 {
short = append(short, "clone "+seconds(t.Clone))
}
if t.Unit > 0 {
short = append(short, "unit "+seconds(t.Unit))
}
switch {
case t.IntegrationSetup > 0 && t.IntegrationRun > 0:
short = append(short, "int "+seconds(t.IntegrationSetup)+"+"+seconds(t.IntegrationRun))
case t.IntegrationRun > 0:
short = append(short, "int "+seconds(t.IntegrationRun))
}
pc.shortTimes = strings.Join(short, " ")
}
if len(v.OverBound) > 0 {
pc.over = "over bound: " + strings.Join(v.OverBound, ", ")
if len(slowest) > 0 {
pc.over += " (slowest " + strings.Join(slowest, ", ") + ")"
}
}
pc.builds = buildWords(builds)
if v.State != delivery.VerdictRunning {
pc.composition = compositionWords(v.Composition)
}
if cs != nil && len(cs.Pulls) > 1 {
pc.group = fmt.Sprintf("group %s of %d pull requests", cs.Group, len(cs.Pulls))
if v.State != delivery.VerdictPass {
pc.group = fmt.Sprintf("%s#%d in group %s", p.Repository, p.Number, cs.Group)
}
}
return pc
}
// Report is the full line of a pull request's status, with no limit: what the read tools and the full report say.
func Report(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string {
return parts(v, cs, p, builds, slowest).join(0)
}
// Line is the words of a status for one pull request of a case (ADR 0447 clause 3, ADR 0484 clause 7): the verdict first (a
// failure names the conflict, the refused step or the test before anything), the head and the trunk commit as twelve
// characters each, the tests run, kept and skipped per level, the time of each part, a level over its bound with its slowest
// tests, the builds of the touched modules, and that the composition was not yet run. It is at most MaxLine characters: the
// builds, then the group drop first, then the times are given in their compact form, then dropped, and the line is cut at last.
func Line(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []string) string {
pc := parts(v, cs, p, builds, slowest)
for drop := 0; drop <= 4; drop++ {
if s := pc.join(drop); utf8.RuneCountInString(s) <= MaxLine {
return s
}
}
runes := []rune(pc.join(4))
return string(runes[:MaxLine-1]) + "…"
}
modules/mesh-delivery-checker/internal/check/line_test.go
+135
View File
@@ -0,0 +1,135 @@
package check
import (
"strings"
"testing"
"unicode/utf8"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
func lineCase(t testing.TB, pulls ...delivery.CandidatePull) (*Case, Pull) {
cs := caseOf(t, pulls...)
return cs, cs.Pulls[0]
}
func passVerdict() delivery.Verdict {
return delivery.Verdict{State: delivery.VerdictPass,
Levels: map[string]delivery.LevelResult{delivery.LevelUnit: {Run: 12, Kept: 301}, delivery.LevelIntegration: {Run: 3, Kept: 40}},
Took: delivery.Took{Queue: 800, Clone: 4100, Unit: 400, IntegrationSetup: 3200, IntegrationRun: 12000},
Composition: &delivery.Composition{State: delivery.CompositionNotRun}}
}
func TestAPassOpensWithTheVerdictAndSaysTheCommitsTheCountsTheTimesAndThatTheCompositionWasNotRun(t *testing.T) { // 1300/AC1
cs, p := lineCase(t, pullOf(repoA, 7, head1))
full := Report(passVerdict(), cs, p, nil, nil)
for _, want := range []string{"pass 111111111111 on aaaaaaaaaaaa", "unit 12 run, 301 kept", "integration 3 run, 40 kept", "queue 800ms", "clone 4.1s", "unit 400ms", "int setup 3.2s", "int run 12s", "composition not yet run"} {
if !strings.Contains(full, want) {
t.Errorf("the report %q lacks %q", full, want)
}
}
if !strings.HasPrefix(full, "pass ") || strings.Contains(full, "composition passed") {
t.Errorf("report = %q", full)
}
line := Line(passVerdict(), cs, p, nil, nil)
if n := utf8.RuneCountInString(line); n > MaxLine {
t.Errorf("the line is %d characters, want at most %d: %q", n, MaxLine, line)
}
if !strings.HasPrefix(line, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(line, "composition not yet run") || !strings.Contains(line, "unit 12 run 301 kept") ||
!strings.Contains(line, "clone 4.1s") || !strings.Contains(line, "int 3.2s+12s") {
t.Errorf("line = %q", line)
}
}
func TestAFailureNamesTheTestTheConflictOrTheRefusedStepFirst(t *testing.T) { // 1300/AC2
cs, p := lineCase(t, pullOf(repoA, 7, head1))
for name, c := range map[string]struct {
v delivery.Verdict
want string
}{
"test": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "unit: m/a.TestX", Words: "want 4"}}, "failed: unit: m/a.TestX: want 4"},
"conflict": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "shanks", Words: "novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"}}, "failed: shanks: novox/mesh-delivery#7 does not merge into the trunk: files in conflict: go.mod"},
"refused": {delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Machine: "mesh-build-go", Words: "no go build node qualifies: builders disagree"}}, "failed: mesh-build-go: no go build node qualifies"},
} {
line := Line(c.v, cs, p, nil, nil)
if !strings.HasPrefix(line, c.want) || !strings.Contains(line, "111111111111 on aaaaaaaaaaaa") {
t.Errorf("%s: line = %q, want it to open with %q and name the commits", name, line, c.want)
}
}
}
func TestALevelOverItsBoundIsNamedWithItsSlowestTestsAndTheVerdictIsNotChanged(t *testing.T) { // 1300/AC3
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.OverBound = []string{"unit"}
line := Line(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"})
if !strings.HasPrefix(line, "pass ") || !strings.Contains(Report(v, cs, p, nil, []string{"unit pkg.TestSlow 7.0s"}), "over bound: unit (slowest unit pkg.TestSlow 7.0s)") {
t.Errorf("line = %q", line)
}
}
func TestSkippedTestsAreSaidAndNeverAFlatPass(t *testing.T) { // 1300/AC6
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.Levels[delivery.LevelUnit] = delivery.LevelResult{Run: 12, Kept: 301, Skipped: 2}
if line := Line(v, cs, p, nil, nil); !strings.Contains(line, "unit 12 run, 301 kept, 2 skipped") {
t.Errorf("line = %q", line)
}
}
func TestAGroupsMembersEachCarryTheSameVerdict(t *testing.T) { // 1300/AC4
cs := caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))
a := Line(passVerdict(), cs, cs.Pulls[0], nil, nil)
b := Line(passVerdict(), cs, cs.Pulls[1], nil, nil)
if !strings.HasPrefix(a, "pass ") || !strings.HasPrefix(b, "pass ") || !strings.Contains(Report(passVerdict(), cs, cs.Pulls[0], nil, nil), "group feat/x of 2 pull requests") {
t.Errorf("lines %q and %q", a, b)
}
if !strings.Contains(a, "111111111111") || !strings.Contains(b, "222222222222") {
t.Errorf("each member's line names its own head: %q and %q", a, b)
}
}
func TestTheBuildsOfTheTouchedModulesAreSaidAndDroppedFirstWhenTheLineIsTooLong(t *testing.T) { // ADR 0484 clause 7
cs, p := lineCase(t, pullOf(repoA, 7, head1))
builds := []BuildNote{{"avahi", "built"}, {"mesh-tools", "waiting"}}
if r := Report(passVerdict(), cs, p, builds, nil); !strings.Contains(r, "builds: avahi built, mesh-tools waiting") {
t.Errorf("report = %q", r)
}
line := Line(passVerdict(), cs, p, builds, nil)
if utf8.RuneCountInString(line) > MaxLine {
t.Errorf("line %q is too long", line)
}
short := passVerdict()
short.Took = delivery.Took{}
if l := Line(short, cs, p, builds[:1], nil); !strings.Contains(l, "builds: avahi built") {
t.Errorf("a short line keeps its builds: %q", l)
}
}
func TestALineIsNeverLongerThanTheLimit(t *testing.T) { // 1300/AC1
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := delivery.Verdict{State: delivery.VerdictFail, Failed: delivery.FailedFirst{Test: "integration: " + strings.Repeat("pkg/", 50) + ".TestSomething", Words: strings.Repeat("because ", 40)}}
if line := Line(v, cs, p, nil, nil); utf8.RuneCountInString(line) > MaxLine || !strings.HasPrefix(line, "failed: integration: ") {
t.Errorf("line = %q (%d)", line, utf8.RuneCountInString(line))
}
}
func TestARunningCheckIsPending(t *testing.T) {
cs, p := lineCase(t, pullOf(repoA, 7, head1))
if line := Line(delivery.Verdict{State: delivery.VerdictRunning}, cs, p, nil, nil); line != "checking 111111111111 on aaaaaaaaaaaa" {
t.Errorf("line = %q", line)
}
}
func TestThePassedCompositionIsSaidOnlyWhenItWasRun(t *testing.T) { // ADR 0484 clause 6
cs, p := lineCase(t, pullOf(repoA, 7, head1))
v := passVerdict()
v.Composition = &delivery.Composition{State: delivery.CompositionPass, SDK: "afbe371003a570f088be4a394d83aa7a9458f7ab"}
if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition passed with afbe371003a5") {
t.Errorf("report = %q", r)
}
v.Composition = nil
if r := Report(v, cs, p, nil, nil); !strings.Contains(r, "composition not yet run") || strings.Contains(r, "composition passed") {
t.Errorf("a verdict with no composition: %q", r)
}
}
modules/mesh-delivery-checker/internal/check/pins.go
+106
View File
@@ -0,0 +1,106 @@
package check
import (
"context"
"fmt"
"path"
"regexp"
"strings"
)
// The pin of mesh-sdk in a Go member's go.mod (novox/hq ADR 0484 clause 9, ADR 0482 clause 2, issue 1624): a delivery group
// has one verdict, and the mesh-sdk its Go members are built and tested with is the group's: a pin to the head of the group's
// mesh-sdk pull request, or to a commit already on mesh-sdk's trunk. Any other commit is a stale or a foreign pin, and the
// group fails naming the member and the commit.
// SDKRepository is the repository of mesh-sdk, and SDKModulePath its Go module.
const (
SDKRepository = "novox/mesh-sdk"
SDKModulePath = "git.novox.be/novox/mesh-sdk/go"
)
// Issue1148 is why a pin that only a head carries fails the check.
const Issue1148 = "issue 1148"
// Forge reads a file of a repository at a ref, and says whether a commit is on a repository's trunk.
type Forge interface {
// File is the text of a file at a ref (a branch, a tag or a commit).
File(ctx context.Context, repository, path, ref string) (string, error)
// OnTrunk says whether commit (a full or a 12-digit sha) is already merged into the trunk of the repository: it holds no
// commit the trunk does not.
OnTrunk(ctx context.Context, repository, commit string) (bool, error)
// FullSHA is the 40-digit sha of a commit named by its first 12 digits.
FullSHA(ctx context.Context, repository, commit string) (string, error)
}
var pseudoVersion = regexp.MustCompile(`-([0-9a-f]{12})$`)
// SDKPin is the commit (12 hex digits) a go.mod names for mesh-sdk by a pseudo-version, or none for a go.mod that does not
// require mesh-sdk or requires a version that is no pseudo-version (a tag is a release the SDK itself names).
func SDKPin(goMod string) (commit string, ok bool) {
for _, line := range strings.Split(goMod, "\n") {
f := strings.Fields(strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "require")))
if len(f) >= 2 && f[0] == SDKModulePath {
if m := pseudoVersion.FindStringSubmatch(f[1]); m != nil {
return m[1], true
}
return "", false
}
}
return "", false
}
// checkPins refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the
// group's mesh-sdk pull request. It reads each go.mod the fetch named at the member's head from the forge. It answers nil
// when every pin is the group's.
func (st *state) checkPins(ctx context.Context) verdictPtr {
if st.j.Forge == nil {
return nil
}
sdk := st.c.member(SDKRepository)
for _, t := range st.targets {
if t.module != goModule || t.pull.Repository == SDKRepository {
continue
}
for _, dir := range t.fetched.Modules {
file := path.Join(dir, "go.mod")
st.c.Progress.Set(fmt.Sprintf("read %s of %s#%d", file, t.pull.Repository, t.pull.Number), t.node)
text, err := st.j.Forge.File(ctx, t.pull.Repository, file, t.pull.Head)
if err != nil {
return failed(t.node, fmt.Sprintf("%s#%d: %s cannot be read to check its mesh-sdk pin: %v", t.pull.Repository, t.pull.Number, file, err))
}
pin, ok := SDKPin(text)
if !ok {
continue
}
if sdk != nil && strings.HasPrefix(sdk.Head, pin) {
continue // the group's own head
}
onTrunk, err := st.j.Forge.OnTrunk(ctx, SDKRepository, pin)
if err != nil {
return failed(t.node, fmt.Sprintf("%s#%d: whether mesh-sdk %s is on its trunk cannot be told: %v", t.pull.Repository, t.pull.Number, pin, err))
}
if onTrunk {
continue
}
nor := "the group has no mesh-sdk pull request"
if sdk != nil {
nor = "the head of the group's mesh-sdk pull request is " + short12(sdk.Head)
}
return failed(t.node, fmt.Sprintf("%s#%d: %s names mesh-sdk %s, which is neither on mesh-sdk's trunk nor the group's head (%s)", t.pull.Repository, t.pull.Number, file, pin, nor))
}
}
return nil
}
// failureVerdict is a verdict of failure the check ends with.
// pinHint is added to the failure of a Go member's fetch that names mesh-sdk while the group holds a mesh-sdk pull request:
// a pin to that pull request's head names a commit only the head carries, which go mod download cannot resolve before the
// pull request is merged (issue 1148), so such a check fails and never passes.
func pinHint(c *Case, fetchError string) string {
if c.member(SDKRepository) == nil || !strings.Contains(fetchError, "mesh-sdk") {
return ""
}
return " (a pin to the head of the group's mesh-sdk pull request names a commit only that head carries, which go mod download cannot resolve; " + Issue1148 + ")"
}
modules/mesh-delivery-checker/internal/check/pins_test.go
+204
View File
@@ -0,0 +1,204 @@
package check
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// fakeForge answers the forge's two reads from files and commits it is given.
type fakeForge struct {
goMods map[string]string // `<repository> <path> <ref>` -> text
onTrunk map[string]bool // by commit prefix
err error
}
func (f *fakeForge) File(_ context.Context, repository, path, ref string) (string, error) {
if f.err != nil {
return "", f.err
}
if s, ok := f.goMods[repository+" "+path+" "+ref]; ok {
return s, nil
}
return "", fmt.Errorf("no file %s of %s at %s", path, repository, ref)
}
func (f *fakeForge) FullSHA(_ context.Context, _ string, commit string) (string, error) {
if f.err != nil {
return "", f.err
}
return commit + strings.Repeat("0", 40-len(commit)), nil
}
func (f *fakeForge) OnTrunk(_ context.Context, _ string, commit string) (bool, error) {
if f.err != nil {
return false, f.err
}
return f.onTrunk[commit[:12]], nil
}
// capturedGoMod is the go.mod the forge answered (testdata/captured/forge-get-file-go-mod.json), pinned to the commit given.
func capturedGoMod(t *testing.T, pin string) string {
t.Helper()
raw, err := os.ReadFile(filepath.Join("testdata", "captured", "forge-get-file-go-mod.json"))
if err != nil {
t.Fatal(err)
}
var a struct{ File struct{ Content string } }
if err := json.Unmarshal(raw, &a); err != nil {
t.Fatal(err)
}
return strings.Replace(a.File.Content, "cb54756c3865", pin, 1)
}
func TestTheSDKPinIsTheCommitOfAPseudoVersion(t *testing.T) { // 1624
if c, ok := SDKPin(capturedGoMod(t, "cb54756c3865")); !ok || c != "cb54756c3865" {
t.Errorf("pin = %q, %v", c, ok)
}
if _, ok := SDKPin("module x\n\nrequire git.novox.be/novox/mesh-sdk/go v0.2.0\n"); ok {
t.Error("a tagged version is no commit")
}
if c, ok := SDKPin("module x\n\nrequire git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865\n"); !ok || c != "cb54756c3865" {
t.Errorf("a one-line require: %q, %v", c, ok)
}
if _, ok := SDKPin("module x\n\ngo 1.26\n"); ok {
t.Error("a go.mod that does not require mesh-sdk has a pin")
}
}
// groupBench is a group of mesh-delivery#7 and the SDK's pull request 12, mesh-delivery's go.mod at its head naming pin.
func groupBench(t *testing.T, pin string, trunk ...string) *bench {
b := newBench(t)
b.fetch.Modules = []string{"modules/m"}
forge := &fakeForge{goMods: map[string]string{repoA + " modules/m/go.mod " + head1: capturedGoMod(t, pin)}, onTrunk: map[string]bool{}}
for _, c := range trunk {
forge.onTrunk[c] = true
}
b.ex.Forge = forge
return b
}
func TestAMemberPinnedToTheSDKMembersHeadPassesAndIsBuiltAgainstIt(t *testing.T) { // 1624/AC2, AC4
b := groupBench(t, head2[:12])
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
var besideSDK bool
for _, f := range b.steps.tool("fetch") {
if f.args["repository"] == repoA {
refs, _ := f.args["beside"].([]BesideRef)
for _, r := range refs {
besideSDK = besideSDK || (r.Repository == repoSDK && r.Commit == head2)
}
}
}
if !besideSDK {
t.Error("mesh-delivery's tests were not given the SDK member at its head")
}
}
func TestAMemberPinnedToAnOlderHeadOfTheSDKMemberFailsNamingTheMemberAndTheCommit(t *testing.T) { // 1624/AC2, AC4
b := groupBench(t, "9aaa5c80ac5c") // a commit of the SDK branch, not its head, not on the trunk
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
f := v.Failed
if v.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, repoA+"#7") || !strings.Contains(f.Words, "9aaa5c80ac5c") ||
!strings.Contains(f.Words, "neither on mesh-sdk's trunk nor the group's head") || !strings.Contains(f.Words, short12(head2)) {
t.Fatalf("verdict = %+v", v)
}
if n := len(b.steps.tool("test")); n != 0 {
t.Errorf("%d test steps ran after a stale pin", n)
}
}
func TestAMemberPinnedToACommitOnTheTrunkPasses(t *testing.T) { // 1624/AC2, AC4
b := groupBench(t, "cb54756c3865", "cb54756c3865")
if v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict; v.State != delivery.VerdictPass {
t.Fatalf("verdict = %+v", v)
}
// A lone pull request of a Go member is held to the trunk alone.
b = groupBench(t, "9aaa5c80ac5c")
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the group has no mesh-sdk pull request") {
t.Errorf("a foreign pin with no SDK member: %+v", v)
}
}
func TestAPinOnlyTheSDKHeadCarriesThatDownloadCannotResolveFailsNamingIssue1148(t *testing.T) { // 1624/AC5, ADR 0453 clause 12
b := groupBench(t, head2[:12])
b.steps.handlers["build_go_fetch"] = func(c stepCall) (any, error) {
if c.args["repository"] == repoA {
return nil, fmt.Errorf("go mod download: git.novox.be/novox/mesh-sdk/go@v0.1.14-0.20261021100209-%s: invalid version: unknown revision", head2[:12])
}
return b.fetch, nil
}
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "issue 1148") || !strings.Contains(v.Failed.Words, head2[:12]) {
t.Errorf("verdict = %+v, want a failure naming issue 1148 and the commit", v)
}
}
func TestAPinThatCannotBeCheckedFailsAndNeverPasses(t *testing.T) { // ADR 0453 clause 12
b := groupBench(t, "cb54756c3865")
b.ex.Forge.(*fakeForge).err = fmt.Errorf("the forge is down")
v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "the forge is down") {
t.Errorf("verdict = %+v", v)
}
}
type runsAt map[string]string
func (r runsAt) CommitOf(module string) string { return r[module] }
func besideOf(t *testing.T, b *bench, repository string) map[string]BesideRef {
t.Helper()
out := map[string]BesideRef{}
for _, f := range b.steps.tool("fetch") {
if f.args["repository"] == repository {
refs, _ := f.args["beside"].([]BesideRef)
for _, r := range refs {
out[r.Repository] = r
}
}
}
return out
}
func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR 0484 clause 4
const controller, catalogue, host = "novox/mesh-controller", "novox/mesh-catalog", "novox/mesh-host"
hostRuns := strings.Repeat("7", 40)
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"})
b.ex.Runs = runsAt{"mesh-host": hostRuns}
forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865")}, onTrunk: map[string]bool{"cb54756c3865": true}}
b.ex.Forge = forge
// A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head.
b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2)))
got := besideOf(t, b, controller)
if c := got[catalogue]; c.Commit != trunkS || c.Ref != "" {
t.Errorf("the catalogue beside the controller = %+v, want its trunk %s", c, trunkS)
}
if h := got[host]; h.Commit != hostRuns {
t.Errorf("the node-engine beside the controller = %+v, want the commit the mesh runs %s", h, hostRuns)
}
if s := got[SDKRepository]; s.Commit != "cb54756c3865"+strings.Repeat("0", 28) {
t.Errorf("mesh-sdk beside the controller = %+v, want the commit its go.mod pins, whole", s)
}
}
func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2
b := newBench(t)
b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA})
b.ex.Forge = &fakeForge{goMods: map[string]string{}, onTrunk: map[string]bool{}}
v := b.judge(t, caseOf(t, pullOf("novox/mesh-controller", 7, head1))).Verdict
if v.State != delivery.VerdictFail || !strings.Contains(v.Failed.Words, "go.mod") {
t.Errorf("verdict = %+v", v)
}
}
modules/mesh-delivery-checker/internal/check/preview.go
+86
View File
@@ -0,0 +1,86 @@
package check
import (
"context"
"errors"
"fmt"
"regexp"
"slices"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
var digest64 = regexp.MustCompile(`^[0-9a-f]{64}$`)
// PreviewOf is what the fetch steps said of a pull request's head merged onto its trunk head, as the value of a previews
// key (ADR 0484 clause 3): the merge commit and, for each module directory the fetches named with a manifest, its tree, the
// digest of its manifest and the hashes of its lock files. The fetches of the repository's build modules merge the same
// trees; the first one's merge commit is kept. A module directory with no tree or no manifest digest is left out, so the
// indexer gives its module no fingerprint and says why. A head that does not merge has a preview of the files in conflict
// alone, so the indexer says the candidate's modules have no fingerprint because it does not merge (issue 1445). It
// answers false when no fetch was made.
func PreviewOf(p Pull, fetched []Fetched) (delivery.Preview, bool) {
var merge string
var conflicts []string
mods := map[string]delivery.PreviewModule{}
for _, f := range fetched {
if len(f.Conflicts) > 0 {
conflicts = append(conflicts, f.Conflicts...)
}
}
if len(conflicts) > 0 {
slices.Sort(conflicts)
return delivery.Preview{Repository: p.Repository, Head: p.Head, Trunk: p.Trunk, Conflicts: slices.Compact(conflicts)}, true
}
for _, f := range fetched {
if f.Merge == "" {
continue
}
if merge == "" {
merge = f.Merge
}
for dir, t := range f.Trees {
if t.Tree == "" || !digest64.MatchString(t.Manifest) {
continue
}
mods[dir] = delivery.PreviewModule{Tree: t.Tree, Manifest: t.Manifest, Locks: t.Locks}
}
}
if merge == "" {
return delivery.Preview{}, false
}
pv := delivery.Preview{Repository: p.Repository, Head: p.Head, Trunk: p.Trunk, Merge: merge}
if len(mods) > 0 {
pv.Modules = mods
}
return pv, true
}
// writePreview writes the preview of a pull request by a create: one writer per key, and the same head on the same trunk
// head merges to the same trees, so a key that exists stands. A head that did not merge has no preview. A failure is said
// and does not fail the check: the preview is for the indexer's fingerprints, not for the verdict.
func (j *Examiner) writePreview(ctx context.Context, p Pull, fetched []Fetched) {
if j.Previews == nil {
return
}
pv, ok := PreviewOf(p, fetched)
if !ok {
return
}
key, err := delivery.PreviewKey(p.Repository, p.Head, p.Trunk)
if err == nil {
var data []byte
if data, err = delivery.EncodePreview(pv); err == nil {
_, err = j.Previews.Create(ctx, key, data)
if errors.Is(err, ErrConflict) {
return
}
if err == nil && j.Announce != nil {
j.Announce(ctx, EventPreviewWritten, key)
}
}
}
if err != nil {
j.logf("%s#%d: the preview cannot be written: %v", p.Repository, p.Number, fmt.Errorf("%w", err))
}
}
modules/mesh-delivery-checker/internal/check/read.go
+193
View File
@@ -0,0 +1,193 @@
package check
import (
"encoding/json"
"fmt"
"sort"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the
// status it owes in full, so a check is read live without a shell. They write nothing.
// The read tools, named as the module is: checker_ and the noun.
const (
VerdictsReadTool = "checker_verdicts"
PullReadTool = "checker_pull"
KeptReadTool = "checker_kept"
)
// StateRead reads one state: its keys and a key's stored value (nil for a key with none).
type StateRead struct {
Keys func() ([]string, error)
Get func(key string) (json.RawMessage, error)
}
// Reader answers the read tools from the checker's verdicts and kept states and the indexer's candidates.
type Reader struct{ VerdictState, KeptState, CandidateState StateRead }
// VerdictRead is a verdict as stored with the full report of the status of each pull request it judged (Report), when the
// candidate still holds those heads; without it the report is that of the verdict alone.
type VerdictRead struct {
Key string `json:"key"`
Verdict json.RawMessage `json:"verdict"`
Reports map[string]string `json:"reports,omitempty"`
Note string `json:"note,omitempty"`
}
func (r Reader) candidate(group string) (delivery.Candidate, bool) {
key, err := delivery.CandidatesKey(group)
if err != nil || r.CandidateState.Get == nil {
return delivery.Candidate{}, false
}
raw, err := r.CandidateState.Get(key)
if err != nil || len(raw) == 0 {
return delivery.Candidate{}, false
}
c, err := delivery.DecodeCandidate(raw)
return c, err == nil
}
func (r Reader) read(key string, raw json.RawMessage) (VerdictRead, error) {
out := VerdictRead{Key: key, Verdict: raw}
v, err := delivery.DecodeVerdict(raw)
if err != nil {
return out, fmt.Errorf("%s holds what is not a verdict: %w", key, err)
}
c, ok := r.candidate(v.Group)
if !ok || c.HeadsHash != v.Heads {
out.Note = "the candidate no longer holds these heads"
return out, nil
}
cs := CaseOf(c, v.Trunk)
out.Reports = map[string]string{}
if cs == nil {
return out, nil
}
for _, p := range cs.Pulls {
out.Reports[fmt.Sprintf("%s#%d", p.Repository, p.Number)] = Report(v, cs, p, nil, nil)
}
for _, l := range cs.Left {
out.Reports[strings.SplitN(l, ": ", 2)[0]] = strings.SplitN(l, ": ", 2)[1]
}
return out, nil
}
// Verdicts are the verdicts the checker holds, by key; with a group only that group's.
func (r Reader) Verdicts(group string) ([]VerdictRead, error) {
if r.VerdictState.Keys == nil || r.VerdictState.Get == nil {
return nil, fmt.Errorf("the verdicts state cannot be read here")
}
keys, err := r.VerdictState.Keys()
if err != nil {
return nil, err
}
sort.Strings(keys)
var out []VerdictRead
for _, k := range keys {
if group != "" {
if g, _, err := delivery.ParseCheckerVerdictKey(k); err != nil || g != group {
continue
}
}
raw, err := r.VerdictState.Get(k)
if err != nil {
return nil, err
}
if len(raw) == 0 {
continue
}
vr, err := r.read(k, raw)
if err != nil {
return nil, err
}
out = append(out, vr)
}
return out, nil
}
// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request,
// that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does.
func (r Reader) Pull(repository string, number int) (VerdictRead, error) {
if r.CandidateState.Keys == nil || r.CandidateState.Get == nil {
return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here")
}
keys, err := r.CandidateState.Keys()
if err != nil {
return VerdictRead{}, err
}
sort.Strings(keys)
for _, k := range keys {
raw, err := r.CandidateState.Get(k)
if err != nil || len(raw) == 0 {
continue
}
c, err := delivery.DecodeCandidate(raw)
if err != nil {
continue
}
for _, p := range c.Pulls {
if p.Repository != repository || p.Number != number {
continue
}
if repository == HQRepository {
return VerdictRead{Key: k, Note: HQNote}, nil
}
vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash)
if err != nil {
return VerdictRead{}, err
}
vraw, err := r.VerdictState.Get(vkey)
if err != nil {
return VerdictRead{}, err
}
if len(vraw) == 0 {
return VerdictRead{Key: vkey, Note: "no verdict yet for the heads of its candidate"}, nil
}
return r.read(vkey, vraw)
}
}
return VerdictRead{}, fmt.Errorf("no candidate holds %s#%d", repository, number)
}
// Kept is the kept verdicts: one key's value as stored, or with an empty key the keys of a level (or every level) and their values.
func (r Reader) Kept(level, digest string) (map[string]json.RawMessage, error) {
if r.KeptState.Keys == nil || r.KeptState.Get == nil {
return nil, fmt.Errorf("the kept state cannot be read here")
}
if digest != "" {
key, err := delivery.KeptKey(level, digest)
if err != nil {
return nil, err
}
raw, err := r.KeptState.Get(key)
if err != nil {
return nil, err
}
if len(raw) == 0 {
return nil, fmt.Errorf("no kept verdict %s", key)
}
return map[string]json.RawMessage{key: raw}, nil
}
keys, err := r.KeptState.Keys()
if err != nil {
return nil, err
}
sort.Strings(keys)
out := map[string]json.RawMessage{}
for _, k := range keys {
if l, _, err := delivery.ParseKeptKey(k); err != nil || (level != "" && l != level) {
continue
}
raw, err := r.KeptState.Get(k)
if err != nil {
return nil, err
}
if len(raw) > 0 {
out[k] = raw
}
}
return out, nil
}
modules/mesh-delivery-checker/internal/check/read_test.go
+104
View File
@@ -0,0 +1,104 @@
package check
import (
"context"
"encoding/json"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
type fakeState map[string][]byte
func (f fakeState) read() StateRead {
return StateRead{
Keys: func() ([]string, error) {
var k []string
for key := range f {
k = append(k, key)
}
return k, nil
},
Get: func(key string) (json.RawMessage, error) {
if v, ok := f[key]; ok {
return json.RawMessage(v), nil
}
return nil, nil
},
}
}
// readerOf is a reader over a verdict the checker wrote through its own code and the candidate it judged.
func readerOf(t *testing.T) (Reader, string) {
t.Helper()
r := newRig(t, passJudge())
cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(HQRepository, 3, head2))
r.ch.Candidate(context.Background(), "feat%2Fx", cand)
r.ch.Wait()
ckey, _ := delivery.CandidatesKey("feat/x")
verdicts, kept := fakeState{}, fakeState{}
for k, v := range r.verdicts.vals {
verdicts[k] = v
}
if _, err := (Kept{Store: r.verdicts}).Keep(context.Background(), "unit", digA, keptValue("m/a", 3)); err != nil {
t.Fatal(err)
}
kept["unit."+digA] = r.verdicts.vals["unit."+digA]
return Reader{VerdictState: verdicts.read(), KeptState: kept.read(), CandidateState: fakeState{ckey: cand}.read()}, keyOf(t, "feat/x", cand)
}
func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest(t *testing.T) { // N9/AC1, 1686
rd, key := readerOf(t)
vs, err := rd.Verdicts("")
if err != nil || len(vs) != 1 || vs[0].Key != key {
t.Fatalf("verdicts = %+v, %v", vs, err)
}
if _, err := delivery.DecodeVerdict(vs[0].Verdict); err != nil {
t.Errorf("the verdict is not as stored: %v", err)
}
if got := vs[0].Reports["novox/mesh-delivery#7"]; !strings.HasPrefix(got, "pass 111111111111 on aaaaaaaaaaaa") || !strings.Contains(got, "composition not yet run") {
t.Errorf("reports = %v", vs[0].Reports)
}
if only, _ := rd.Verdicts("feat/other"); len(only) != 0 {
t.Errorf("another group's verdicts = %+v", only)
}
}
func TestAnHqPullRequestIsSaidToBeJudgedByMergeCheckUntilIssue1296(t *testing.T) { // ADR 0484 clause 8
rd, _ := readerOf(t)
hq, err := rd.Pull(HQRepository, 3)
if err != nil || hq.Note != "hq: judged by merge-check.sh until issue 1296" || hq.Verdict != nil {
t.Errorf("hq pull = %+v, %v", hq, err)
}
p, err := rd.Pull(repoA, 7)
if err != nil || p.Note != "" || !strings.HasPrefix(p.Reports["novox/mesh-delivery#7"], "pass ") {
t.Errorf("pull = %+v, %v", p, err)
}
if _, err := rd.Pull(repoA, 99); err == nil {
t.Error("a pull request no candidate holds was answered")
}
}
func TestTheKeptToolAnswersTheKeysAsStoredByLevelAndByDigest(t *testing.T) { // N9/AC1
rd, _ := readerOf(t)
all, err := rd.Kept("", "")
if err != nil || len(all) != 1 {
t.Fatalf("kept = %v, %v", all, err)
}
one, err := rd.Kept("unit", digA)
if err != nil || len(one) != 1 {
t.Fatalf("one kept = %v, %v", one, err)
}
for _, raw := range one {
if k, err := delivery.DecodeKept(raw); err != nil || k.Package != "m/a" {
t.Errorf("kept value = %+v, %v", k, err)
}
}
if none, _ := rd.Kept("integration", ""); len(none) != 0 {
t.Errorf("integration = %v", none)
}
if _, err := rd.Kept("unit", digB); err == nil {
t.Error("a digest nothing kept was answered")
}
}
modules/mesh-delivery-checker/internal/check/results.go
+95
View File
@@ -0,0 +1,95 @@
package check
import (
"encoding/json"
"strings"
)
// The shapes of what the build modules' steps answer, as their code writes them (mesh-catalog modules/mesh-build-go
// cmd/mesh-build-go levels.go TestResult, digest.go DigestResult, fetch.go FetchResult; modules/mesh-build-node
// cmd/mesh-build-node check.go TestResult, fetch.go FetchResult) with the fields the fetch step adds for a merge (ADR 0484
// clause 3). Only the fields the checker reads are named; the rest is ignored.
// MergeTree is what a fetch that merged says of one module directory: the tree sha, the digest of its manifest and the
// hashes of its lock files by name. A bare string is the tree alone.
type MergeTree struct {
Tree string `json:"tree"`
Manifest string `json:"manifest,omitempty"`
Locks map[string]string `json:"locks,omitempty"`
}
// UnmarshalJSON takes a string as the tree alone, or an object.
func (t *MergeTree) UnmarshalJSON(data []byte) error {
if s := strings.TrimSpace(string(data)); strings.HasPrefix(s, `"`) {
return json.Unmarshal(data, &t.Tree)
}
type plain MergeTree
return json.Unmarshal(data, (*plain)(t))
}
// Fetched is the answer of a build module's fetch step.
type Fetched struct {
Workspace string `json:"workspace"`
Commit string `json:"commit"`
Modules []string `json:"modules"`
// What a fetch with a base adds: the trunk commit, the merge commit, the files in conflict (a conflict is an answer),
// the paths the merge changes and the trees of the module directories.
Base string `json:"base,omitempty"`
Merge string `json:"merge,omitempty"`
Conflicts []string `json:"conflicts,omitempty"`
Changed []string `json:"changed,omitempty"`
Trees map[string]MergeTree `json:"trees,omitempty"`
}
// GoPackage is one package of a Go test result.
type GoPackage struct {
Dir string `json:"dir"`
Package string `json:"package"`
Passed bool `json:"passed"`
Elapsed float64 `json:"elapsed"`
Tests int `json:"tests"`
NoTests bool `json:"no_tests,omitempty"`
}
// Failure is a test that failed, or a package that failed with no test to name.
type Failure struct {
Package string `json:"package"`
Test string `json:"test,omitempty"`
Output string `json:"output,omitempty"`
}
// TestTime is one test and the seconds it took.
type TestTime struct {
Package string `json:"package"`
Test string `json:"test"`
Elapsed float64 `json:"elapsed"`
}
// Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one
// names the package directory as Module and counts none).
type Tested struct {
Level string `json:"level"`
Module string `json:"module,omitempty"`
Passed bool `json:"passed"`
Why string `json:"why,omitempty"`
NoTests bool `json:"no_tests,omitempty"`
Tests int `json:"tests"`
Skipped int `json:"skipped"`
Packages []GoPackage `json:"packages,omitempty"`
Elapsed float64 `json:"elapsed"`
Setup float64 `json:"setup,omitempty"`
Bound float64 `json:"bound"`
SetupBound float64 `json:"setup_bound,omitempty"`
Slowest []TestTime `json:"slowest,omitempty"`
Failed []Failure `json:"failed,omitempty"`
OverBound bool `json:"over_bound"`
OverWhich string `json:"over_which,omitempty"`
}
// Digested is the answer of the Go build module's digest step: one digest per package that has tests at the level, by import
// path.
type Digested struct {
Level string `json:"level"`
Packages map[string]string `json:"packages"`
WithoutTests []string `json:"without_tests,omitempty"`
}
modules/mesh-delivery-checker/internal/check/store.go
+19
View File
@@ -0,0 +1,19 @@
package check
import (
"context"
"errors"
)
// ErrConflict is what a KV answers when a Create finds the key or an Update finds another revision than the one given.
var ErrConflict = errors.New("the key exists or changed")
// KV is one of the checker's buckets: read, create and compare-and-set, each write answering the key's new revision. Only
// the checker's credential writes them (ADR 0444 clause 5, ADR 0484 clause 2).
type KV interface {
Get(ctx context.Context, key string) (value []byte, revision uint64, found bool, err error)
Create(ctx context.Context, key string, value []byte) (uint64, error)
Update(ctx context.Context, key string, value []byte, revision uint64) (uint64, error)
// Keys are every key of the bucket.
Keys(ctx context.Context) ([]string, error)
}
modules/mesh-delivery-checker/internal/check/testdata/captured/README
Vendored
+21
View File
@@ -0,0 +1,21 @@
go-test-failing.json, go-test-slow.json, go-test-empty.json and go-digest.json are what mesh-catalog's mesh-build-go answers
(TestResult and DigestResult, cmd/mesh-build-go levels.go and digest.go), marshalled by the module's own code: a throwaway
test beside the module's test_test.go and digest_test.go (workspaceFor + runUnit on the fixtures testdata/captured/failing,
slow and empty, and digestFixture + digests on testdata/repos/digest) at mesh-catalog origin/main cdbdb474, captured on
2026-10-21 for issue 1679 and not kept there. The fixtures' package names are the build module's own (fixture/failing/bad...).
go-fetch-clean.json and go-fetch-conflict.json are what mesh-build-go's fetch step ends with (FetchResult, cmd/mesh-build-go
fetch.go and preview.go) for a fetch with a base, the Result bytes of the step in mesh-catalog's own tests
TestFetchWithABaseMergesAndAnswersThePreview and TestFetchWithAConflictEndsPassedAndNamesTheFiles (preview_test.go), copied
into a throwaway test that wrote them out, at mesh-catalog feat/r4c5-checker d5804927 (the git commands are the test's fake,
so the shas are its constants; the shape and the field names are the module's own marshalling). node-fetch-clean.json is the
FetchResult json.Marshal of mesh-build-node's fetch (fetch.go, preview.go) in its test TestFetchWithABaseMergesAndAnswersThePreview
(cmd/mesh-build-node/preview_test.go) at the same commit. Captured on 2026-10-21 for issue 1679.
forge-get-file-go-mod.json is the answer of the forge's gitea_get_file (mesh-catalog modules/gitea, tools/index.ts) for modules/mesh-delivery-builder/go.mod of
novox/mesh-delivery at main, and forge-compare-trunk.json the answer of gitea_compare for novox/mesh-sdk base main, head cb54756c3865 (a commit on the
trunk: no commit main does not hold); both called through the mesh on 2026-10-21 for issue 1624. forge-compare-branch.json is the answer of the same call for a
head that is not on the trunk (novox/mesh-sdk base main, head afbe371003a5, the branch feat/r4c5-sdk), shortened to two of its seven commits and one of its
twenty-six files, with the head and the totals set to match.
forge-commits.json is the answer of the forge's gitea_commits for novox/mesh-sdk, ref cb54756c3865, limit 1, called through the mesh on 2026-10-21 (review D2).
modules/mesh-delivery-checker/internal/check/testdata/captured/forge-commits.json
Vendored
+14
View File
@@ -0,0 +1,14 @@
{
"commits": [
{
"author": "mesh-tools",
"date": "2026-10-21T10:02:09Z",
"sha": "cb54756c3865df5ac9c0dc0481ad9ebe2bf013a1",
"subject": "Merge pull request 'Give delivery.Module the toolchains its fingerprint was computed with, and '.' as a root module's path, so the builder builds a key only with its own toolchain (issue 1615)' (#55) from fix/1615-module-toolchains-root into main"
}
],
"more": true,
"page": 1,
"ref": "cb54756c3865",
"repository": "novox/mesh-sdk"
}
modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-branch.json
Vendored
+31
View File
@@ -0,0 +1,31 @@
{
"base": "main",
"commits": [
{
"author": "jochen",
"date": "2026-10-21T19:35:14+02:00",
"files": 2,
"sha": "9aaa5c80ac5c4820c4eddf488fbb90ac6981a624",
"subject": "Accept the gate's recheck status mesh/repo-check among the owed contexts, so a pinned recheck is not refused (issue 1673)"
},
{
"author": "jochen",
"date": "2026-10-21T19:26:02+02:00",
"files": 3,
"sha": "0730ef609216798e787ee8b6542912f5ec613a77",
"subject": "Keep the unit level cheap: the merge's real-git tests start processes, so they are integration tests (issue 895)"
}
],
"files": [
{
"path": "go/delivery/gate.go",
"status": "modified"
}
],
"head": "9aaa5c80ac5c",
"repository": "novox/mesh-sdk",
"said": [
"9aaa5c80ac5c holds 2 commits main does not, touching 1 files"
],
"total_commits": 2
}
modules/mesh-delivery-checker/internal/check/testdata/captured/forge-compare-trunk.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
{
"base": "main",
"commits": [],
"files": [],
"head": "cb54756c3865",
"repository": "novox/mesh-sdk",
"said": [
"cb54756c3865 holds 0 commits main does not, touching 0 files",
"a file's status is the last listed commit's change to it; for the lines, gitea_pull_request_diff or gitea_get_file"
],
"total_commits": 0
}
modules/mesh-delivery-checker/internal/check/testdata/captured/forge-get-file-go-mod.json
Vendored
+9
View File
@@ -0,0 +1,9 @@
{
"file": {
"content": "module mesh-delivery-builder\n\ngo 1.26.0\n\nrequire (\n\tgit.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865\n\tgithub.com/nats-io/nats-server/v2 v2.11.17\n\tgithub.com/nats-io/nats.go v1.54.0\n)\n\nrequire (\n\tgithub.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect\n\tgithub.com/google/go-tpm v0.9.8 // indirect\n\tgithub.com/klauspost/compress v1.20.0 // indirect\n\tgithub.com/minio/highwayhash v1.0.4 // indirect\n\tgithub.com/nats-io/jwt/v2 v2.8.1 // indirect\n\tgithub.com/nats-io/nkeys v0.4.16 // indirect\n\tgithub.com/nats-io/nuid v1.0.1 // indirect\n\tgolang.org/x/crypto v0.57.0 // indirect\n\tgolang.org/x/sys v0.48.0 // indirect\n\tgolang.org/x/time v0.15.0 // indirect\n)\n",
"path": "modules/mesh-delivery-builder/go.mod",
"ref": "main",
"sha": "4431597e2e4d2a26b59006f9f8c54659a964ec68",
"size": 691
}
}
modules/mesh-delivery-checker/internal/check/testdata/captured/go-digest.json
Vendored
+11
View File
@@ -0,0 +1,11 @@
{
"level": "unit",
"packages": {
"fixture/digest/a": "c288afe6252c38d9e7286ecb5f28b3c961780871fcc253670dcc3151c11c1df2",
"fixture/digest/c": "11abf5f32f0f12caafd76c565e4c2c2f3bf2e3ce6c2c7fcb6c2264a06db40087"
},
"without_tests": [
"fixture/digest/b",
"fixture/digest/d"
]
}
modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-clean.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"workspace":"mvii1iwu5414e4","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"fedcba9876543210fedcba9876543210fedcba98","changed":["a.go","modules/m/main.go"],"trees":{"modules/m":{"tree":"1111111111111111111111111111111111111111","manifest":"ca3d163bab055381827226140568f3bef7eaac187cebd76878e0b63e9e442356","locks":{"go.sum":"c5fc83c01e92404452b986527d239140ccf9a48b88e0c268fbf38c2e1429e9c9"}}}}
modules/mesh-delivery-checker/internal/check/testdata/captured/go-fetch-conflict.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"workspace":"mvii1iwz5050eb","commit":"0123456789abcdef0123456789abcdef01234567","modules":["."],"base":"abcdef0123456789abcdef0123456789abcdef01","conflicts":["a.go","sub/b.go"]}
modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-empty.json
Vendored
+21
View File
@@ -0,0 +1,21 @@
{
"level": "unit",
"passed": true,
"why": "the repository has no tests at the unit level",
"no_tests": true,
"tests": 0,
"skipped": 0,
"packages": [
{
"dir": ".",
"package": "fixture/empty",
"passed": true,
"elapsed": 0,
"tests": 0,
"no_tests": true
}
],
"elapsed": 0,
"bound": 5,
"over_bound": false
}
modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-failing.json
Vendored
+45
View File
@@ -0,0 +1,45 @@
{
"level": "unit",
"passed": false,
"why": "1 failed: fixture/failing/bad TestThreeIsFour",
"tests": 2,
"skipped": 0,
"packages": [
{
"dir": ".",
"package": "fixture/failing/bad",
"passed": false,
"elapsed": 0.003,
"tests": 1
},
{
"dir": ".",
"package": "fixture/failing/good",
"passed": true,
"elapsed": 0.002,
"tests": 1
}
],
"elapsed": 0.003,
"bound": 5,
"slowest": [
{
"package": "fixture/failing/bad",
"test": "TestThreeIsFour",
"elapsed": 0
},
{
"package": "fixture/failing/good",
"test": "TestTwo",
"elapsed": 0
}
],
"failed": [
{
"package": "fixture/failing/bad",
"test": "TestThreeIsFour",
"output": " bad_test.go:7: Three() = 3, want 4\n--- FAIL: TestThreeIsFour (0.00s)"
}
],
"over_bound": false
}
modules/mesh-delivery-checker/internal/check/testdata/captured/go-test-slow.json
Vendored
+30
View File
@@ -0,0 +1,30 @@
{
"level": "unit",
"passed": true,
"tests": 2,
"skipped": 0,
"packages": [
{
"dir": ".",
"package": "fixture/slow",
"passed": true,
"elapsed": 0.065,
"tests": 2
}
],
"elapsed": 0.065,
"bound": 5,
"slowest": [
{
"package": "fixture/slow",
"test": "TestTakesAWhile",
"elapsed": 0.06
},
{
"package": "fixture/slow",
"test": "TestQuick",
"elapsed": 0
}
],
"over_bound": false
}
modules/mesh-delivery-checker/internal/check/testdata/captured/node-fetch-clean.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"workspace":"mvii1sxt979f67","commit":"0123456789abcdef0123456789abcdef01234567","modules":["modules/m"],"installed":[],"refused":[],"base":"abcdef0123456789abcdef0123456789abcdef01","merge":"0123456789abcdef0123456789abcdef01234567","changed":["modules/m/package.json"],"trees":{"modules/m":{"tree":"0123456789abcdef0123456789abcdef01234567","manifest":"0581b699d9d05517e69b21f5cedbeb2eee8aba2b205280b561be731594d7feef"}}}
modules/mesh-delivery-checker/manifest_test.go
+220
View File
@@ -0,0 +1,220 @@
package manifest
import (
"bytes"
"encoding/json"
"io/fs"
"os"
"path/filepath"
"reflect"
"slices"
"strings"
"testing"
)
// The controller's manifest reader is mesh-controller internal/catalogue (Manifest, manifest.go: Consumes :521,
// Invokes :590, Streams :613, Reads :659; StreamDeclaration, streams.go), an internal package of another repository.
// These tests decode the manifest strictly into the fields those types name (DisallowUnknownFields) and hold the
// rules of issues 1678 and 1683 (ADR 0453 §4, ADR 0484). The merge gate's module check proves the controller takes the manifest.
type state struct {
Name string `json:"name"`
History int `json:"history,omitempty"`
}
type artifact struct {
Name string `json:"name"`
Kind string `json:"kind"`
Language string `json:"language"`
System string `json:"system"`
From string `json:"from"`
Binary string `json:"binary"`
Loads []string `json:"loads"`
}
type stream struct {
Name string `json:"name"`
Events []string `json:"events"`
}
type manifest struct {
Streams []stream `json:"streams"`
Tools []string `json:"tools"`
Module string `json:"module"`
Version string `json:"version"`
Slug string `json:"slug"`
Consumes []string `json:"consumes"`
State []state `json:"state"`
Reads []string `json:"reads"`
Invokes []string `json:"invokes"`
Config map[string]struct {
Type string `json:"type"`
Description string `json:"description"`
Pattern string `json:"pattern"`
Default string `json:"default"`
} `json:"config"`
OwnSecrets map[string]string `json:"own-secrets"`
SecretsOwner string `json:"secrets-owner"`
Resources []map[string]any `json:"resources"`
Build struct {
Artifacts []artifact `json:"artifacts"`
} `json:"build"`
}
func load(t *testing.T) (manifest, map[string]json.RawMessage) {
t.Helper()
raw, err := os.ReadFile("module.json")
if err != nil {
t.Fatal(err)
}
var m manifest
dec := json.NewDecoder(bytes.NewReader(raw))
dec.DisallowUnknownFields()
if err := dec.Decode(&m); err != nil {
t.Fatalf("the controller's reader refuses a key this manifest has: %v", err)
}
var top map[string]json.RawMessage
if err := json.Unmarshal(raw, &top); err != nil {
t.Fatal(err)
}
return m, top
}
func sorted(s []string) []string {
s = slices.Clone(s)
slices.Sort(s)
return s
}
func TestManifestDeclarations(t *testing.T) { // 1678/AC1
m, top := load(t)
if m.Module != "mesh-delivery-checker" {
t.Errorf("module = %q", m.Module)
}
// delivery.CheckerKeptHistory, CheckerOwedHistory, PreviewsHistory, CheckerVerdictsHistory; the names follow the SDK's buckets.
if want := []state{{Name: "kept", History: 1}, {Name: "owed", History: 5}, {Name: "previews", History: 5}, {Name: "verdicts", History: 5}}; !reflect.DeepEqual(m.State, want) {
t.Errorf("state = %+v, want %+v", m.State, want)
}
if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains",
"mesh-controller.plans-building", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) {
t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5) and the forge's two reads for a mesh-sdk pin (clause 9)", m.Invokes)
}
// The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts.
if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if b, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 || b != "${dir:state}/broker" {
t.Errorf("own-secrets = %v, want exactly broker", m.OwnSecrets)
}
if m.SecretsOwner != "mesh-delivery-checker" {
t.Errorf("secrets-owner = %q", m.SecretsOwner)
}
// The checker says each time it wrote a preview or a status, so the indexer's and the forge bridge's watches find the buckets (ADR 0484 clauses 2 and 3).
if want := []stream{{Name: "checks", Events: []string{"preview.**", "owed.**"}}}; !reflect.DeepEqual(m.Streams, want) {
t.Errorf("streams = %+v, want %+v", m.Streams, want)
}
// The read tools (issue 1686) are served by the tools bundle, not by the checker's process.
if want := []string{"checker_kept", "checker_pull", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) {
t.Errorf("tools = %v, want %v", m.Tools, want)
}
for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} {
if _, ok := top[k]; ok {
t.Errorf("the manifest declares %q: the checker has none", k)
}
}
}
func TestManifestProcess(t *testing.T) { // 1678/AC1
m, _ := load(t)
var procs []map[string]any
for _, r := range m.Resources {
if r["type"] == "process" {
procs = append(procs, r)
}
if r["type"] == "tool" || r["type"] == "seat" {
t.Errorf("resource %v: the checker serves no tool and holds no seat", r)
}
}
if len(procs) != 1 {
t.Fatalf("%d process resources, want one", len(procs))
}
p := procs[0]
if p["artifact"] != "program" || !reflect.DeepEqual(p["run"], []any{"./mesh-delivery-checker"}) {
t.Errorf("process = %v", p)
}
env, _ := p["env"].(map[string]any)
// The capacity of each node is the module's setting `capacity` (ADR 0484 clause 12, as the builder's, issue 1661), laid into the
// settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity.
if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_CHECKER_SETTINGS"] != "${dir:state}/settings.json" {
t.Errorf("env = %v", p["env"])
}
if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) {
t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"])
}
c, ok := m.Config["capacity"]
if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "" || c.Description == "" {
t.Errorf("config = %+v, want one string setting, capacity, empty (one step at a time on every node) to start", m.Config)
}
var settings map[string]any
for _, r := range m.Resources {
if r["id"] == "settings" {
settings = r
}
}
if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" {
t.Errorf("settings resource = %v", settings)
}
if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": ""`) {
t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err)
}
if _, bus := p["bus"]; bus {
t.Errorf("the process declares a bus: it runs on its own broker credential")
}
if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[2].Name != "files" || m.Build.Artifacts[2].Kind != "archive" || m.Build.Artifacts[2].From != "files" {
t.Fatalf("artifacts = %+v, want the program, the tools and the files", m.Build.Artifacts)
}
if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Language != "go" || a.Binary != "mesh-delivery-checker-tools" ||
!reflect.DeepEqual(a.Loads, []string{"mesh-delivery-checker-tools"}) {
t.Errorf("tools artifact = %+v", a)
}
if st, err := os.Stat(m.Build.Artifacts[1].From); err != nil || !st.IsDir() {
t.Errorf("the tools' source %s: %v", m.Build.Artifacts[1].From, err)
}
a := m.Build.Artifacts[0]
if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-checker" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" {
t.Errorf("artifact = %+v", a)
}
}
func TestServesNoRequestSubjectAndHoldsNoTicker(t *testing.T) { // 1678/AC6 (ADR 0437 §8: no timer, a wait is a context's bound)
forbidden := []string{"Subscribe(", "QueueSubscribe", "ChanSubscribe", "time.NewTicker", "time.Tick", "time.NewTimer", "time.AfterFunc", "time.After(", "time.Sleep"}
var files int
for _, root := range []string{"cmd", "internal"} {
err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() || !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
return err
}
files++
src, err := os.ReadFile(path)
if err != nil {
return err
}
for _, f := range forbidden {
if strings.Contains(string(src), f) {
t.Errorf("%s holds %s: the checker serves no request subject and holds no timer", path, f)
}
}
return nil
})
if err != nil {
t.Fatal(err)
}
}
if files < 1 {
t.Fatalf("read %d source files, want the program's", files)
}
}
modules/mesh-delivery-checker/module.json
+135
View File
@@ -0,0 +1,135 @@
{
"module": "mesh-delivery-checker",
"version": "1",
"slug": "checker",
"consumes": [
"mesh-build-go.toolchain.reported",
"mesh-build-node.toolchain.reported"
],
"streams": [
{ "name": "checks", "events": ["preview.**", "owed.**"] }
],
"state": [
{ "name": "kept", "history": 1 },
{ "name": "owed", "history": 5 },
{ "name": "previews", "history": 5 },
{ "name": "verdicts", "history": 5 }
],
"reads": [
"mesh-build-go.steps",
"mesh-build-go.toolchains",
"mesh-build-node.steps",
"mesh-build-node.toolchains",
"mesh-controller.plans-building",
"mesh-delivery-builder.builds",
"mesh-delivery-gate.source",
"mesh-delivery-indexer.candidates",
"mesh-delivery-indexer.manifests",
"mesh-delivery-indexer.modules",
"mesh-delivery-watchman.presence",
"node-engine.have"
],
"tools": [
"checker_kept",
"checker_pull",
"checker_verdicts"
],
"invokes": [
"gitea.gitea_commits",
"gitea.gitea_compare",
"gitea.gitea_get_file",
"mesh-build-go.build_go_digest",
"mesh-build-go.build_go_fetch",
"mesh-build-go.build_go_test",
"mesh-build-node.build_node_fetch",
"mesh-build-node.build_node_test"
],
"config": {
"capacity": {
"type": "string",
"description": "How many build steps the checker runs at once on each node, as node=steps pairs separated by commas, such as shanks=2,ace=1 (ADR 0484 clause 12); a node set to 0 never checks, a node not named runs one at a time, and empty is one everywhere. The builder's capacity is the builder's own.",
"pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*",
"default": ""
}
},
"own-secrets": {
"broker": "${dir:state}/broker"
},
"secrets-owner": "mesh-delivery-checker",
"resources": [
{
"id": "account",
"type": "user",
"name": "mesh-delivery-checker",
"shell": "/usr/bin/nologin",
"home": "/var/lib/mesh-delivery-checker"
},
{
"id": "state",
"type": "directory",
"mode": "0700",
"owner": "mesh-delivery-checker",
"place": ".",
"unknown": "program"
},
{
"id": "settings",
"trusted": false,
"type": "file",
"path": "${dir:state}/settings.json",
"mode": "0600",
"owner": "mesh-delivery-checker",
"takes-config": ["capacity"],
"merge": "json",
"template": "files/settings.json"
},
{
"id": "checker",
"type": "process",
"name": "mesh-delivery-checker",
"artifact": "program",
"run": [
"./mesh-delivery-checker"
],
"env": {
"MESH_BROKER_FILE": "${dir:state}/broker",
"MESH_CHECKER_SETTINGS": "${dir:state}/settings.json"
},
"restart-on": [
"settings"
],
"user": "mesh-delivery-checker",
"health": {
"kind": "unit"
}
}
],
"build": {
"artifacts": [
{
"name": "program",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-checker",
"binary": "mesh-delivery-checker"
},
{
"name": "tools",
"kind": "bundle",
"language": "go",
"system": "arch",
"from": "cmd/mesh-delivery-checker-tools",
"binary": "mesh-delivery-checker-tools",
"loads": [
"mesh-delivery-checker-tools"
]
},
{
"name": "files",
"kind": "archive",
"from": "files"
}
]
}
}
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate.go
+27 -2
View File
@@ -91,9 +91,34 @@ func (p *performer) refuse(ctx context.Context, doneKey string, rev uint64, a br
return true
}
// owed performs the act an owed key asks at revision rev, once.
// owed performs the act an owed key of the gate asks at revision rev, once.
func (p *performer) owed(ctx context.Context, key string, rev uint64, value []byte) {
doneKey := "owed." + key
p.perform(ctx, "owed."+key, key, rev, value)
}
// checkOwed performs the status the checker owes on a head (mesh-delivery-checker_owed; novox/hq ADR 0484 clause 2) at
// revision rev, once. Its keys are kept apart from the gate's, and a key that is not the checker's one act, the status
// mesh/check, is left.
func (p *performer) checkOwed(ctx context.Context, key string, rev uint64, value []byte) {
doneKey := "check." + key
if p.seen(ctx, doneKey, rev) {
return
}
if _, _, _, err := delivery.ParseCheckerOwedKey(key); err != nil {
p.logger.Printf("%s at revision %d is not a status of the checker, and is left: %v", key, rev, err)
p.keep(ctx, doneKey, rev)
return
}
if _, err := delivery.DecodeCheckerOwed(value); err != nil {
p.logger.Printf("%s at revision %d is not a status of the checker, and is left: %v", key, rev, err)
p.keep(ctx, doneKey, rev)
return
}
p.perform(ctx, doneKey, key, rev, value)
}
// perform does the act of an owed key once for the revision, under the bookkeeping key doneKey.
func (p *performer) perform(ctx context.Context, doneKey, key string, rev uint64, value []byte) {
if p.seen(ctx, doneKey, rev) {
return
}
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/gate_test.go
+54
View File
@@ -208,3 +208,57 @@ func TestAMergeWithTwoHandedStopsAtTheFirstRefusal(t *testing.T) { // ADR 0441
t.Errorf("calls %d messages %d, want the first refusal to stop the pass", len(f.calls), len(pub.data))
}
}
// checkerOwed is the status the checker owes, as it writes it (mesh-sdk go/delivery CheckerOwedKey, EncodeCheckerOwed).
func checkerOwed(t *testing.T, number int, state, text string) (string, []byte) {
t.Helper()
key, err := delivery.CheckerOwedKey("novox/mesh-delivery", number, headA)
if err != nil {
t.Fatal(err)
}
v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Text: text, Target: "https://view.example/checks/g.0123456789ab"})
if err != nil {
t.Fatal(err)
}
return key, v
}
func TestTheCheckersStatusIsSetOncePerRevisionAndAKeyOfTheGateIsNotItsKey(t *testing.T) { // 1683/AC5 (ADR 0484 clause 2)
f := &forgeFake{}
p, _, done, _ := newPerformer(f)
key, v := checkerOwed(t, 12, "success", "pass aaaaaaaaaaaa on bbbbbbbbbbbb")
p.checkOwed(context.Background(), key, 3, v)
p.checkOwed(context.Background(), key, 3, v)
if len(f.calls) != 1 || f.calls[0].key != "gitea.gitea_commit_status" ||
f.calls[0].body != `{"context":"mesh/check","description":"pass aaaaaaaaaaaa on bbbbbbbbbbbb","owner":"novox","repo":"mesh-delivery","sha":"`+headA+`","state":"success","target_url":"https://view.example/checks/g.0123456789ab"}` {
t.Fatalf("calls = %+v", f.calls)
}
p.checkOwed(context.Background(), key, 4, v)
if len(f.calls) != 2 {
t.Errorf("a new revision was not performed: %+v", f.calls)
}
if _, ok := done["check."+key]; !ok {
t.Errorf("revision kept under %v, want check.%s", done, key)
}
// The gate's own owed bookkeeping is another key: the same owed key seen in the gate's bucket is its own act.
if _, ok := done["owed."+key]; ok {
t.Errorf("the checker's status was kept under the gate's key: %v", done)
}
}
func TestAnythingButTheCheckersStatusInItsBucketIsLeft(t *testing.T) { // ADR 0484 clause 2: the checker owes one act
f := &forgeFake{}
p, _, done, logs := newPerformer(f)
// A comment, and a status of another context: the SDK refuses both to the checker's encoder, so they are written raw.
commentKey, commentValue := owedKV(t, "novox/mesh-delivery", 12, "comment.1", delivery.Owed{Act: delivery.OwedComment, Text: "hello"})
gateKey, gateValue := owedKV(t, "novox/mesh-delivery", 12, "status=mesh/merge-gate",
delivery.Owed{Act: delivery.OwedStatus, Context: "mesh/merge-gate", State: "success", Text: "x", Target: "https://x.example"})
p.checkOwed(context.Background(), commentKey, 1, commentValue)
p.checkOwed(context.Background(), gateKey, 1, gateValue)
if len(f.calls) != 0 {
t.Errorf("the forge was called: %+v", f.calls)
}
if !strings.Contains(logs.String(), commentKey) || !strings.Contains(logs.String(), gateKey) || len(done) != 2 {
t.Errorf("log %q, kept %v: both keys are named and kept so they are not tried again", logs.String(), done)
}
}
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/integration_test.go
+83
View File
@@ -30,6 +30,10 @@ import (
// The gate's buckets mesh-delivery-gate_merges and mesh-delivery-gate_owed are read and watched (the manifest's reads), the
// bucket performed is the bridge's own (its state), and the five forge tools of the gate's acts are invoked (issue 1341, 1342).
//
// The checker's owed bucket is read like the gate's (reads) and the checker's stream checks is heard like EVENTS is (consumes
// mesh-delivery-checker.owed.written; ADR 0484 clause 2): those entries are written by analogy with the ones the controller
// composes for the gate's buckets and for the EVENTS consumer, not taken from a composed file.
//
// The lists are what mesh-controller's broker.PermissionsFor answers (internal/broker/nats.go:577) for a module principal
// with the manifest of module.json: consumes the three gitea.pull events, invokes gitea.gitea_open_pulls,
// gitea.gitea_commit_trees and gitea.gitea_get_file, state heads, and the stream forge; `**` is written `>` on the wire (modstreams.go
@@ -104,6 +108,14 @@ accounts {
"mesh.mod.gitea.tool.gitea_add_comment"
"mesh.mod.gitea.tool.gitea_add_comment.*"
"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"
"$JS.ACK.CHECKS.n1_mesh-delivery-forge-bridge.>"
"$JS.API.CONSUMER.INFO.CHECKS.n1_mesh-delivery-forge-bridge"
"$JS.API.CONSUMER.MSG.NEXT.CHECKS.n1_mesh-delivery-forge-bridge"
"$JS.API.CONSUMER.CREATE.KV_mesh-delivery-checker_owed.>"
"$JS.API.CONSUMER.DELETE.KV_mesh-delivery-checker_owed.>"
"$JS.API.DIRECT.GET.KV_mesh-delivery-checker_owed.>"
"$JS.API.STREAM.INFO.KV_mesh-delivery-checker_owed"
"$JS.FC.KV_mesh-delivery-checker_owed.>"
] }
subscribe { allow: [
"_INBOX.n1.mesh-delivery-forge-bridge.>"
@@ -205,6 +217,15 @@ func startBed(t *testing.T, openReply string, prepare func(ctx context.Context,
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "FORGE", Subjects: []string{"mesh.stream.mesh-delivery-forge-bridge.event.forge.>"}}); err != nil {
t.Fatal(err)
}
// The checker's stream checks and the bridge's durable consumer on its owed.written event; the checker's owed bucket is made
// only by the test that needs it, as when the checker is assigned after the bridge started.
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "CHECKS", Subjects: []string{"mesh.stream.mesh-delivery-checker.event.>"}}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, "CHECKS", jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-forge-bridge", AckPolicy: jetstream.AckExplicitPolicy,
FilterSubjects: []string{"mesh.stream.mesh-delivery-checker.event.owed.written"}}); err != nil {
t.Fatal(err)
}
// The bridge's state bucket, made by the controller from the manifest's state entry before the process starts.
if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: headsBucket, History: 1}); err != nil {
t.Fatal(err)
@@ -706,3 +727,65 @@ func forgeMerged(t *testing.T, b *bed) []string {
}
func sha256Sum(b []byte) []byte { s := sha256.Sum256(b); return s[:] }
// TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey: the checker's owed bucket is made after the bridge started;
// the checker writes the status mesh/check it owes on a head (the SDK's CheckerOwedKey and EncodeCheckerOwed, which the checker
// itself writes with) and says so on its stream; the bridge sets the status on the forge within 10 s, with the context
// mesh/check, the state and the line; the same revision is not set again, a new revision is (ADR 0484 clause 2).
func TestBridgePerformsTheCheckersStatusOncePerRevisionOfItsOwedKey(t *testing.T) { // 1683/AC5, ADR 0484 §2
b := startBed(t, `{"result":{"pulls":[]}}`, nil)
calls := make(chan string, 8)
sub, err := b.ctl.Subscribe("mesh.mod.gitea.tool.gitea_commit_status", func(m *nats.Msg) {
calls <- string(m.Data)
_ = m.Respond([]byte(`{"result":{},"node":"n0"}`))
})
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = sub.Unsubscribe() })
if err := b.ctl.Flush(); err != nil {
t.Fatal(err)
}
kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.CheckerOwedBucket, History: delivery.CheckerOwedHistory})
if err != nil {
t.Fatal(err)
}
key, err := delivery.CheckerOwedKey("novox/mesh-delivery", 12, head)
if err != nil {
t.Fatal(err)
}
put := func(state, text string) {
v, err := delivery.EncodeCheckerOwed(delivery.Owed{Act: delivery.OwedStatus, Context: delivery.StatusContextCheck, State: state, Text: text, Target: "https://view.example/checks/g.0123456789ab"})
if err != nil {
t.Fatal(err)
}
if _, err := kv.Put(b.ctx, key, v); err != nil {
t.Fatal(err)
}
if _, err := b.js.Publish(b.ctx, owedWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil {
t.Fatal(err)
}
}
next := func() string {
select {
case got := <-calls:
return got
case <-time.After(10 * time.Second):
t.Fatal("the status was not set within 10 s")
}
return ""
}
put("pending", "checking 0123456789ab on aaaaaaaaaaaa")
if got := next(); !strings.Contains(got, `"context":"mesh/check"`) || !strings.Contains(got, `"state":"pending"`) || !strings.Contains(got, `"sha":"`+head+`"`) || !strings.Contains(got, `"repo":"mesh-delivery"`) {
t.Fatalf("first call = %s", got)
}
put("success", "pass 0123456789ab on aaaaaaaaaaaa | unit 2 run, 3 kept")
if got := next(); !strings.Contains(got, `"state":"success"`) || !strings.Contains(got, "unit 2 run, 3 kept") {
t.Fatalf("second call = %s", got)
}
select {
case extra := <-calls:
t.Errorf("a third call was made: %s", extra)
case <-time.After(500 * time.Millisecond):
}
}
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/late.go
+82
View File
@@ -0,0 +1,82 @@
package main
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// checksStream is the stream the checker carries its events on, as the bus names it (declared by the checker as the stream
// checks, heard by this module through consumes; novox/hq ADR 0484 clause 2).
const checksStream = "CHECKS"
// owedWrittenSubject is the subject of the checker's owed.written event.
const owedWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.owed.written"
// lateWatch watches a bucket that may be made after the bridge started: the checker's owed bucket. The bridge holds no timer
// (ADR 0437 §8), so it watches the bucket where it exists at start, and otherwise when the checker says it wrote a status
// (owed.written, handled by Written).
type lateWatch struct {
ctx context.Context
js interface {
KeyValue(ctx context.Context, bucket string) (jetstream.KeyValue, error)
}
bucket string
react func(ctx context.Context, key string, rev uint64, value []byte)
logger *log.Logger
errs chan error
mu sync.Mutex
watched bool
}
// ensure starts the watch if the bucket exists and is not watched.
func (w *lateWatch) ensure() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.watched {
return nil
}
kv, err := w.js.KeyValue(w.ctx, w.bucket)
switch {
case errors.Is(err, jetstream.ErrBucketNotFound):
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a status", w.bucket)
return nil
case err != nil:
return fmt.Errorf("the bucket %s: %w", w.bucket, err)
}
w.watched = true
go func() {
w.errs <- watch(w.ctx, kv, func(key string, rev uint64, value []byte) {
hctx, cancel := context.WithTimeout(w.ctx, syncDeadline)
defer cancel()
w.react(hctx, key, rev, value)
})
}()
return nil
}
// Written takes one owed.written event of the checker in: its bucket is watched if it is not. A subject that is not the
// checker's is terminated; a failure stays unacknowledged, and the server delivers it again.
func (w *lateWatch) Written(m interface {
Subject() string
Ack() error
Term() error
}) {
if !strings.HasPrefix(m.Subject(), owedWrittenSubject) {
w.logger.Printf("%s: not the checker's owed.written, dropped", m.Subject())
_ = m.Term()
return
}
if err := w.ensure(); err != nil {
w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err)
return
}
_ = m.Ack()
}
modules/mesh-delivery-forge-bridge/cmd/mesh-delivery-forge-bridge/main.go
+19 -1
View File
@@ -202,7 +202,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return fmt.Errorf("the bucket %s: %w", performedBucket, err)
}
perf := &performer{call: call, pub: rec, done: kvRevisions{performedKV}, logger: logger}
watching := make(chan error, 2)
watching := make(chan error, 3)
for _, w := range []struct {
bucket string
react func(ctx context.Context, key string, rev uint64, value []byte)
@@ -225,6 +225,24 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
})
}()
}
// The checker's statuses (mesh/check, unrequired until release R4f): its bucket is made when the checker is assigned, which
// may be after the bridge started, and the bridge holds no timer; the checker says on its stream each time it wrote a
// status, and the bucket is then watched (late.go).
late := &lateWatch{ctx: ctx, js: js, bucket: delivery.CheckerOwedBucket, react: perf.checkOwed, logger: logger, errs: watching}
if err := late.ensure(); err != nil {
return err
}
switch ccons, err := js.Consumer(ctx, checksStream, cred.Node+"_"+cred.Module); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's statuses are not heard until the bridge starts again: %v", cred.Node, cred.Module, checksStream, err)
default:
cc, err := ccons.Consume(func(m jetstream.Msg) { late.Written(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
defer cc.Stop()
}
if err := said(ctx); err != nil {
return err
}
modules/mesh-delivery-forge-bridge/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-forge-bridge
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-forge-bridge/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960 h1:FCId3IuRScG03X7Gx9vfKfGNZH9jWnfx9RChfJ3eWfo=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020182814-4024909a2960/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-forge-bridge/manifest_test.go
+2 -2
View File
@@ -86,10 +86,10 @@ func TestManifestDeclarations(t *testing.T) { // 1235/AC1
if !reflect.DeepEqual(m.Streams, []stream{{Name: "forge", Events: []string{"forge.**"}}}) {
t.Errorf("streams = %+v", m.Streams)
}
if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
if want := []string{"gitea.pull.closed", "gitea.pull.merged", "gitea.pull.updated", "mesh-delivery-checker.owed.written"}; !reflect.DeepEqual(sorted(m.Consumes), want) {
t.Errorf("consumes = %v, want %v", m.Consumes, want)
}
if want := []string{"mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) {
if want := []string{"mesh-delivery-checker.owed", "mesh-delivery-gate.merges", "mesh-delivery-gate.owed"}; !reflect.DeepEqual(sorted(m.Reads), want) {
t.Errorf("reads = %v, want %v", m.Reads, want)
}
if want := []string{"gitea.gitea_add_comment", "gitea.gitea_commit_status", "gitea.gitea_commit_trees", "gitea.gitea_delivery_view",
modules/mesh-delivery-forge-bridge/module.json
+3 -1
View File
@@ -5,7 +5,8 @@
"consumes": [
"gitea.pull.updated",
"gitea.pull.merged",
"gitea.pull.closed"
"gitea.pull.closed",
"mesh-delivery-checker.owed.written"
],
"streams": [
{ "name": "forge", "events": ["forge.**"] }
@@ -15,6 +16,7 @@
{ "name": "performed", "history": 1 }
],
"reads": [
"mesh-delivery-checker.owed",
"mesh-delivery-gate.merges",
"mesh-delivery-gate.owed"
],
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go
+89
View File
@@ -351,3 +351,92 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes
t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got)
}
}
// TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBucket: the indexer starts with the stream PREVIEWS
// and its durable consumer (made by the controller from the checker's declaration and the indexer's consumes) and no bucket
// mesh-delivery-checker_previews; a pull request of mesh-catalog touching avahi makes the candidate with the reason
// 'no merge preview is computed'; the checker then makes its bucket, writes the preview of that head onto the trunk head
// the source names, and says so on its stream; the candidate's avahi gets, within 5 s and with no restart, the reason
// computed from the preview (the toolchain is not reported here, so 'toolchain not yet reported').
func TestIndexerGivesACandidateItsFingerprintWhenTheCheckerCreatesThePreviewsBucket(t *testing.T) { // 1445/AC4
manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json")
if err != nil {
t.Fatal(err)
}
const goSum = "3b5028f0c1e9bde04dd766abd63eb5931bc516bae695cf46f00abcd8510b0381"
const inputs = `{"Module":"avahi","Tree":"oldtree","Bases":null,"Contexts":null,"Toolchains":{"code":"go sha256:ccc 0123456789abcdef"},"Unpinned":""}`
const trunk, head = "5555555555555555555555555555555555555555", "0123456789abcdef0123456789abcdef01234567"
js, ctx, candidates, _ := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) {
rec := `{"id":"build-1","repository":"novox/mesh-catalog","ref":"main","module":"avahi","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/avahi","at":"2026-10-19T09:38:00Z","source-inputs":` + inputs + `}`
if _, err := builds.Put(ctx, delivery.BuildKey("avahi", "build-1"), []byte(rec)); err != nil {
t.Fatal(err)
}
if _, err := builds.Put(ctx, delivery.NewestKey("avahi"), []byte("build-1")); err != nil {
t.Fatal(err)
}
}, func(ctx context.Context, js jetstream.JetStream) {
if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: PreviewsStream, Subjects: []string{previewWrittenSubject}}); err != nil {
t.Fatal(err)
}
if _, err := js.CreateOrUpdateConsumer(ctx, PreviewsStream, jetstream.ConsumerConfig{Durable: "n1_mesh-delivery-indexer", AckPolicy: jetstream.AckExplicitPolicy}); err != nil {
t.Fatal(err)
}
source, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.SourceBucket, History: 4})
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodeSource(delivery.Source{Group: "g", Heads: map[string]string{"novox/mesh-catalog": trunk}})
if err != nil {
t.Fatal(err)
}
if _, err := source.Put(ctx, delivery.SourceMainKey, data); err != nil {
t.Fatal(err)
}
})
pull, err := delivery.EncodeForge(delivery.Forge{Kind: delivery.ForgePull, Repository: "novox/mesh-catalog", Branch: "feat/x", Number: 301, Head: head, Action: delivery.PullUpdated,
Dirs: []string{"modules/avahi"}, DirsKnown: true, Manifests: map[string]string{"modules/avahi": string(manifest)}})
if err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, "mesh.stream.mesh-delivery-forge-bridge.event.forge.novox.mesh-catalog.pull", pull); err != nil {
t.Fatal(err)
}
none2 := func(want string) string {
var got string
for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) {
if e, err := candidates.Get(ctx, "feat/x"); err == nil {
if c, err := delivery.DecodeCandidate(e.Value()); err == nil && len(c.Pulls) == 1 {
got = c.Pulls[0].Fps["avahi"].None2
if got == want {
break
}
}
}
}
return got
}
if got := none2("no merge preview is computed"); got != "no merge preview is computed" {
t.Fatalf("none2 = %q within 5 s of the pull request, want 'no merge preview is computed'", got)
}
// The checker's bucket is made only now, as when the checker is assigned after the indexer started.
kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PreviewsBucket, History: delivery.PreviewsHistory})
if err != nil {
t.Fatal(err)
}
pv := delivery.Preview{Repository: "novox/mesh-catalog", Head: head, Trunk: trunk, Merge: "6666666666666666666666666666666666666666",
Modules: map[string]delivery.PreviewModule{"modules/avahi": {Tree: "newtree", Manifest: delivery.ManifestDigest(manifest), Locks: map[string]string{"go.sum": goSum}}}}
key, _ := delivery.PreviewKey(pv.Repository, pv.Head, pv.Trunk)
data, err := delivery.EncodePreview(pv)
if err != nil {
t.Fatal(err)
}
if _, err := kv.Create(ctx, key, data); err != nil {
t.Fatal(err)
}
if _, err := js.Publish(ctx, previewWrittenSubject, []byte(`{"key":"`+key+`"}`)); err != nil {
t.Fatal(err)
}
if got := none2(delivery.ToolchainNoReport); got != delivery.ToolchainNoReport {
t.Errorf("none2 = %q within 5 s of the preview, want %q (computed from the preview)", got, delivery.ToolchainNoReport)
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go
+19
View File
@@ -309,6 +309,12 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
if err := tools.start(); err != nil {
return err
}
// The checker's merge previews give the candidates their fingerprints: the bucket is watched where it exists, and
// otherwise when the checker says it wrote a preview.
previews := newPreviewWatch(ctx, js, ix, logger)
if err := previews.ensure(); err != nil {
return err
}
// Only now, with the toolchains read, are the builds heard to compute from (a build before it would write every Go
// module as having no toolchain reported, and back); the builds heard until now are computed once.
started.Store(ix)
@@ -332,6 +338,17 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
}
defer tc.Stop()
}
switch pcons, err := js.Consumer(ctx, PreviewsStream, cred.Node+"_"+cred.Module); {
case err != nil:
logger.Printf("the consumer %s_%s on %s does not exist yet, so the checker's previews are not heard until the indexer starts again: %v", cred.Node, cred.Module, PreviewsStream, err)
default:
pc, err := pcons.Consume(func(m jetstream.Msg) { previews.Written(m) },
jetstream.ConsumeErrHandler(func(_ jetstream.ConsumeContext, err error) { logger.Printf("hearing the bus: %v", err) }))
if err != nil {
return err
}
defer pc.Stop()
}
cons, err := js.Consumer(ctx, forgeStream, cred.Node+"_"+cred.Module)
if err != nil {
return fmt.Errorf("the consumer %s_%s on %s: %w", cred.Node, cred.Module, forgeStream, err)
@@ -351,5 +368,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro
return err
case err := <-tools.errs:
return err
case err := <-previews.errs:
return err
}
}
modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/previews.go
+88
View File
@@ -0,0 +1,88 @@
package main
import (
"context"
"errors"
"fmt"
"log"
"strings"
"sync"
"github.com/nats-io/nats.go/jetstream"
"git.novox.be/novox/mesh-sdk/go/delivery"
"mesh-delivery-indexer/internal/index"
)
// PreviewsStream is the stream the checker carries its events on, as the bus names it (declared by the
// checker as the stream checks, heard by this module through consumes; novox/hq ADR 0484 clause 3).
const PreviewsStream = "CHECKS"
// previewWrittenSubject is the subject of the checker's preview.written event.
const previewWrittenSubject = "mesh.stream." + delivery.CheckerModule + ".event.preview.written"
// previewWatch reads the merge previews the checker writes (mesh-delivery-checker_previews) and follows them: every change
// writes the candidates' fingerprints again (index.Indexer.ApplyPreview). The checker's bucket is made when it is assigned,
// after the indexer started, and the indexer holds no timer (ADR 0437 §8): it starts with the bucket absent, and the
// checker says each time it wrote a preview (preview.written, handled by Written); the bucket is then watched and settled
// as the others, with no restart.
type previewWatch struct {
ctx context.Context
js bucketOpener
ix *index.Indexer
logger *log.Logger
errs chan error
mu sync.Mutex
watched bool
}
func newPreviewWatch(ctx context.Context, js bucketOpener, ix *index.Indexer, logger *log.Logger) *previewWatch {
return &previewWatch{ctx: ctx, js: js, ix: ix, logger: logger, errs: make(chan error, 1)}
}
// ensure watches the previews bucket if it is not watched and exists, settling what it holds, then writes the candidates'
// fingerprints from it.
func (w *previewWatch) ensure() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.watched {
return nil
}
kv, err := w.js.KeyValue(w.ctx, delivery.PreviewsBucket)
switch {
case errors.Is(err, jetstream.ErrBucketNotFound):
w.logger.Printf("the bucket %s does not exist yet: it is watched when the checker says it wrote a preview", delivery.PreviewsBucket)
return nil
case err != nil:
return fmt.Errorf("the bucket %s: %w", delivery.PreviewsBucket, err)
}
watcher, err := settle(w.ctx, kv, func(e jetstream.KeyValueEntry) { w.ix.PutPreview(e.Key(), e.Value(), !opOf(e)) })
if err != nil {
return err
}
w.watched = true
go func() {
w.errs <- follow(w.ctx, watcher, func(e jetstream.KeyValueEntry) {
if err := w.ix.ApplyPreview(w.ctx, e.Key(), e.Value(), !opOf(e)); err != nil {
w.logger.Printf("%s.%s could not be taken into the candidates: %v", delivery.PreviewsBucket, e.Key(), err)
}
})
}()
return w.ix.RecomputeCandidates(w.ctx)
}
// Written takes one preview.written event of the checker in: its bucket is watched if it is not. A subject that is not the
// checker's is terminated; a failure stays unacknowledged, and the server delivers it again.
func (w *previewWatch) Written(m delivered) {
if !strings.HasPrefix(m.Subject(), previewWrittenSubject) {
w.logger.Printf("%s: not the checker's preview.written, dropped", m.Subject())
_ = m.Term()
return
}
if err := w.ensure(); err != nil {
w.logger.Printf("%s: left for redelivery: %v", m.Subject(), err)
return
}
_ = m.Ack()
}
modules/mesh-delivery-indexer/go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module mesh-delivery-indexer
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
)
modules/mesh-delivery-indexer/go.sum
+2 -2
View File
@@ -1,5 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865 h1:VLir9NZbyi1LdH1YwWDIVRuL+lDC+AYlm6FZPlvmlm4=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021100209-cb54756c3865/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc h1:ZJScJSnF6LgFl55/VJ9RT/lNroVWun469sXtx0nlVYw=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261021192148-78364e318bdc/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
modules/mesh-delivery-indexer/internal/index/candidates.go
+3 -1
View File
@@ -48,8 +48,9 @@ type Indexer struct {
Builds *Builds
Logf func(format string, args ...any)
mu sync.Mutex // guards source and toolchains, which the watches and the forge messages all reach
mu sync.Mutex // guards source, toolchains and previews, which the watches and the forge messages all reach
source sourceNow
previews map[string]delivery.Preview // by previews key (previews.go)
toolchains map[string]map[string]delivery.Toolchain // language, then node (toolchains.go)
recompute sync.Mutex // one Recompute at a time
}
@@ -102,6 +103,7 @@ func (ix *Indexer) Apply(ctx context.Context, f delivery.Forge) error {
}
next := ix.next(cur, f, digests)
if next != nil {
ix.withPreviews(ctx, next)
if err := ix.refuseDisagreement(*next); err != nil {
return nil, false, err
}
modules/mesh-delivery-indexer/internal/index/previews.go
+252
View File
@@ -0,0 +1,252 @@
package index
import (
"context"
"fmt"
"maps"
"slices"
"strings"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// The merge previews the checker writes (mesh-delivery-checker_previews; novox/hq ADR 0484 clause 3, issue 1445): a pull
// request's head merged onto its trunk's head, with the tree, the manifest digest and the lock hashes of each module
// directory. From one the indexer gives the candidate's touched module its new-form fingerprint, by the rules a module of
// the trunk is given its own (Recompute), or says why it has none.
// The reasons a candidate's touched module has no fingerprint of the new form while a preview exists or is missing.
const (
// NonePreviewStale is for a module whose preview was made onto a trunk head that is not the trunk's head now.
NonePreviewStale = "the merge preview is out of date: the trunk moved since it was made"
// NonePreviewConflict opens the reason of a module whose pull request's head does not merge into the trunk; the files
// in conflict follow.
NonePreviewConflict = "does not merge: the head conflicts with the trunk in "
// NonePreviewNoModule is for a module whose directory the preview names no manifest for.
NonePreviewNoModule = "the merge preview names no manifest for the module's directory"
// NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold.
NonePreviewNoManifest = "the manifest of the merge is not kept"
// NoneNoDirectory is for a module whose directory in its repository the builds do not tell.
NoneNoDirectory = "the module's directory in its repository is not known"
)
// PutPreview takes a change of the previews bucket in without writing any candidate: a put of a preview (key
// delivery.PreviewKey) or its delete. It answers whether the previews held changed. A key or a value that is not a
// preview is said in the log and left out.
func (ix *Indexer) PutPreview(key string, value []byte, deleted bool) bool {
if _, _, _, err := delivery.ParsePreviewKey(key); err != nil {
ix.logf("previews.%s is not a preview key, left out: %v", key, err)
return false
}
ix.mu.Lock()
defer ix.mu.Unlock()
if deleted {
if _, had := ix.previews[key]; !had {
return false
}
delete(ix.previews, key)
return true
}
pv, err := delivery.DecodePreview(value)
if err != nil {
ix.logf("previews.%s is not a preview, left out: %v", key, err)
return false
}
if ix.previews == nil {
ix.previews = map[string]delivery.Preview{}
}
if cur, had := ix.previews[key]; had && delivery.SamePreview(cur, pv) {
return false
}
ix.previews[key] = pv
return true
}
// ApplyPreview takes one change of the previews in (PutPreview) and, when the previews held changed, writes the
// candidates' fingerprints again (RecomputeCandidates).
func (ix *Indexer) ApplyPreview(ctx context.Context, key string, value []byte, deleted bool) error {
if !ix.PutPreview(key, value, deleted) {
return nil
}
return ix.RecomputeCandidates(ctx)
}
// RecomputeCandidates writes again every candidate whose pull requests' fingerprints, from the previews held, are no longer
// what it holds. A candidate is written only when it differs, by compare-and-set on the revision read.
func (ix *Indexer) RecomputeCandidates(ctx context.Context) error {
if ix.Store == nil {
return nil
}
ix.mu.Lock()
none := len(ix.previews) == 0
ix.mu.Unlock()
if none {
return nil // no preview held: every candidate has what the pull messages gave it
}
keys, err := ix.Store.Keys(ctx)
if err != nil {
return fmt.Errorf("listing the candidates: %w", err)
}
for _, key := range keys {
err := write(ctx, ix.Store, key, func(value []byte, found bool) ([]byte, bool, error) {
if !found {
return nil, false, nil
}
c, err := delivery.DecodeCandidate(value)
if err != nil {
ix.logf("candidates.%s holds what is not a candidate, left: %v", key, err)
return nil, false, nil
}
if !ix.withPreviews(ctx, &c) {
return nil, false, nil
}
data, err := delivery.EncodeCandidate(c)
return data, false, err
})
if err != nil {
return fmt.Errorf("candidates.%s: %w", key, err)
}
}
return nil
}
// previewOf is the preview of a head merged onto the trunk head the source names now: found, or stale when only a preview
// onto another trunk head exists, or neither (found false, stale false) when the source is not known or none exists.
func (ix *Indexer) previewOf(repository, head string) (pv delivery.Preview, found, stale bool) {
ix.mu.Lock()
defer ix.mu.Unlock()
trunk := ix.source.heads[repository]
if trunk == "" {
return delivery.Preview{}, false, false
}
if key, err := delivery.PreviewKey(repository, head, trunk); err == nil {
if pv, ok := ix.previews[key]; ok {
return pv, true, false
}
}
for _, pv := range ix.previews {
if pv.Repository == repository && pv.Head == head {
return delivery.Preview{}, false, true
}
}
return delivery.Preview{}, false, false
}
// withPreviews gives each touched module of the candidate's pull requests the fingerprint its preview gives it, or the
// reason it has none, and the candidate its fpset. It answers whether the candidate changed. A module of a pull request with
// no preview keeps what it had.
func (ix *Indexer) withPreviews(ctx context.Context, c *delivery.Candidate) bool {
changed := false
for i := range c.Pulls {
p := &c.Pulls[i]
pv, found, stale := ix.previewOf(p.Repository, p.Head.Head)
if !found && !stale {
continue
}
dirs := map[string]string{}
for _, k := range ix.Builds.Modules(p.Repository) {
dirs[k.Module] = k.Path
}
fps := maps.Clone(p.Fps)
for _, m := range p.Modules {
next := delivery.CandidateModule{Manifest: p.Fps[m].Manifest}
switch {
case stale:
next.None2 = NonePreviewStale
case len(pv.Conflicts) > 0:
next.None2 = NonePreviewConflict + strings.Join(pv.Conflicts, ", ")
default:
dir, known := dirs[m]
if !known {
next.None2 = NoneNoDirectory
break
}
next = ix.candidateModule(ctx, m, dir, pv)
}
if fps == nil {
fps = map[string]delivery.CandidateModule{}
}
if fps[m] != next {
fps[m] = next
changed = true
}
}
if changed || !maps.Equal(fps, p.Fps) {
p.Fps = fps
}
}
if changed {
c.FPSet = delivery.FingerprintSet(c.Pulls)
}
return changed
}
// candidateModule is the fingerprint of module m, whose directory is dir, at the preview: the module's inputs from the
// merge (tree, manifest, locks), its bases' and the builder's fingerprints as the trunk's modules hold them, and the toolchains
// the build modules agree on, by the rules Recompute applies to a module of the trunk.
func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delivery.Preview) delivery.CandidateModule {
pm, ok := pv.Modules[dir]
if !ok && dir == "" {
pm, ok = pv.Modules[delivery.RootPath]
}
if !ok {
return delivery.CandidateModule{None2: NonePreviewNoModule}
}
out := delivery.CandidateModule{Manifest: pm.Manifest}
if ix.Manifests == nil {
out.None2 = NonePreviewNoManifest
return out
}
value, _, found, err := ix.Manifests.Get(ctx, pm.Manifest)
if err != nil || !found {
out.None2 = NonePreviewNoManifest
return out
}
kept, err := delivery.DecodeManifest(value)
if err != nil {
out.None2 = NonePreviewNoManifest
return out
}
in := inputsOf(m, kept.JSON, pm.Tree, pm.Locks, ix.baseFP(ctx))
if in.Kept == nil {
out.None2 = in.None2
return out
}
if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Builder {
out.None2 = in.None2 // a context, or a base with no fingerprint
return out
}
tools := ix.agreed()
toolchains := map[string]delivery.Toolchain2{}
for _, artifact := range slices.Sorted(maps.Keys(in.Languages)) {
t := tools[in.Languages[artifact]]
if t.none != "" {
out.None2 = t.none
return out
}
toolchains[artifact] = t.tc
}
inputs := *in.Kept
if m == BuilderModule {
src, none := ix.builderInput()
if src == "" {
out.None2 = none
return out
}
inputs.Builder = src
} else {
fp, none := ix.baseFP(ctx)(BuilderModule)
if fp == "" {
out.None2 = strings.TrimSuffix(None2Builder+": "+none, ": ")
return out
}
inputs.Builder = fp
}
fp, none := delivery.SourceFingerprint2(inputs.With(m, toolchains))
if fp == "" {
out.None2 = none
return out
}
out.FP2 = fp
return out
}
modules/mesh-delivery-indexer/internal/index/previews_test.go
+193
View File
@@ -0,0 +1,193 @@
package index
import (
"context"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
const (
trunkNow = "5555555555555555555555555555555555555555"
trunkOld = "4444444444444444444444444444444444444444"
mergeSha = "6666666666666666666666666666666666666666"
)
// A trunk of the catalogue with avahi (and the builder, a base and others) merged, the toolchain agreed, and the gate's source
// naming the catalogue's trunk head: avahi has a fingerprint of the new form at tree-avahi.
func previewBed(t *testing.T, report bool) (*Indexer, *memStore, *memStore) {
t.Helper()
ix, mods := newFormIndexer(t, builderModules(t)...)
if report {
reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1")
}
if err := ix.ApplySource(context.Background(), 5, delivery.Source{Group: "g", Heads: map[string]string{catalogueRepo: trunkNow}}); err != nil {
t.Fatal(err)
}
return ix, mods, ix.Store.(*memStore)
}
// avahiOnTools is avahi's manifest as builderModules has it: standing on mesh-tools.
func avahiOnTools(t *testing.T) string {
return standingOn(t, catalogueManifest(t, "avahi"), "mesh-tools")
}
// avahiPull is the pull message of the catalogue's pull request 301 touching avahi, with its manifest.
func avahiPull(t *testing.T, head string) delivery.Forge {
t.Helper()
f := manifestPull(t, 301, head, avahiOnTools(t))
f.Manifests = map[string]string{"modules/avahi": avahiOnTools(t)}
return f
}
func avahiPreview(t *testing.T, trunk, head string) delivery.Preview {
return delivery.Preview{Repository: catalogueRepo, Head: head, Trunk: trunk, Merge: mergeSha, Modules: map[string]delivery.PreviewModule{
"modules/avahi": {Tree: "tree-avahi", Manifest: delivery.ManifestDigest([]byte(avahiOnTools(t))), Locks: map[string]string{"go.sum": avahiGoSum}}}}
}
func applyPreview(t *testing.T, ix *Indexer, pv delivery.Preview) {
t.Helper()
key, err := delivery.PreviewKey(pv.Repository, pv.Head, pv.Trunk)
if err != nil {
t.Fatal(err)
}
data, err := delivery.EncodePreview(pv)
if err != nil {
t.Fatal(err)
}
if err := ix.ApplyPreview(context.Background(), key, data, false); err != nil {
t.Fatal(err)
}
}
func fpOf(t *testing.T, st *memStore) delivery.CandidateModule {
t.Helper()
c := candidateOf(t, st)
if len(c.Pulls) != 1 {
t.Fatalf("candidate = %+v", c)
}
return c.Pulls[0].Fps["avahi"]
}
func TestACandidatesModuleGetsTheFingerprintATrunkModuleOfTheSameTreeHas(t *testing.T) { // 1445/AC2
ix, mods, cands := previewBed(t, true)
trunk, _ := moduleOf(t, mods, "avahi")
if trunk.FP2 == "" {
t.Fatalf("the trunk's avahi has no fingerprint of the new form: %+v", trunk)
}
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
if got := fpOf(t, cands); got.None2 != NoneNoPreview {
t.Fatalf("before the preview: %+v", got)
}
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
got := fpOf(t, cands)
if got.FP2 != trunk.FP2 || got.None2 != "" || got.Manifest != delivery.ManifestDigest([]byte(avahiOnTools(t))) {
t.Fatalf("candidate avahi = %+v, want the trunk's fingerprint %s", got, trunk.FP2)
}
if c := candidateOf(t, cands); c.FPSet != delivery.FingerprintSet(c.Pulls) || c.FPSet == "" {
t.Errorf("fpset = %q", c.FPSet)
}
// Another tree gives another fingerprint.
pv := avahiPreview(t, trunkNow, h1)
m := pv.Modules["modules/avahi"]
m.Tree = "tree-changed"
pv.Modules["modules/avahi"] = m
pv.Merge = "7777777777777777777777777777777777777777"
applyPreview(t, ix, pv)
if other := fpOf(t, cands); other.FP2 == "" || other.FP2 == trunk.FP2 {
t.Errorf("a changed tree: %+v", other)
}
}
func TestAnUnknownToolchainLeavesTheCandidatesModuleNamingIt(t *testing.T) { // 1445/AC2
ix, _, cands := previewBed(t, false)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
if got := fpOf(t, cands); got.FP2 != "" || !strings.Contains(got.None2, delivery.ToolchainNoReport) {
t.Errorf("candidate avahi = %+v, want none2 naming %q", got, delivery.ToolchainNoReport)
}
}
func TestAPreviewMadeOntoAnOldTrunkIsOutOfDate(t *testing.T) { // 1445/AC3
ix, _, cands := previewBed(t, true)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
applyPreview(t, ix, avahiPreview(t, trunkOld, h1))
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NonePreviewStale {
t.Errorf("candidate avahi = %+v, want it out of date", got)
}
// A preview onto the trunk head now makes it good again, and the trunk moving again makes it stale.
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
if got := fpOf(t, cands); got.FP2 == "" {
t.Fatalf("after the right preview: %+v", got)
}
if err := ix.ApplySource(context.Background(), 6, delivery.Source{Group: "g", Heads: map[string]string{catalogueRepo: "8888888888888888888888888888888888888888"}}); err != nil {
t.Fatal(err)
}
if got := fpOf(t, cands); got.None2 != NonePreviewStale {
t.Errorf("after the trunk moved: %+v", got)
}
}
func TestAPreviewHeardBeforeItsPullRequestGivesTheFingerprintWhenTheCandidateIsWritten(t *testing.T) { // 1445/AC2
ix, mods, cands := previewBed(t, true)
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
trunk, _ := moduleOf(t, mods, "avahi")
if got := fpOf(t, cands); got.FP2 != trunk.FP2 {
t.Errorf("candidate avahi = %+v, want %s", got, trunk.FP2)
}
// The same pull request at a new head has no preview of it yet.
if err := ix.Apply(context.Background(), avahiPull(t, h2)); err != nil {
t.Fatal(err)
}
if got := fpOf(t, cands); got.FP2 != "" || got.None2 != NoneNoPreview {
t.Errorf("a new head: %+v, want no preview", got)
}
}
func TestAPreviewNamingNoManifestOfTheModulesDirectoryGivesAReason(t *testing.T) {
ix, _, cands := previewBed(t, true)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
pv := avahiPreview(t, trunkNow, h1)
pv.Modules = map[string]delivery.PreviewModule{"modules/other": pv.Modules["modules/avahi"]}
applyPreview(t, ix, pv)
if got := fpOf(t, cands); got.None2 != NonePreviewNoModule || !strings.Contains(NonePreviewNoModule, "manifest") {
t.Errorf("candidate avahi = %+v", got)
}
}
func TestTheSamePreviewAppliedAgainWritesNothing(t *testing.T) {
ix, _, cands := previewBed(t, true)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
w := cands.writes
applyPreview(t, ix, avahiPreview(t, trunkNow, h1))
if cands.writes != w {
t.Errorf("candidates written %d times for the same preview, want none", cands.writes-w)
}
}
func TestAPreviewOfConflictsGivesTheCandidatesModuleTheReasonItDoesNotMerge(t *testing.T) { // 1445/AC5
ix, _, cands := previewBed(t, true)
if err := ix.Apply(context.Background(), avahiPull(t, h1)); err != nil {
t.Fatal(err)
}
applyPreview(t, ix, delivery.Preview{Repository: catalogueRepo, Head: h1, Trunk: trunkNow, Conflicts: []string{"modules/avahi/main.go", "go.mod"}})
got := fpOf(t, cands)
if got.FP2 != "" || !strings.HasPrefix(got.None2, "does not merge") || !strings.Contains(got.None2, "go.mod, modules/avahi/main.go") && !strings.Contains(got.None2, "modules/avahi/main.go, go.mod") {
t.Errorf("candidate avahi = %+v, want it not merging and naming the files", got)
}
}
modules/mesh-delivery-indexer/internal/index/source.go
+3 -2
View File
@@ -44,7 +44,7 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
ix.source = sourceNow{rev: rev, heads: maps.Clone(s.Heads)}
ix.mu.Unlock()
if ix.Modules == nil {
return nil
return ix.RecomputeCandidates(ctx)
}
repositories := slices.Sorted(maps.Keys(s.Heads))
for _, repository := range repositories {
@@ -76,5 +76,6 @@ func (ix *Indexer) ApplySource(ctx context.Context, rev uint64, s delivery.Sourc
}
}
}
return nil
// A trunk that moved makes the previews made onto the old one out of date.
return ix.RecomputeCandidates(ctx)
}

Some files were not shown because too many files have changed in this diff Show More