diff --git a/merge-check.sh b/merge-check.sh deleted file mode 100644 index af75a30..0000000 --- a/merge-check.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -# mesh-check-toolchain: go -# -# This repository's own check, mesh/repo-check (novox/hq ADR 0238 §4), written with its first commit (ADR 0428 §4): -# every Go module in the tree is vetted, its unit tests run, and then its integration tests (-tags integration). -# A tree with no Go module has nothing to test, and the check says so and passes. -# -# No module commits its dependencies (no vendor directory: .gitignore). A module's dependencies are fetched, or a -# requirement of the mesh's own forge (the pinned mesh-sdk commit) is taken from its repository's clone beside the check -# (MESH_CHECK_BESIDE), through a go.mod of its own (-modfile), by mesh-catalog's helper checks/deps, built from the -# clone of mesh-catalog beside this check and not copied here: the build seat forgets the forge credential before the -# check runs, so a commit the seat's module cache does not hold cannot be fetched from the forge (novox/hq issue 385). A -# module whose dependencies cannot be had **fails the check**, by name and why: its tests could not run, and a test that -# could not run is a failure, never a note. Run by hand, without MESH_CHECK_BESIDE, the dependencies are fetched. -set -eu -cd "$(dirname "$0")" - -# The toolchain the check runs in sets -mod=vendor; there is no vendor directory, so the modules are used as they are. -GOFLAGS="$(printf '%s' "${GOFLAGS:-}" | sed 's/-mod=[a-z]*//g') -mod=mod" -export GOFLAGS -export GOPRIVATE=git.novox.be - -modules=$(find . -name go.mod -not -path '*/vendor/*' -not -path './.git/*' | sort) -if [ -z "$modules" ]; then - echo "This tree holds no Go module, so there are no tests to run; the check passes." - exit 0 -fi - -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT -deps="" -helper="${MESH_CHECK_BESIDE:-}/mesh-catalog/checks/deps" -if [ -n "${MESH_CHECK_BESIDE:-}" ] && [ -f "$helper/main.go" ]; then - if ! (cd "$helper" && go build -o "$work/deps" .); then - echo "FAILED: the dependency helper of mesh-catalog beside this check could not be built" - exit 1 - fi - deps="$work/deps" -elif [ -n "${MESH_CHECK_BESIDE:-}" ]; then - echo "NOT TAKEN FROM BESIDE: no mesh-catalog/checks/deps beside this check ($MESH_CHECK_BESIDE); the dependencies are fetched" -fi - -untestable="" -for mod in $modules; do - dir=$(dirname "$mod") - modfile="" - if [ -n "$deps" ]; then - if ! modfile=$("$deps" -beside "$MESH_CHECK_BESIDE" -work "$work" "$dir"); then - echo "FAILED: $dir could not be tested: $modfile" - untestable="$untestable $dir" - continue - fi - [ -n "$modfile" ] && echo "$dir takes its forge dependencies from the clones beside the check" - elif ! fetched=$(cd "$dir" && go mod download 2>&1); then - echo "FAILED: $dir could not be tested: its dependencies cannot be fetched: $(printf '%s\n' "$fetched" | tail -n 1)" - untestable="$untestable $dir" - continue - fi - mf="" - [ -n "$modfile" ] && mf="-modfile=$modfile" - unformatted=$(cd "$dir" && gofmt -l .) - if [ -n "$unformatted" ]; then - echo "$dir is not gofmt'd: $unformatted" - exit 1 - fi - echo "== $dir: go vet and the unit tests" - (cd "$dir" && go vet $mf ./... && go test $mf ./...) - echo "== $dir: the integration tests (-tags integration)" - (cd "$dir" && go vet $mf -tags integration ./... && go test $mf -tags integration ./...) -done -if [ -n "$untestable" ]; then - echo "the check fails: the tests of$untestable could not run (novox/hq issue 385)" - exit 1 -fi diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go new file mode 100644 index 0000000..6a79f7c --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/integration_test.go @@ -0,0 +1,302 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/provider" +) + +// bed is an embedded bus with JetStream, every bucket the assigner reads and keeps, and the stream the mesh's events +// are kept in, as the controller's broker raises them. +type bed struct { + srv *server.Server + js jetstream.JetStream + ctx context.Context + dir string + buckets map[string]jetstream.KeyValue +} + +func startBed(t *testing.T, leaveOut string) *bed { + t.Helper() + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "assigner") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(srv.Shutdown) + nc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, err := jetstream.New(nc) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + t.Cleanup(cancel) + b := &bed{srv: srv, js: js, ctx: ctx, dir: dir, buckets: map[string]jetstream.KeyValue{}} + for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { + if name == leaveOut { + continue + } + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: 5}) + if err != nil { + t.Fatal(err) + } + b.buckets[name] = kv + } + if _, err := js.CreateStream(ctx, jetstream.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}}); err != nil { + t.Fatal(err) + } + return b +} + +// start runs the program as the node does, through run with its environment and a broker file; it answers the +// channel run's error arrives on. +func (b *bed) start(t *testing.T) <-chan error { + t.Helper() + broker := filepath.Join(b.dir, "broker") + cred, _ := json.Marshal(process.Account{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-assigner", User: "n1.mesh-delivery-assigner", Password: "pw"}) + if err := os.WriteFile(broker, cred, 0o600); err != nil { + t.Fatal(err) + } + env := map[string]string{"MESH_BROKER_FILE": broker} + ctx, stop := context.WithCancel(b.ctx) + done := make(chan error, 1) + go func() { done <- run(ctx, func(k string) string { return env[k] }, io.Discard) }() + t.Cleanup(func() { + stop() + select { + case <-done: + case <-time.After(10 * time.Second): + } + }) + return done +} + +// 1297/B2b: started whole on its credential, the assigner says on the bus the four states it keeps, each readable, +// having opened every bucket it names. +func TestTheAssignerStartsAndSaysWhatItKeeps(t *testing.T) { + b := startBed(t, "") + done := b.start(t) + events, err := b.js.Stream(b.ctx, "EVENTS") + if err != nil { + t.Fatal(err) + } + var said announce.States + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + select { + case err := <-done: + t.Fatalf("the assigner stopped: %v", err) + default: + } + m, err := events.GetLastMsgForSubject(b.ctx, announce.Subject("mesh-delivery-assigner")) + if err != nil { + continue + } + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + break + } + if said.Module != "mesh-delivery-assigner" || said.Node != "n1" || len(said.States) != 4 || !said.Readable("asks") || + !said.Readable("wants") || !said.Readable("differences") || !said.Readable("compositions") { + t.Fatalf("the assigner said %+v, want its four states, each readable", said) + } +} + +// 1297/B2b: a bucket it reads that the bus does not hold stops the start, naming the bucket. +func TestTheAssignerDoesNotStartWithoutABucketItReads(t *testing.T) { + for _, missing := range readBuckets { + b := startBed(t, missing) + select { + case err := <-b.start(t): + if err == nil || !strings.Contains(err.Error(), missing) { + t.Errorf("without %s the assigner stopped with %v, want the bucket named", missing, err) + } + case <-time.After(15 * time.Second): + t.Errorf("without %s the assigner started", missing) + } + } +} + +// putJSON writes a value into one of the bed's buckets. +func (b *bed) putJSON(t *testing.T, bucket, key string, value []byte) { + t.Helper() + if _, err := b.buckets[bucket].Put(b.ctx, key, value); err != nil { + t.Fatalf("%s %s: %v", bucket, key, err) + } +} + +// manifest keeps a manifest in the indexer's bucket by its digest and answers the digest. +func (b *bed) manifest(t *testing.T, module, body string) string { + t.Helper() + digest := delivery.ManifestDigest([]byte(body)) + key, _ := delivery.ManifestsKey(digest) + value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body}) + if err != nil { + t.Fatal(err) + } + b.putJSON(t, delivery.ManifestsBucket, key, value) + return digest +} + +// 1297/AC1: against the real bus, a fixture change touching one module on one machine is composed and answered within +// seconds under the checker's key of the check, naming the machine composed and the mesh-sdk it composed with. +func TestAChangeIsComposedOnTheBus(t *testing.T) { + b := startBed(t, "") + trunk := b.manifest(t, "web", `{"module":"web","version":"1"}`) + changed := b.manifest(t, "web", `{"module":"web","version":"2"}`) + mod, err := delivery.EncodeModule(delivery.Module{FP2: delivery.SourcePrefix2 + strings.Repeat("a", 64), Manifest: trunk, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + if err != nil { + t.Fatal(err) + } + b.putJSON(t, delivery.ModulesBucket, "web", mod) + b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"build-1"}`)) + b.putJSON(t, provider.AssignmentsBucket, "ace.other", []byte(`{"node":"ace","module":"other","build":"build-2"}`)) + heads := []delivery.Head{{Repository: "novox/web", Number: 7, Head: strings.Repeat("2", 40)}} + hash, _ := delivery.HeadsHash(heads) + cand, err := delivery.EncodeCandidate(delivery.Candidate{Group: "feat/x", HeadsHash: hash, Pulls: []delivery.CandidatePull{{Head: heads[0], + Modules: []string{"web"}, Fps: map[string]delivery.CandidateModule{"web": {FP2: delivery.SourcePrefix2 + strings.Repeat("b", 64), Manifest: changed}}}}}) + if err != nil { + t.Fatal(err) + } + ckey, _ := delivery.CandidatesKey("feat/x") + b.putJSON(t, delivery.CandidatesBucket, ckey, cand) + began := time.Now() + b.start(t) + key, _ := delivery.CompositionKey("feat/x", hash) + var v delivery.CompositionVerdict + for deadline := time.Now().Add(10 * time.Second); ; time.Sleep(50 * time.Millisecond) { + if time.Now().After(deadline) { + t.Fatalf("no composition under %s after 10 s", key) + } + e, err := b.buckets[delivery.AssignerCompositionsBucket].Get(b.ctx, key) + if err != nil { + continue + } + if v, err = delivery.DecodeCompositionVerdict(e.Value()); err != nil { + t.Fatal(err) + } + break + } + if v.State != delivery.CompositionPass || len(v.Machines) != 1 || v.Machines[0] != "g14" || v.SDK == "" { + t.Fatalf("composition = %+v, want a pass on g14 alone naming its mesh-sdk", v) + } + if took := time.Since(began); took > 5*time.Second { + t.Errorf("the composition took %s", took) + } +} + +// get waits for a key of one of the bed's buckets and answers its value. +func (b *bed) get(t *testing.T, bucket, key string) []byte { + t.Helper() + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err := b.buckets[bucket].Get(b.ctx, key); err == nil { + return e.Value() + } + } + t.Fatalf("no %s %s after 10 s", bucket, key) + return nil +} + +// 1297/B4, B5 on the bus: the controller sent F to g14 while the canary ace does not run F healthy yet. Started whole, +// the assigner wants F on the canary, writes the difference on g14 explained by rule 3, and on ace one unexplained (the +// controller sends ace the older fingerprint while ace wants F). +func TestWantsAndTheirDifferencesOnTheBus(t *testing.T) { + b := startBed(t, "") + fNew, fOld := delivery.SourcePrefix2+strings.Repeat("b", 64), delivery.SourcePrefix2+strings.Repeat("a", 64) + digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, "web", mod) + built, err := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, By: "builder.novox.1", Step: "pushed", Manifest: digest}) + if err != nil { + t.Fatal(err) + } + bkey, _ := delivery.BuilderBuildKey("web", fNew) + b.putJSON(t, delivery.BuilderBuildsBucket, bkey, built) + gkey, _ := delivery.ControllerGrantKey("web", fNew) + grant, _ := delivery.EncodeControllerGrant(delivery.ControllerGrant{}) + b.putJSON(t, delivery.ControllerGrantsBucket, gkey, grant) + for _, node := range []string{"ace", "g14"} { + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + b.putJSON(t, delivery.PresenceBucket, node, on) + have, _ := delivery.EncodeHave(delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) + b.putJSON(t, delivery.HaveBucket, node+".web", have) + } + b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-8"}`)) + b.putJSON(t, provider.AssignmentsBucket, "g14.web", []byte(`{"node":"g14","module":"web","build":"b-9"}`)) + b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-8"), []byte(`{"id":"b-8","repository":"novox/web","source-fingerprint":"`+fOld+`"}`)) + b.putJSON(t, delivery.BuildsBucket, delivery.BuildKey("web", "b-9"), []byte(`{"id":"b-9","repository":"novox/web","source-fingerprint":"`+fNew+`"}`)) + b.start(t) + + wkey, _ := delivery.WantKey("ace", "web") + want, err := delivery.DecodeWant(b.get(t, delivery.AssignerWantsBucket, wkey)) + if err != nil || want.FP != fNew { + t.Fatalf("the canary ace's want = %+v, %v", want, err) + } + dkey, _ := delivery.DifferenceKey("g14", "web") + d, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, dkey)) + if err != nil || d.Rule != delivery.RuleCanary || d.Sent != fNew { + t.Fatalf("g14's difference = %+v, %v; want rule 3", d, err) + } + akey, _ := delivery.DifferenceKey("ace", "web") + a, err := delivery.DecodeDifference(b.get(t, delivery.AssignerDifferencesBucket, akey)) + if err != nil || !a.Unexplained || a.Want != fNew || a.Sent != fOld { + t.Fatalf("ace's difference = %+v, %v; want it unexplained", a, err) + } + if e, err := b.buckets[delivery.AssignerWantsBucket].Get(b.ctx, "want.g14.web"); err == nil { + t.Errorf("g14, held back by the canary with no want before, was given one: %s", e.Value()) + } +} + +// 1297/B9 (the assigner's half, ADR 0528 clause 6): started whole, the assigner holds a fingerprint whose health check +// failed on a node and writes the ask about it in its own asks, which the view and the teller read; its asks bucket is +// the one its manifest declares (ownBuckets, held equal to the manifest by TestOpensWhatItsManifestNames). +func TestAHeldFingerprintIsAskedAbout(t *testing.T) { + b := startBed(t, "") + fNew := delivery.SourcePrefix2 + strings.Repeat("c", 64) + digest := b.manifest(t, "web", `{"module":"web","version":"1"}`) + mod, _ := delivery.EncodeModule(delivery.Module{FP2: fNew, Manifest: digest, Repository: "novox/web", Commit: strings.Repeat("1", 40), Path: delivery.RootPath}) + b.putJSON(t, delivery.ModulesBucket, "web", mod) + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + b.putJSON(t, delivery.PresenceBucket, "ace", on) + failed, _ := delivery.EncodeHave(delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"}) + b.putJSON(t, delivery.HaveBucket, "ace.web", failed) + b.putJSON(t, provider.AssignmentsBucket, "ace.web", []byte(`{"node":"ace","module":"web","build":"b-1"}`)) + b.start(t) + hkey, _ := delivery.HeldKey("web", fNew) + if h, err := delivery.DecodeHeld(b.get(t, delivery.AssignerWantsBucket, hkey)); err != nil || h.Node != "ace" { + t.Fatalf("held = %+v, %v", h, err) + } + akey, _ := delivery.HoldAskKey("web", fNew) + a, err := delivery.DecodeAssignerAsk(akey, b.get(t, delivery.AssignerAsksBucket, akey)) + if err != nil || a.Ask.ID != delivery.AskingID(akey, 0) || len(a.Ask.Options) != 2 { + t.Fatalf("the ask = %+v, %v", a, err) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go new file mode 100644 index 0000000..8900bd8 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main.go @@ -0,0 +1,225 @@ +// Command mesh-delivery-assigner is delivery's assigner (novox/hq ADR 0429 §1 and §6, ADR 0446, ADR 0528, issue 1297): a +// process on its own bus credential that watches what the indexer, the builder, the controller, the node-engines and +// the watchman write, and writes what each node should run (`want..`, by the six rules), the +// fingerprints it holds, every difference between a want and what the controller sends, and the composition check's +// verdict of each group the checker judges. In release 5 the wants are only compared: the controller still sends what +// each node runs (ADR 0429 §7). It holds no timer: every pass is started by a change it watched. +package main + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/signal" + "strings" + "syscall" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/process" + "git.novox.be/novox/mesh-sdk/go/provider" +) + +func main() { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + if err := run(ctx, os.Getenv, os.Stderr); err != nil { + fmt.Fprintln(os.Stderr, "mesh-delivery-assigner:", err) + os.Exit(1) + } +} + +// readBuckets are the buckets of other parts the assigner reads, each named in its manifest's reads (the manifest test +// holds the two lists equal), so the controller's broker grants exactly these. +var readBuckets = []string{ + provider.AssignmentsBucket, + delivery.BuildsBucket, + delivery.ControllerGrantsBucket, + delivery.BuilderBuildsBucket, + delivery.CandidatesBucket, + delivery.ManifestsBucket, + delivery.ModulesBucket, + delivery.PresenceBucket, + delivery.HaveBucket, +} + +// ownBuckets are the assigner's own states, as its manifest declares them. +var ownBuckets = []string{delivery.AssignerAsksBucket, delivery.AssignerCompositionsBucket, delivery.AssignerDifferencesBucket, delivery.AssignerWantsBucket} + +// keptStates is what the assigner keeps and who may read it (ADR 0495 §2-§3): its wants by the watchman and the view +// (and the node agent from release 6), its differences by the view, its compositions by the checker, its asks by the view +// and the teller. +var keptStates = []announce.State{ + {Name: "asks", Readable: true}, + {Name: "compositions", Readable: true}, + {Name: "differences", Readable: true}, + {Name: "wants", Readable: true}, +} + +// buckets is every bucket the assigner opened, by name. +type buckets map[string]jetstream.KeyValue + +// open opens every bucket the assigner reads and writes; a bucket it cannot open stops the start, named, since a pass +// without it would write wants from half the mesh. +func open(ctx context.Context, js jetstream.JetStream) (buckets, error) { + out := buckets{} + for _, name := range append(append([]string{}, readBuckets...), ownBuckets...) { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return nil, fmt.Errorf("the bucket %s cannot be opened: %w", name, err) + } + out[name] = kv + } + return out, nil +} + +// run is the program: it returns when ctx ends or the bus cannot be used. +func run(ctx context.Context, getenv func(string) string, logOut io.Writer) error { + logger := log.New(logOut, "mesh-delivery-assigner: ", log.LstdFlags) + path := strings.TrimSpace(getenv("MESH_BROKER_FILE")) + if path == "" { + return errors.New("MESH_BROKER_FILE is not set, so there is no credential to read the bus with") + } + account, err := process.ReadAccount(path) + if err != nil { + return err + } + opts := []nats.Option{ + nats.Name(account.Node + "." + account.Module), + nats.UserInfo(account.User, account.Password), + nats.CustomInboxPrefix(account.InboxPrefix()), + nats.MaxReconnects(-1), + nats.RetryOnFailedConnect(true), + } + if strings.TrimSpace(account.Fingerprint) != "" { + opts = append(opts, nats.Secure(process.PinnedToFingerprint(account.Fingerprint))) + } + conn, err := nats.Connect(account.URL, opts...) + if err != nil { + return fmt.Errorf("connecting to the bus: %w", err) + } + defer conn.Close() + js, err := jetstream.New(conn) + if err != nil { + return err + } + kvs, err := open(ctx, js) + if err != nil { + return err + } + // What the assigner keeps, said on the bus (ADR 0495 §2-§3): a reader is granted from it. Not fatal when it cannot + // be said now: process.Conn says it again at every reconnect. + said, err := process.Connect(account, nats.Name(account.Node+"."+account.Module+".keeps"), nats.RetryOnFailedConnect(true)) + if err != nil { + return err + } + defer said.Close() + if err := said.Keeps(keptStates...); err != nil { + logger.Printf("what the assigner keeps could not be said on the bus, so its readers are not granted until it is: %v", err) + } + logger.Printf("started on the account %s, reading %d buckets", account.User, len(kvs)) + return follow(ctx, kvs, logger) +} + +// watched are the buckets whose every value a pass reads; the manifests are read by digest when a pass needs one, and +// watched only for keys, so a manifest that arrives wakes a composition that waits for it. +var watched = []string{ + provider.AssignmentsBucket, + delivery.ControllerGrantsBucket, + delivery.BuilderBuildsBucket, + delivery.CandidatesBucket, + delivery.ModulesBucket, + delivery.PresenceBucket, + delivery.HaveBucket, + delivery.AssignerAsksBucket, + delivery.AssignerCompositionsBucket, + delivery.AssignerDifferencesBucket, + delivery.AssignerWantsBucket, +} + +// follow watches every bucket a pass reads and makes a pass at start, once every watch handed what stands, and after +// each change; changes that come during a pass make one more. It returns when ctx ends or a watch ends. +func follow(ctx context.Context, kvs buckets, logger *log.Logger) error { + w := newWorld() + dirty := make(chan struct{}, 1) + wake := func() { + select { + case dirty <- struct{}{}: + default: + } + } + errs := make(chan error, len(watched)+1) + settled := make(chan struct{}, len(watched)+1) + watch := func(name string, opts ...jetstream.WatchOpt) error { + kw, err := kvs[name].WatchAll(ctx, opts...) + if err != nil { + return fmt.Errorf("the bucket %s cannot be watched: %w", name, err) + } + go func() { + defer kw.Stop() + for { + select { + case <-ctx.Done(): + errs <- nil + return + case e, ok := <-kw.Updates(): + if !ok { + errs <- fmt.Errorf("the watch of %s ended", name) + return + } + if e == nil { + settled <- struct{}{} + continue + } + var value []byte + if e.Operation() == jetstream.KeyValuePut { + value = e.Value() + } + w.put(name, e.Key(), value) + wake() + } + } + }() + return nil + } + for _, name := range watched { + var opts []jetstream.WatchOpt + if name == delivery.HaveBucket { + opts = append(opts, jetstream.IncludeHistory()) // what each node ran healthy before, for a set-back + } + if err := watch(name, opts...); err != nil { + return err + } + } + if err := watch(delivery.ManifestsBucket, jetstream.MetaOnly()); err != nil { + return err + } + for range len(watched) + 1 { + select { + case <-ctx.Done(): + return nil + case err := <-errs: + return err + case <-settled: + } + } + p := &passer{kvs: kvs, w: w, logf: logger.Printf} + wake() + for { + select { + case <-ctx.Done(): + return nil + case err := <-errs: + return err + case <-settled: + case <-dirty: + p.pass(ctx) + } + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go new file mode 100644 index 0000000..f52e1fb --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/main_test.go @@ -0,0 +1,59 @@ +package main + +import ( + "context" + "io" + "os" + "path/filepath" + "reflect" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// 1297/B2a: the buckets the program opens are exactly the manifest's reads and states, as the broker names them +// (`.` read as the bucket `_`), so it is granted what it reads and reads nothing else. +func TestOpensWhatItsManifestNames(t *testing.T) { + raw, err := os.ReadFile(filepath.Join("..", "..", "module.json")) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + var reads []string + for _, r := range m.Reads { + reads = append(reads, strings.Replace(r, ".", "_", 1)) + } + var states []string + for _, s := range m.State { + states = append(states, m.Module+"_"+s.Name) + } + for _, pair := range []struct { + what string + code, want []string + }{{"reads", readBuckets, reads}, {"states", ownBuckets, states}} { + got, want := slices.Sorted(slices.Values(pair.code)), slices.Sorted(slices.Values(pair.want)) + if !reflect.DeepEqual(got, want) { + t.Errorf("the program's %s are %v, the manifest's %v", pair.what, got, want) + } + } + var kept []string + for _, s := range keptStates { + kept = append(kept, m.Module+"_"+s.Name) + } + if got, want := slices.Sorted(slices.Values(kept)), slices.Sorted(slices.Values(ownBuckets)); !reflect.DeepEqual(got, want) { + t.Errorf("the program announces %v, keeps %v", got, want) + } +} + +// 1297/B2a: without a credential the program does not start, and says why. +func TestNoCredential(t *testing.T) { + err := run(context.Background(), func(string) string { return "" }, io.Discard) + if err == nil || !strings.Contains(err.Error(), "MESH_BROKER_FILE") { + t.Errorf("run without a credential = %v", err) + } +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go new file mode 100644 index 0000000..a626f95 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/pass.go @@ -0,0 +1,222 @@ +package main + +import ( + "context" + "errors" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/sdkversion" + + "mesh-delivery-assigner/internal/assign" +) + +// passer makes the assigner's passes: from what the world holds now, it writes each key whose value differs from the +// one written, and only those (the Same functions). +type passer struct { + kvs buckets + w *world + logf func(string, ...any) + sdk string +} + +func (p *passer) pass(ctx context.Context) { + p.compositions(ctx) + rules := p.wants(ctx) + p.differences(ctx, rules) +} + +// differences writes, for each node and module the assigner wants or the controller sends, the difference between the +// two with the rule that explains it or unexplained (ADR 0528 clause 4), and deletes the key of a difference that +// ended; its history keeps what it was. +func (p *passer) differences(ctx context.Context, rules map[string]map[string]assign.Holding) { + _, assignments := p.w.assigned() + wanted := p.w.wantsOf() + trunk := p.w.trunk() + pairs := map[[2]string]bool{} + for _, a := range assignments { + pairs[[2]string{a.Node, a.Module}] = true + } + for module, byNode := range wanted { + for node := range byNode { + pairs[[2]string{node, module}] = true + } + } + written := p.w.read(delivery.AssignerDifferencesBucket) + keep := map[string]bool{} + for pair := range pairs { + node, module := pair[0], pair[1] + key, err := delivery.DifferenceKey(node, module) + if err != nil { + continue + } + var want *delivery.Want + if w, ok := wanted[module][node]; ok { + want = &w + } + var sent assign.Sent + if a, ok := assignments[node+"."+module]; ok { + sent = assign.Sent{Sends: true, Build: a.Build, FP: p.w.sentFP(ctx, kvGetter(p.kvs[delivery.BuildsBucket]), module, a.Build)} + } + d, ok := assign.Compare(want, sent, trunk[module].FP2, rules[module][node]) + if !ok { + continue + } + keep[key] = true + if before, err := delivery.DecodeDifference(written[key]); err == nil { + before.V = 0 + if delivery.SameDifference(before, d) { + continue + } + } + p.put(ctx, delivery.AssignerDifferencesBucket, key, func() ([]byte, error) { return delivery.EncodeDifference(d) }) + } + for key := range written { + if keep[key] { + continue + } + if err := p.kvs[delivery.AssignerDifferencesBucket].Delete(ctx, key); err != nil { + p.logf("the difference %s that ended could not be deleted: %v", key, err) + continue + } + p.w.put(delivery.AssignerDifferencesBucket, key, nil) + } +} + +// wants writes, for every module the trunk holds a fingerprint of, each node's want by the six rules (ADR 0429 §6), and +// the hold of a fingerprint whose health check failed. It returns, by module then node, the rule that holds a node +// back, for the comparison. +func (p *passer) wants(ctx context.Context) map[string]map[string]assign.Holding { + get := kvGetter(p.kvs[delivery.ManifestsBucket]) + assigned, _ := p.w.assigned() + trunk := p.w.trunk() + manifests := map[string]catalogue.Manifest{} + for name, mod := range trunk { + if mod.Manifest == "" { + continue + } + if got := p.w.manifest(ctx, get, mod.Manifest); got.err == "" { + manifests[name] = got.manifest + } + } + nodes := p.w.nodes(manifests, assigned) + nodesOf := map[string][]string{} + for node, modules := range assigned { + for _, m := range modules { + nodesOf[m] = append(nodesOf[m], node) + } + } + wanted := p.w.wantsOf() + rules := map[string]map[string]assign.Holding{} + names := map[string]bool{} + for name := range trunk { + names[name] = true + } + for name := range wanted { + names[name] = true + } + for name := range names { + fp := trunk[name].FP2 + m := assign.Module{Name: name, FP: fp, Manifest: manifests[name], Nodes: nodesOf[name]} + if fp != "" { + held, _ := delivery.HeldKey(name, fp) + grant, _ := delivery.ControllerGrantKey(name, fp) + m.Built, m.Held, m.Granted = p.w.built(name, fp), p.w.has(delivery.AssignerWantsBucket, held), p.w.has(delivery.ControllerGrantsBucket, grant) + } + last := map[string]string{} + for _, node := range m.Nodes { + last[node] = p.w.lastHealthy(node, name, fp) + } + out := assign.Wants(m, nodes, wanted[name], last) + rules[name] = out.Rules + if out.Held != nil { + key, err := delivery.HeldKey(name, fp) + if err == nil { + held := *out.Held + p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeHeld(held) }) + } + } + if out.Held != nil || m.Held { + p.holdAsk(ctx, name, fp) + } + for node, want := range out.Wants { + key, err := delivery.WantKey(node, name) + if err != nil { + p.logf("no want of %s on %s is written: %v", name, node, err) + continue + } + want := want + p.put(ctx, delivery.AssignerWantsBucket, key, func() ([]byte, error) { return delivery.EncodeWant(want) }) + } + } + return rules +} + +// compositions writes the composition check's verdict of every group the indexer holds a candidate for, under the +// checker's key of that check: waits, with why, while the indexer holds what it composes from only in part. +func (p *passer) compositions(ctx context.Context) { + if p.sdk == "" { + p.sdk = sdkversion.Version() + } + written := p.w.read(delivery.AssignerCompositionsBucket) + trunk := p.w.trunk() + var mesh *assign.Mesh + for key, value := range p.w.read(delivery.CandidatesBucket) { + c, err := delivery.DecodeCandidate(value) + if err != nil { + p.logf("the candidate %s cannot be read, so it is not composed: %v", key, err) + continue + } + out, err := delivery.CompositionKey(c.Group, c.HeadsHash) + if err != nil { + p.logf("the candidate %s names no composition key: %v", key, err) + continue + } + began := time.Now() + ch, waits := p.w.change(ctx, kvGetter(p.kvs[delivery.ManifestsBucket]), c, trunk) + var v delivery.CompositionVerdict + if waits != "" { + v = delivery.CompositionVerdict{Group: c.Group, Heads: c.HeadsHash, State: delivery.CompositionWaits, Words: waits} + } else { + if mesh == nil { + m := p.w.mesh(ctx, kvGetter(p.kvs[delivery.ManifestsBucket])) + mesh = &m + } + v = assign.Compose(*mesh, ch, p.sdk, time.Since(began)) + } + if before, err := delivery.DecodeCompositionVerdict(written[out]); err == nil { + before.Took, before.V = v.Took, 0 + if delivery.SameCompositionVerdict(before, v) { + continue + } + } + p.put(ctx, delivery.AssignerCompositionsBucket, out, func() ([]byte, error) { return delivery.EncodeCompositionVerdict(v) }) + } +} + +// put writes one key of the assigner's own bucket; a write that fails is said and made again at the next pass. +func (p *passer) put(ctx context.Context, bucket, key string, encode func() ([]byte, error)) { + value, err := encode() + if err != nil { + p.logf("%s %s is not written: %v", bucket, key, err) + return + } + if _, err := p.kvs[bucket].Put(ctx, key, value); err != nil && !errors.Is(err, context.Canceled) { + p.logf("%s %s could not be written: %v", bucket, key, err) + return + } + p.w.put(bucket, key, value) +} + +// holdAsk writes, once, the ask about a held fingerprint (ADR 0528 clause 6, ADR 0437 clause 10): the first ask that needs +// no proof of who answered, which the view may answer and the teller fills. In release 5 an answer changes nothing on a +// node (wants are only compared, ADR 0429 §7), and the options say so. +func (p *passer) holdAsk(ctx context.Context, module, fp string) { + key, err := delivery.HoldAskKey(module, fp) + if err != nil || p.w.has(delivery.AssignerAsksBucket, key) { + return + } + a := assign.HoldAsk(key, module, fp, time.Now()) + p.put(ctx, delivery.AssignerAsksBucket, key, func() ([]byte, error) { return delivery.EncodeAssignerAsk(key, a) }) +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go new file mode 100644 index 0000000..fcd91d8 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world.go @@ -0,0 +1,370 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" + "git.novox.be/novox/mesh-sdk/go/provider" + + "mesh-delivery-assigner/internal/assign" +) + +// noModuleWords is the indexer's reason for a module of a candidate whose directory the merge preview holds no manifest +// for (mesh-sdk delivery.CandidateNoneNoModule, which the indexer writes): the change removes the module. +const noModuleWords = delivery.CandidateNoneNoModule + +// assignment holds only the fields of the controller's Assignment the assigner reads (the view's own, mesh-delivery-view +// internal/view/view.go): an assignment stands while it has not ended and has no state. +type assignment struct { + Node string `json:"node"` + Module string `json:"module"` + Build string `json:"build"` + Ended time.Time `json:"ended"` + State string `json:"state"` +} + +func (a assignment) open() bool { return a.Ended.IsZero() && a.State == "" } + +// world is what the assigner watched, as it stands now: every value by bucket and key, and the manifests read by +// digest. The watches write it; a pass reads it. +type world struct { + mu sync.Mutex + values map[string]map[string][]byte + manifests map[string]parsed + // healthy is, by have key, every fingerprint the node ran healthy, oldest first, each once: what a set-back goes to. + healthy map[string][]string + // sent is, by build id, the fingerprint of the new form the controller's build record names, once it names one. + sent map[string]string +} + +// parsed is a manifest read by digest: its module and its manifest, or why it cannot be read. +type parsed struct { + module string + manifest catalogue.Manifest + err string +} + +func newWorld() *world { + return &world{values: map[string]map[string][]byte{}, manifests: map[string]parsed{}, healthy: map[string][]string{}, sent: map[string]string{}} +} + +// put takes a change of a watched bucket in: a value, or nil for a key deleted or purged. +func (w *world) put(bucket, key string, value []byte) { + w.mu.Lock() + defer w.mu.Unlock() + if w.values[bucket] == nil { + w.values[bucket] = map[string][]byte{} + } + if value == nil { + delete(w.values[bucket], key) + return + } + w.values[bucket][key] = value + if bucket == delivery.HaveBucket { + if h, err := delivery.DecodeHave(value); err == nil && h.FP != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK { + list := slices.DeleteFunc(w.healthy[key], func(fp string) bool { return fp == h.FP }) + w.healthy[key] = append(list, h.FP) + } + } +} + +// lastHealthy is the newest fingerprint other than fp the node ran the module healthy at, or empty. +func (w *world) lastHealthy(node, module, fp string) string { + key, err := delivery.HaveKey(node, module) + if err != nil { + return "" + } + w.mu.Lock() + defer w.mu.Unlock() + list := w.healthy[key] + for i := len(list) - 1; i >= 0; i-- { + if list[i] != fp { + return list[i] + } + } + return "" +} + +// read is a copy of what a bucket holds now. +func (w *world) read(bucket string) map[string][]byte { + w.mu.Lock() + defer w.mu.Unlock() + out := make(map[string][]byte, len(w.values[bucket])) + for k, v := range w.values[bucket] { + out[k] = v + } + return out +} + +// getter reads one key's value; errNotThere when the key does not exist. +type getter func(ctx context.Context, key string) ([]byte, error) + +var errNotThere = errors.New("no such key") + +// kvGetter reads keys of a bucket. +func kvGetter(kv jetstream.KeyValue) getter { + return func(ctx context.Context, key string) ([]byte, error) { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return nil, errNotThere + } + if err != nil { + return nil, err + } + return e.Value(), nil + } +} + +// manifest reads the manifest of a digest from the indexer's bucket once and keeps it: a digest's manifest never changes. +func (w *world) manifest(ctx context.Context, get getter, digest string) parsed { + w.mu.Lock() + p, ok := w.manifests[digest] + w.mu.Unlock() + if ok { + return p + } + key, err := delivery.ManifestsKey(digest) + if err != nil { + return parsed{err: err.Error()} + } + value, err := get(ctx, key) + if errors.Is(err, errNotThere) { + return parsed{err: "the indexer holds no manifest of digest " + digest + " yet"} + } + if err != nil { + return parsed{err: err.Error()} + } + m, err := delivery.DecodeManifest(value) + if err != nil { + p = parsed{err: err.Error()} + } else if man, err := catalogue.ParseManifest([]byte(m.JSON)); err != nil { + p = parsed{module: m.Module, err: err.Error()} + } else { + p = parsed{module: m.Module, manifest: man} + } + w.mu.Lock() + w.manifests[digest] = p + w.mu.Unlock() + return p +} + +// assigned is each node's modules with an open assignment, read from the controller's assignments, in name order. +func (w *world) assigned() (map[string][]string, map[string]assignment) { + out, all := map[string][]string{}, map[string]assignment{} + for _, value := range w.read(provider.AssignmentsBucket) { + var a assignment + if json.Unmarshal(value, &a) != nil || a.Node == "" || a.Module == "" || !a.open() { + continue + } + out[a.Node] = append(out[a.Node], a.Module) + all[a.Node+"."+a.Module] = a + } + for _, mods := range out { + sort.Strings(mods) + } + return out, all +} + +// trunk is every module the indexer holds at the trunk, with its value. +func (w *world) trunk() map[string]delivery.Module { + out := map[string]delivery.Module{} + for key, value := range w.read(delivery.ModulesBucket) { + name, err := delivery.ParseModulesKey(key) + if err != nil { + continue + } + if m, err := delivery.DecodeModule(value); err == nil { + out[name] = m + } + } + return out +} + +// mesh is the mesh as the trunk holds it, for the composition check. +func (w *world) mesh(ctx context.Context, manifests getter) assign.Mesh { + m := assign.Mesh{Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}} + m.Assigned, _ = w.assigned() + for name, mod := range w.trunk() { + if mod.Manifest == "" { + continue + } + p := w.manifest(ctx, manifests, mod.Manifest) + if p.err != "" { + m.Unreadable[name] = p.err + continue + } + m.Manifests[name] = p.manifest + } + return m +} + +// change is a candidate's change for the composition check, or why the composition waits for it. +func (w *world) change(ctx context.Context, manifests getter, c delivery.Candidate, trunk map[string]delivery.Module) (assign.Change, string) { + ch := assign.Change{Group: c.Group, Heads: c.HeadsHash, Manifests: map[string]catalogue.Manifest{}, Unreadable: map[string]string{}, Twice: map[string][]string{}} + definedIn := map[string]map[string]bool{} + for _, p := range c.Pulls { + if p.Repository == assign.SDKRepository { + ch.SDKHead = p.Head.Head + } + for _, name := range p.Modules { + fp, ok := p.Fps[name] + if !ok || fp.Manifest == "" { + if ok && fp.None2 == noModuleWords { + ch.Removed = append(ch.Removed, name) + continue + } + return ch, fmt.Sprintf("the indexer holds no manifest of %s for %s#%d yet", name, p.Repository, p.Number) + } + if fp.FP2 != "" && fp.FP2 != trunk[name].FP2 { + ch.Rebuilt++ + } + got := w.manifest(ctx, manifests, fp.Manifest) + if strings.HasPrefix(got.err, "the indexer holds no manifest") { + return ch, got.err + } + if got.err != "" { + ch.Unreadable[name] = got.err + continue + } + ch.Manifests[got.module] = got.manifest + if definedIn[got.module] == nil { + definedIn[got.module] = map[string]bool{} + } + definedIn[got.module][p.Repository] = true + if t, ok := trunk[got.module]; ok && t.Repository != "" && t.Repository != p.Repository { + definedIn[got.module][t.Repository] = true + } + } + } + for name, repos := range definedIn { + if len(repos) > 1 { + for r := range repos { + ch.Twice[name] = append(ch.Twice[name], r) + } + sort.Strings(ch.Twice[name]) + } + } + return ch, "" +} + +// wantsOf is the wants the assigner wrote, by module then node. +func (w *world) wantsOf() map[string]map[string]delivery.Want { + out := map[string]map[string]delivery.Want{} + for key, value := range w.read(delivery.AssignerWantsBucket) { + node, module, err := delivery.ParseWantKey(key) + if err != nil { + continue + } + want, err := delivery.DecodeWant(value) + if err != nil { + continue + } + if out[module] == nil { + out[module] = map[string]delivery.Want{} + } + out[module][node] = want + } + return out +} + +// nodes is what the six rules read of the nodes: presence, haves, and every provision's providers with their nodes. +func (w *world) nodes(manifests map[string]catalogue.Manifest, assigned map[string][]string) assign.Nodes { + n := assign.Nodes{Online: map[string]bool{}, Haves: map[string]map[string]delivery.Have{}, Providers: map[string]map[string][]string{}} + for key, value := range w.read(delivery.PresenceBucket) { + node, _, _, err := delivery.ParsePresenceKey(key) + if err != nil || node == "" { + continue + } + if p, err := delivery.DecodePresence(value); err == nil { + n.Online[node] = p.Online + } + } + for key, value := range w.read(delivery.HaveBucket) { + node, module, isNode, err := delivery.ParseHaveKey(key) + if err != nil || isNode { + continue + } + if h, err := delivery.DecodeHave(value); err == nil { + if n.Haves[node] == nil { + n.Haves[node] = map[string]delivery.Have{} + } + n.Haves[node][module] = h + } + } + nodesOf := map[string][]string{} + for node, modules := range assigned { + for _, m := range modules { + nodesOf[m] = append(nodesOf[m], node) + } + } + for name, man := range manifests { + for _, o := range man.Provides { + if n.Providers[o.Name] == nil { + n.Providers[o.Name] = map[string][]string{} + } + n.Providers[o.Name][name] = nodesOf[name] + } + } + return n +} + +// has says whether a bucket holds a key now. +func (w *world) has(bucket, key string) bool { + w.mu.Lock() + defer w.mu.Unlock() + _, ok := w.values[bucket][key] + return ok +} + +// built says whether the builder's build of a module at a fingerprint is done. +func (w *world) built(module, fp string) bool { + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return false + } + w.mu.Lock() + value, ok := w.values[delivery.BuilderBuildsBucket][key] + w.mu.Unlock() + if !ok { + return false + } + b, err := delivery.DecodeBuilderBuild(value) + return err == nil && b.State == delivery.BuildDone +} + +// sentFP is the fingerprint of the new form of the build the controller's assignment names, read from its build record +// (mesh-controller_builds `.build.`) once and kept; empty when the record names none. +func (w *world) sentFP(ctx context.Context, get getter, module, build string) string { + if build == "" { + return "" + } + w.mu.Lock() + fp, ok := w.sent[build] + w.mu.Unlock() + if ok { + return fp + } + value, err := get(ctx, delivery.BuildKey(module, build)) + if err != nil { + return "" + } + var rec delivery.BuildRecord + if json.Unmarshal(value, &rec) != nil || !strings.HasPrefix(rec.SourceFingerprint, delivery.SourcePrefix2) { + return "" + } + w.mu.Lock() + w.sent[build] = rec.SourceFingerprint + w.mu.Unlock() + return rec.SourceFingerprint +} diff --git a/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go new file mode 100644 index 0000000..ee3a8e3 --- /dev/null +++ b/modules/mesh-delivery-assigner/cmd/mesh-delivery-assigner/world_test.go @@ -0,0 +1,90 @@ +package main + +import ( + "context" + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-assigner/internal/assign" +) + +// fakeManifests is the indexer's manifests bucket in memory. +type fakeManifests map[string][]byte + +func (f fakeManifests) add(t *testing.T, module, body string) string { + t.Helper() + digest := delivery.ManifestDigest([]byte(body)) + value, err := delivery.EncodeManifest(delivery.Manifest{Module: module, JSON: body}) + if err != nil { + t.Fatal(err) + } + f[digest] = value + return digest +} + +func (f fakeManifests) get(_ context.Context, key string) ([]byte, error) { + if v, ok := f[key]; ok { + return v, nil + } + return nil, errNotThere +} + +func fp(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } + +func pull(repo string, number int, modules map[string]delivery.CandidateModule) delivery.CandidatePull { + p := delivery.CandidatePull{Head: delivery.Head{Repository: repo, Number: number, Head: strings.Repeat("3", 40)}, Fps: modules} + for m := range modules { + p.Modules = append(p.Modules, m) + } + slices.Sort(p.Modules) + return p +} + +// 1297/B3b: a candidate's change reads each module's manifest by its digest, counts the modules rebuilt, takes a +// module with no manifest in the preview as removed, a mesh-sdk member's head, and a module the trunk holds from +// another repository as defined twice. +func TestTheChangeOfACandidate(t *testing.T) { + f := fakeManifests{} + web := f.add(t, "web", `{"module":"web","version":"2"}`) + mail := f.add(t, "mail", `{"module":"mail","version":"1"}`) + trunk := map[string]delivery.Module{"web": {FP2: fp('a'), Repository: "novox/web"}, "mail": {FP2: fp('c'), Repository: "novox/mail"}} + c := delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{ + pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: web}, "old": {None2: noModuleWords}}), + pull("novox/other", 2, map[string]delivery.CandidateModule{"mail": {FP2: fp('c'), Manifest: mail}}), + pull(assign.SDKRepository, 3, nil), + }} + ch, waits := newWorld().change(context.Background(), f.get, c, trunk) + if waits != "" { + t.Fatalf("the change waits: %s", waits) + } + if ch.Manifests["web"].Version != "2" || ch.Rebuilt != 1 || !slices.Equal(ch.Removed, []string{"old"}) || ch.SDKHead != strings.Repeat("3", 40) { + t.Errorf("change = %+v", ch) + } + if !slices.Equal(ch.Twice["mail"], []string{"novox/mail", "novox/other"}) || len(ch.Twice) != 1 { + t.Errorf("twice = %v, want mail in novox/mail and novox/other", ch.Twice) + } +} + +// 1297/B3b: a candidate whose module has no manifest digest yet, or whose manifest the indexer does not hold yet, waits, +// saying for what; an unreadable manifest is the change's, not a wait. +func TestAChangeWaitsForItsManifests(t *testing.T) { + f := fakeManifests{} + bad := f.add(t, "web", `{"module":"web","version":"2","scope":"x"}`) + for _, c := range []delivery.CandidatePull{ + pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {None2: "no preview yet"}}), + pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: strings.Repeat("9", 64)}}), + } { + _, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", Pulls: []delivery.CandidatePull{c}}, nil) + if waits == "" { + t.Errorf("a change without its manifest %+v does not wait", c.Fps) + } + } + ch, waits := newWorld().change(context.Background(), f.get, delivery.Candidate{Group: "g", HeadsHash: "0123456789ab", + Pulls: []delivery.CandidatePull{pull("novox/web", 1, map[string]delivery.CandidateModule{"web": {FP2: fp('b'), Manifest: bad}})}}, nil) + if waits != "" || ch.Unreadable["web"] == "" { + t.Errorf("an unreadable manifest: waits %q, change %+v", waits, ch) + } +} diff --git a/modules/mesh-delivery-assigner/go.mod b/modules/mesh-delivery-assigner/go.mod new file mode 100644 index 0000000..1408845 --- /dev/null +++ b/modules/mesh-delivery-assigner/go.mod @@ -0,0 +1,22 @@ +module mesh-delivery-assigner + +go 1.26.0 + +require ( + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 + github.com/nats-io/nats-server/v2 v2.11.17 + github.com/nats-io/nats.go v1.54.0 +) + +require ( + github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect + github.com/google/go-tpm v0.9.8 // indirect + github.com/klauspost/compress v1.20.0 // indirect + github.com/minio/highwayhash v1.0.4 // indirect + github.com/nats-io/jwt/v2 v2.8.1 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + golang.org/x/crypto v0.57.0 // indirect + golang.org/x/sys v0.48.0 // indirect + golang.org/x/time v0.15.0 // indirect +) diff --git a/modules/mesh-delivery-assigner/go.sum b/modules/mesh-delivery-assigner/go.sum new file mode 100644 index 0000000..2a3e9f3 --- /dev/null +++ b/modules/mesh-delivery-assigner/go.sum @@ -0,0 +1,27 @@ +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= +github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= +github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= +github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA= +github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= +github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4= +github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU= +github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg= +github.com/nats-io/nats-server/v2 v2.11.17 h1:GKEghcFK6A+aFx11Yf1LjgLC3txAwvyhnYzhBIQZA8I= +github.com/nats-io/nats-server/v2 v2.11.17/go.mod h1:B1sFVz4StNosQ903ak4N1G01Fl/9f8e06mXpFIE2K24= +github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA= +github.com/nats-io/nats.go v1.54.0/go.mod h1:y+DZoD1oBOYfZTU681eTUiUjI0vbqYGixNVFHcjHJ0k= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= diff --git a/modules/mesh-delivery-assigner/internal/assign/asks.go b/modules/mesh-delivery-assigner/internal/assign/asks.go new file mode 100644 index 0000000..52d7cb7 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/asks.go @@ -0,0 +1,29 @@ +package assign + +import ( + "fmt" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// HoldAsk is the ask about a module's fingerprint the assigner holds, at its key (delivery.HoldAskKey), first asking: keep +// it held or release it. Both options acknowledge and bind nothing: in release 5 the wants are only compared, so no answer +// changes what a node runs (ADR 0429 §7); the node agent acts on the hold from release 6. +func HoldAsk(key, module, fp string, now time.Time) delivery.AssignerAsk { + return delivery.AssignerAsk{Ask: asks.Ask{ + ID: delivery.AskingID(key, 0), + Headline: fmt.Sprintf("%s is held at %s", module, short(fp)), + Explanation: fmt.Sprintf("A node's health check of %s failed at %s, so the assigner holds that version and wants "+ + "no node to move to it. Until release 6 this changes nothing a node runs: the controller still sends.", module, short(fp)), + Options: []asks.Option{ + {ID: "keep", Label: "Keep it held", Does: "the version stays held", Level: asks.Acknowledge}, + {ID: "release", Label: "Release it", Does: "the version is released when wants are applied, from release 6", Level: asks.Acknowledge}, + }, + Who: asks.Operator, + Expires: now.Add(7 * 24 * time.Hour), + OnExpiry: "the version stays held", + About: key, + }} +} diff --git a/modules/mesh-delivery-assigner/internal/assign/asks_test.go b/modules/mesh-delivery-assigner/internal/assign/asks_test.go new file mode 100644 index 0000000..7de1348 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/asks_test.go @@ -0,0 +1,28 @@ +package assign + +import ( + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1297/B9 (the assigner's half): the ask about a held fingerprint is a valid ask of the router and of mesh-sdk's layout at +// its key, of a kind the view may answer. +func TestTheHoldAskIsValid(t *testing.T) { + key, err := delivery.HoldAskKey("mail", fNew) + if err != nil { + t.Fatal(err) + } + now := time.Now() + a := HoldAsk(key, "mail", fNew, now) + if err := a.Ask.Check(now); err != nil { + t.Errorf("the router refuses the ask: %v", err) + } + if _, err := delivery.EncodeAssignerAsk(key, a); err != nil { + t.Errorf("the ask cannot be written: %v", err) + } + if _, err := delivery.ViewAnswerKey(key); err != nil { + t.Errorf("the view may not answer it: %v", err) + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/compare.go b/modules/mesh-delivery-assigner/internal/assign/compare.go new file mode 100644 index 0000000..952acd3 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/compare.go @@ -0,0 +1,52 @@ +package assign + +import ( + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Sent is what the controller sends a node of a module: whether it sends it at all (an open assignment), the build its +// assignment names, and that build's fingerprint of the new form (empty when its record names none). +type Sent struct { + Sends bool + Build string + FP string +} + +// Compare is the difference between a node's want for a module and what the controller sends it (ADR 0528 clause 4), +// and whether there is one. want is nil when the assigner wrote none; newest is the module's newest fingerprint and +// hold the rule that holds the node back from it (Rule 0 when none does). A difference is explained by a rule when the +// controller sent the newest fingerprint while that rule holds the node's want back; every other is unexplained, said +// in words, for acceptance test 17 of design 47 to read. +func Compare(want *delivery.Want, sent Sent, newest string, hold Holding) (delivery.Difference, bool) { + var d delivery.Difference + removed := want != nil && want.Removed + if want != nil && !removed { + d.Want = want.FP + } + d.WantRemoved = removed + if !sent.Sends { + if want == nil || removed { + return delivery.Difference{}, false + } + d.Unexplained, d.Words = true, "the node wants "+short(want.FP)+" and the controller sends it nothing" + return d, true + } + d.Sent, d.SentBuild = sent.FP, sent.Build + switch { + case sent.FP == "": + d.Unexplained, d.Words = true, fmt.Sprintf("the build %s the controller sends names no fingerprint of the new form", sent.Build) + case removed: + d.Unexplained, d.Words = true, fmt.Sprintf("the want is removed and the controller still sends %s", short(sent.FP)) + case sent.FP == d.Want: + return delivery.Difference{}, false + case hold.Rule != 0 && sent.FP == newest: + d.Rule, d.Words = hold.Rule, fmt.Sprintf("the controller sent %s while %s", short(sent.FP), hold.Words) + case d.Want == "": + d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node has no want", short(sent.FP)) + default: + d.Unexplained, d.Words = true, fmt.Sprintf("the controller sends %s and the node wants %s", short(sent.FP), short(d.Want)) + } + return d, true +} diff --git a/modules/mesh-delivery-assigner/internal/assign/compare_test.go b/modules/mesh-delivery-assigner/internal/assign/compare_test.go new file mode 100644 index 0000000..da4a14f --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/compare_test.go @@ -0,0 +1,59 @@ +package assign + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// 1297/B5: given the controller sent F to a node while the canary is not yet healthy on F, the difference names rule 3. +func TestADifferenceExplainedByTheCanary(t *testing.T) { + m, n := ready() + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + out := Wants(m, n, wantedOld(), nil) + want := delivery.Want{FP: fOld, Newest: fNew, Rule: delivery.RuleCanary} + d, ok := Compare(&want, Sent{Sends: true, Build: "b-9", FP: fNew}, fNew, out.Rules["g14"]) + if !ok || d.Rule != delivery.RuleCanary || d.Unexplained || d.Want != fOld || d.Sent != fNew { + t.Fatalf("difference = %+v, %v; want rule 3", d, ok) + } + if _, err := delivery.EncodeDifference(d); err != nil { + t.Errorf("the difference cannot be written: %v", err) + } +} + +// 1297/B5: a want equal to what is sent is no difference; nothing sent and no want is none; every other is written +// unexplained, with words, and is writable. +func TestCompare(t *testing.T) { + w := delivery.Want{FP: fNew} + if _, ok := Compare(&w, Sent{Sends: true, Build: "b", FP: fNew}, fNew, Holding{}); ok { + t.Errorf("an equal want and send is a difference") + } + if _, ok := Compare(nil, Sent{}, fNew, Holding{}); ok { + t.Errorf("no want and nothing sent is a difference") + } + removed := delivery.Want{Removed: true} + if _, ok := Compare(&removed, Sent{}, fNew, Holding{}); ok { + t.Errorf("a removed want and nothing sent is a difference") + } + old := delivery.Want{FP: fOld} + for name, c := range map[string]struct { + want *delivery.Want + sent Sent + }{ + "sends an older one": {&w, Sent{Sends: true, Build: "b", FP: fOld}}, + "names no fingerprint": {&w, Sent{Sends: true, Build: "b"}}, + "sends a removed module": {&removed, Sent{Sends: true, Build: "b", FP: fNew}}, + "sends nothing": {&w, Sent{}}, + "no want": {nil, Sent{Sends: true, Build: "b", FP: fNew}}, + "no rule holds it back": {&old, Sent{Sends: true, Build: "b", FP: fNew}}, + } { + d, ok := Compare(c.want, c.sent, fNew, Holding{}) + if !ok || !d.Unexplained || d.Words == "" { + t.Errorf("%s: difference = %+v, %v; want it unexplained, with words", name, d, ok) + continue + } + if _, err := delivery.EncodeDifference(d); err != nil { + t.Errorf("%s: the difference cannot be written: %v", name, err) + } + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/compose.go b/modules/mesh-delivery-assigner/internal/assign/compose.go new file mode 100644 index 0000000..d609c87 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/compose.go @@ -0,0 +1,269 @@ +// Package assign is the assigner's judgement, as functions over what it read from the bus: the composition check of a +// change (novox/hq ADR 0446 clauses 1 to 5, ADR 0528 clause 5, issue 1297), the six rules of what each node should +// run (ADR 0429 §6, ADR 0528 clause 3) and the comparison with what the controller sends (ADR 0528 clause 4). It reads +// and writes nothing itself: the program hands it what it watched and writes what it answers. +package assign + +import ( + "fmt" + "slices" + "sort" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/compose" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// SDKRepository is the repository of mesh-sdk: a group with a member there changes the composition itself. +const SDKRepository = "novox/mesh-sdk" + +// NotJudged names, in a composition's warnings, the failures of ADR 0446 clause 1 the assigner cannot judge from the +// bus, because what they need is only in the controller's facts snapshot, which is not on the bus yet (issue 1461; +// the design limit of R4g, issue 1297): said on every verdict, never passed silently. +const NotJudged = "not judged until the facts snapshot is on the bus (issue 1461): a consumer left out of a grant, a " + + "credential left bound elsewhere, a stored setting the changed definition cannot keep, and a new module the " + + "node-engine's validator would refuse" + +// Mesh is the mesh as the trunk holds it: every module's manifest by name, the manifests that cannot be read with +// why, and each node's open assignments. +type Mesh struct { + Manifests map[string]catalogue.Manifest + Unreadable map[string]string + Assigned map[string][]string +} + +// Change is a group's change as the indexer's candidate holds it: the manifest of every module it moves or adds, those +// it cannot read and why, the modules it removes, every module defined twice with where, the heads of a mesh-sdk +// member (empty when none), and how many modules it rebuilds. +type Change struct { + Group, Heads string + Manifests map[string]catalogue.Manifest + Unreadable map[string]string + Removed []string + Twice map[string][]string + SDKHead string + Rebuilt int +} + +// Touched is the machines a change touches (ADR 0446 clause 3): those running a module the change moves, adds or +// removes, those running a consumer of a provision such a module provides (before or after), and those running a +// provider of a provision such a module requires after the change. A module the change adds that is assigned nowhere +// touches no machine. +func Touched(m Mesh, c Change) []string { + moved := map[string]bool{} + for name := range c.Manifests { + moved[name] = true + } + for name := range c.Unreadable { + moved[name] = true + } + for _, name := range c.Removed { + moved[name] = true + } + provided, required := map[string]bool{}, map[string]bool{} + for name := range moved { + for _, man := range []catalogue.Manifest{m.Manifests[name], c.Manifests[name]} { + for _, o := range man.Provides { + provided[o.Name] = true + } + } + for _, r := range c.Manifests[name].Requires { + required[r] = true + } + } + var out []string + for node, modules := range m.Assigned { + for _, mod := range modules { + man := m.Manifests[mod] + if moved[mod] || slices.ContainsFunc(man.Requires, func(r string) bool { return provided[r] }) || + slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return required[o.Name] }) { + out = append(out, node) + break + } + } + } + sort.Strings(out) + return out +} + +// input is the composition's input for the mesh's catalogue and assignments: the nodes by name with what is assigned +// to them, and every seat the assigned modules claim as held there. What the bus does not carry (places on the +// private network, settings, credentials, licences) is left empty, so a trunk composition and a change's are made +// with the same gaps and compared (ADR 0446 clause 1: what was already broken is said and fails nothing). +func input(catalog map[string]catalogue.Manifest, assigned map[string][]string) compose.Input { + in := compose.Input{Catalogue: catalog} + nodes := make([]string, 0, len(assigned)) + for n := range assigned { + nodes = append(nodes, n) + } + sort.Strings(nodes) + for _, n := range nodes { + in.Nodes = append(in.Nodes, compose.Node{Name: n, Assigned: slices.Clone(assigned[n])}) + for _, mod := range assigned[n] { + for _, cl := range catalog[mod].Claims { + scope := cl.Scope + if seat, ok := catalogue.SeatNamed(cl.Name); ok && scope == "" { + scope = seat.Scope + } + in.Holdings = append(in.Holdings, catalogue.Held{Claim: cl.Name, Scope: scope, Node: n, Module: mod}) + } + } + } + return in +} + +// Compose is the composition check of change c over the mesh m, composed with the mesh-sdk version sdk: every touched +// machine resolved as the trunk holds it and as the change leaves it. It fails on a manifest of the change that cannot +// be read, a module a machine runs that the change removes, a module defined twice, and a machine that resolves on the +// trunk and not with the change; a machine that does not resolve on either is said and fails nothing. It warns past +// CompositionRebuildWarning modules rebuilt, says what it does not judge (NotJudged), and for a group with a mesh-sdk +// member says it composed with its own mesh-sdk, not the member's head (issue 1621). took is how long it took. +func Compose(m Mesh, c Change, sdk string, took time.Duration) delivery.CompositionVerdict { + v := delivery.CompositionVerdict{Group: c.Group, Heads: c.Heads, SDK: sdk, Took: took.Milliseconds()} + var fails []string + failAt := func(machine, words string) { + if v.Machine == "" && machine != "" { + v.Machine = machine + } + fails = append(fails, words) + } + for _, name := range sortedKeys(c.Unreadable) { + failAt("", fmt.Sprintf("the manifest of %s cannot be read: %s", name, c.Unreadable[name])) + } + for _, name := range sortedKeys(c.Twice) { + failAt("", fmt.Sprintf("%s is defined twice, in %s", name, strings.Join(c.Twice[name], " and "))) + } + machines := Touched(m, c) + v.Machines = machines + for _, node := range machines { + for _, name := range c.Removed { + if slices.Contains(m.Assigned[node], name) { + failAt(node, fmt.Sprintf("%s runs %s, which the change removes from its source", node, name)) + } + } + } + trunk := input(m.Manifests, m.Assigned) + changed := map[string]catalogue.Manifest{} + for name, man := range m.Manifests { + changed[name] = man + } + for name, man := range c.Manifests { + changed[name] = man + } + for _, name := range c.Removed { + delete(changed, name) + } + after := input(changed, m.Assigned) + for _, node := range machines { + _, before := compose.Resolve(trunk, node) + _, now := compose.Resolve(after, node) + switch { + case now == nil: + case before != nil: + v.Warnings = append(v.Warnings, fmt.Sprintf("%s does not compose on the trunk either, so the change is not blamed: %v", node, before)) + default: + failAt(node, fmt.Sprintf("%s composes on the trunk and not with the change: %v", node, now)) + } + } + if c.Rebuilt > delivery.CompositionRebuildWarning { + v.Warnings = append(v.Warnings, fmt.Sprintf("the merge rebuilds %d modules, more than %d", c.Rebuilt, delivery.CompositionRebuildWarning)) + } + if c.SDKHead != "" { + v.Warnings = append(v.Warnings, fmt.Sprintf("composed with the assigner's own mesh-sdk %s, not the group's mesh-sdk head %s, "+ + "and a field the running controller cannot read is not judged (issues 1621, 1622, 1623)", sdk, c.SDKHead)) + } + v.Warnings = append(v.Warnings, NotJudged) + if could := CouldRun(m, c, machines); len(could) > 0 { + v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+ + "the machines that could run a moved or added module and are not assigned it: "+strings.Join(could, ", ")) + } + if grants := GrantsAlone(m, c, machines); len(grants) > 0 { + v.Warnings = append(v.Warnings, "not judged until the facts snapshot is on the bus (issue 1461, ADR 0537 clause 1): "+ + "the machines whose grants alone the change alters: "+strings.Join(grants, ", ")) + } + if len(machines) == 0 { + v.Warnings = append(v.Warnings, "the change touches no machine: no module it moves is assigned anywhere") + } + if len(fails) > 0 { + v.State, v.Words = delivery.CompositionFail, strings.Join(fails, "; ") + return v + } + v.State = delivery.CompositionPass + return v +} + +// CouldRun is, by name, every machine of the mesh not composed that is not assigned a module the change moves or adds +// (ADR 0537 clause 1): which of them could run it, the bus does not say (capabilities, settings and places are only in +// the facts snapshot), so each is named as not judged. +func CouldRun(m Mesh, c Change, composed []string) []string { + var out []string + for node, modules := range m.Assigned { + if slices.Contains(composed, node) { + continue + } + for name := range c.Manifests { + if !slices.Contains(modules, name) { + out = append(out, node) + break + } + } + } + sort.Strings(out) + return out +} + +// grantsOf is what of a manifest makes grants: its bus declarations and the provisions it requires and offers. +func grantsOf(man catalogue.Manifest) string { + var offers []string + for _, o := range man.Provides { + offers = append(offers, o.Name) + } + return fmt.Sprint(man.Requires, offers, man.Reads, man.State, man.Emits, man.Consumes, man.Invokes, man.Claims, man.Grants) +} + +// GrantsAlone is, by name, every machine not composed whose grants alone the change alters (ADR 0537 clause 1): for a +// moved module whose grants differ from the trunk's, the machines assigned the bus's holder (whose user list carries +// every module's bus grants) and those assigned a provider of a provision it requires, before or after. +func GrantsAlone(m Mesh, c Change, composed []string) []string { + provisions := map[string]bool{} + altered := false + for name, man := range c.Manifests { + before, had := m.Manifests[name] + if had && grantsOf(before) == grantsOf(man) { + continue + } + altered = true + for _, r := range append(slices.Clone(man.Requires), before.Requires...) { + provisions[r] = true + } + } + if !altered { + return nil + } + var out []string + for node, modules := range m.Assigned { + if slices.Contains(composed, node) { + continue + } + for _, mod := range modules { + man := m.Manifests[mod] + if man.ClaimsSeat("mesh-broker") || slices.ContainsFunc(man.Provides, func(o catalogue.Offer) bool { return provisions[o.Name] }) { + out = append(out, node) + break + } + } + } + sort.Strings(out) + return out +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/modules/mesh-delivery-assigner/internal/assign/compose_test.go b/modules/mesh-delivery-assigner/internal/assign/compose_test.go new file mode 100644 index 0000000..96c46ab --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/compose_test.go @@ -0,0 +1,141 @@ +package assign + +import ( + "slices" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func meshOf() Mesh { + return Mesh{ + Manifests: map[string]catalogue.Manifest{ + "mail": {Module: "mail", Version: "1", Requires: []string{"database"}}, + "store": {Module: "store", Version: "1", Provides: []catalogue.Offer{{Name: "database"}}}, + "web": {Module: "web", Version: "1"}, + }, + Assigned: map[string][]string{"ace": {"mail", "store"}, "g14": {"web"}}, + } +} + +func hasWarning(v delivery.CompositionVerdict, part string) bool { + return slices.ContainsFunc(v.Warnings, func(w string) bool { return strings.Contains(w, part) }) +} + +// 1297/AC1 (its unit half): a change touching one module on one machine composes that machine alone, passes, names +// the machine and the mesh-sdk it composed with, and says what it did not judge. +func TestAChangeOfOneModuleComposesItsMachine(t *testing.T) { + c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}} + v := Compose(meshOf(), c, "v0.1.14-x", 0) + if v.State != delivery.CompositionPass || !slices.Equal(v.Machines, []string{"g14"}) || v.SDK != "v0.1.14-x" { + t.Fatalf("composition = %+v, want a pass on g14 alone", v) + } + if !hasWarning(v, "issue 1461") { + t.Errorf("the verdict does not say what it did not judge: %v", v.Warnings) + } + if _, err := delivery.EncodeCompositionVerdict(v); err != nil { + t.Errorf("the verdict cannot be written: %v", err) + } +} + +// 1297/AC2: a change that removes a module a machine runs fails, naming the machine and the module. +func TestRemovingAModuleAMachineRunsFails(t *testing.T) { + c := Change{Group: "g", Heads: "0123456789ab", Removed: []string{"web"}} + v := Compose(meshOf(), c, "sdk", 0) + if v.State != delivery.CompositionFail || v.Machine != "g14" || !strings.Contains(v.Words, "g14 runs web") { + t.Fatalf("composition = %+v, want a fail naming g14 and web", v) + } +} + +// 1297/B3c: a machine that composes on the trunk and not with the change fails it, naming the machine and why; the +// consumer's machine is touched when its provider changes. +func TestAMachineTheChangeBreaksFails(t *testing.T) { + c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"store": {Module: "store", Version: "2"}}} + v := Compose(meshOf(), c, "sdk", 0) + if v.State != delivery.CompositionFail || v.Machine != "ace" || !strings.Contains(v.Words, "database") { + t.Fatalf("composition = %+v, want a fail on ace naming the provision nothing provides", v) + } +} + +// 1297/B3c: what was already broken on the trunk is said and fails nothing. +func TestWhatWasBrokenBeforeFailsNothing(t *testing.T) { + m := meshOf() + m.Manifests["store"] = catalogue.Manifest{Module: "store", Version: "1"} + c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"mail": {Module: "mail", Version: "2", Requires: []string{"database"}}}} + v := Compose(m, c, "sdk", 0) + if v.State != delivery.CompositionPass || !hasWarning(v, "on the trunk either") { + t.Fatalf("composition = %+v, want a pass saying ace was already broken", v) + } +} + +// 1297/B3c: an unreadable manifest and a module defined twice fail; more than twelve rebuilt modules warn. +func TestUnreadableTwiceAndWide(t *testing.T) { + c := Change{Group: "g", Heads: "0123456789ab", Unreadable: map[string]string{"web": "unknown field \"scope\""}, Rebuilt: 13} + v := Compose(meshOf(), c, "sdk", 0) + if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web cannot be read") || !hasWarning(v, "rebuilds 13 modules") { + t.Fatalf("composition = %+v", v) + } + c = Change{Group: "g", Heads: "0123456789ab", Twice: map[string][]string{"web": {"novox/a", "novox/b"}}, Rebuilt: 12} + v = Compose(meshOf(), c, "sdk", 0) + if v.State != delivery.CompositionFail || !strings.Contains(v.Words, "web is defined twice") || hasWarning(v, "rebuilds") { + t.Fatalf("composition = %+v", v) + } +} + +// 1622's criterion as R4g delivers it: a group with a mesh-sdk member is composed with the assigner's own mesh-sdk, and +// the verdict says so, naming the member's head it did not use; a group without one says nothing of it. +func TestAGroupWithAnSDKMemberSaysWhichSDKComposedIt(t *testing.T) { + c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"web": {Module: "web", Version: "2"}}, SDKHead: "f625d98713e4"} + v := Compose(meshOf(), c, "v0.1.14-own", 0) + if !hasWarning(v, "not the group's mesh-sdk head f625d98713e4") || !hasWarning(v, "v0.1.14-own") || !hasWarning(v, "1623") { + t.Errorf("the verdict does not say it composed with its own mesh-sdk: %v", v.Warnings) + } + c.SDKHead = "" + if v := Compose(meshOf(), c, "v0.1.14-own", 0); hasWarning(v, "mesh-sdk head") { + t.Errorf("a group without a mesh-sdk member says it has one: %v", v.Warnings) + } +} + +// 1297/B3a: the machines touched are those running a moved module or a consumer of what it provides; an added module +// assigned nowhere touches none, and the verdict says so. +func TestTouched(t *testing.T) { + m := meshOf() + if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace"}) { + t.Errorf("touched by store = %v", got) + } + m.Assigned["g14"] = append(m.Assigned["g14"], "mail") + if got := Touched(m, Change{Manifests: map[string]catalogue.Manifest{"store": m.Manifests["store"]}}); !slices.Equal(got, []string{"ace", "g14"}) { + t.Errorf("touched by store with a consumer on g14 = %v", got) + } + c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{"new": {Module: "new", Version: "1"}}} + if v := Compose(m, c, "sdk", 0); len(v.Machines) != 0 || !hasWarning(v, "touches no machine") || v.State != delivery.CompositionPass { + t.Errorf("an added module assigned nowhere: %+v", v) + } +} + +// ADR 0537 clause 1: every verdict names, by machine, those that could run a moved or added module and are not assigned +// it, and those whose grants alone the change alters (the bus's machine, a provider of what the module requires), as not +// judged; shanks, providing what web now requires, is composed and so named in neither. A change that alters no grant +// names none of the second. +func TestTheMachinesNotJudgedAreNamed(t *testing.T) { + m := meshOf() + m.Manifests["bus"] = catalogue.Manifest{Module: "bus", Version: "1", Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: "mesh"}}} + m.Manifests["db"] = catalogue.Manifest{Module: "db", Version: "1", Provides: []catalogue.Offer{{Name: "cache"}}} + m.Assigned["novox"] = []string{"bus"} + m.Assigned["shanks"] = []string{"db"} + c := Change{Group: "g", Heads: "0123456789ab", Manifests: map[string]catalogue.Manifest{ + "web": {Module: "web", Version: "2", Reads: []string{"mesh-controller.assignments"}, Requires: []string{"cache"}}}} + v := Compose(m, c, "sdk", 0) + if !hasWarning(v, "could run a moved or added module and are not assigned it: ace, novox") || !slices.Contains(v.Machines, "shanks") { + t.Errorf("the machines that could run web are not named: %v", v.Warnings) + } + if !hasWarning(v, "whose grants alone the change alters: novox") { + t.Errorf("the machines whose grants alone change are not named: %v", v.Warnings) + } + c.Manifests["web"] = catalogue.Manifest{Module: "web", Version: "3"} + if v := Compose(meshOf(), c, "sdk", 0); hasWarning(v, "grants alone") { + t.Errorf("a change altering no grant names machines whose grants change: %v", v.Warnings) + } +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants.go b/modules/mesh-delivery-assigner/internal/assign/wants.go new file mode 100644 index 0000000..524f823 --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/wants.go @@ -0,0 +1,205 @@ +package assign + +import ( + "fmt" + "slices" + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Module is what the six rules read of one module (ADR 0429 §6, ADR 0528 clause 3): its newest fingerprint F at the +// trunk (the indexer's modules), whether the builder's build of F is done, whether F is held, whether the controller +// issued its credential for F, its manifest at F, and the nodes it is assigned to. +type Module struct { + Name string + FP string + Built bool + Held bool + Granted bool + Manifest catalogue.Manifest + Nodes []string +} + +// Node is what the rules read of the nodes: whether each is online (the watchman's presence) and each module's have. +type Nodes struct { + Online map[string]bool + Haves map[string]map[string]delivery.Have + // Providers is, by provision, the modules that provide it, with the nodes each is assigned to. + Providers map[string]map[string][]string +} + +// healthy says whether a have runs fp healthy. +func healthy(h delivery.Have, fp string) bool { + return h.FP == fp && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK +} + +// Canary is the module's canary: the first of its nodes, by name, that is online (ADR 0435 §3; one ring, ordered by +// name, ADR 0528 clause 3). Empty when none is online. +func Canary(nodes []string, online map[string]bool) string { + sorted := slices.Sorted(slices.Values(nodes)) + for _, n := range sorted { + if online[n] { + return n + } + } + return "" +} + +// Holds is whether every rule holds for moving node's want for m to m.FP, and if not, the first rule that does not, +// with why (ADR 0429 §6). Rule 4, at most `step` others ahead, always holds: the mesh keeps no step, and step 0 is no +// limit (ADR 0528 clause 3). +func Holds(m Module, node string, n Nodes) (bool, int, string) { + switch { + case !m.Built: + return false, delivery.RuleBuildDone, fmt.Sprintf("the build of %s at %s is not done", m.Name, short(m.FP)) + case m.Held: + return false, delivery.RuleNotHeld, fmt.Sprintf("%s is held at %s", m.Name, short(m.FP)) + } + canary := Canary(m.Nodes, n.Online) + switch { + case canary == "": + return false, delivery.RuleCanary, fmt.Sprintf("none of the nodes of %s is online, so it has no canary", m.Name) + case node != canary && !healthy(n.Haves[canary][m.Name], m.FP): + return false, delivery.RuleCanary, fmt.Sprintf("the canary %s does not run %s healthy yet", canary, short(m.FP)) + } + for _, provision := range m.Manifest.Requires { + if ok, why := providedHealthy(provision, node, n); !ok { + return false, delivery.RuleProviders, why + } + } + if !m.Granted { + return false, delivery.RuleGrant, fmt.Sprintf("the controller has not issued the credential of %s for %s", m.Name, short(m.FP)) + } + return true, 0, "" +} + +// providedHealthy is rule 5 for one provision: some module providing it runs healthy on the node, or, assigned +// elsewhere only, at its canary. A provision no module provides is not a module's to judge (the mesh or the node +// offers it) and holds. +func providedHealthy(provision, node string, n Nodes) (bool, string) { + providers := n.Providers[provision] + if len(providers) == 0 { + return true, "" + } + names := make([]string, 0, len(providers)) + for p := range providers { + names = append(names, p) + } + sort.Strings(names) + for _, p := range names { + nodes := providers[p] + at := node + if !slices.Contains(nodes, node) { + at = Canary(nodes, n.Online) + } + if h, ok := n.Haves[at][p]; ok && at != "" && h.Phase == delivery.PhaseRunning && h.Health == delivery.HealthOK { + return true, "" + } + } + return false, fmt.Sprintf("no provider of %s runs healthy on %s or at its canary", provision, node) +} + +// keepsData says whether a module holds data a set-back would lose: an own item of a class that keeps (irreplaceable, +// valuable or rebuildable; cache and none do not). +func keepsData(m catalogue.Manifest) bool { + for _, it := range m.DataItems() { + if it.Class == catalogue.ClassIrreplaceable || it.Class == catalogue.ClassValuable || it.Class == catalogue.ClassRebuildable { + return true + } + } + return false +} + +// Outcome is what one module's pass decides: each node's want to write (only those that change), and a hold to write. +type Outcome struct { + Wants map[string]delivery.Want + // Held is the hold of m.FP to write, when a node's health check failed on it and it is not held yet. + Held *delivery.Held + // Rules is, by node, the rule holding the node back from m.FP and why, for the comparison. + Rules map[string]Holding +} + +// Holding is a rule holding a node back, and why. +type Holding struct { + Rule int + Words string +} + +// Wants is the pass of one module m: for each of its nodes, the want by the six rules, starting from the want written +// before (wants, by node); lastHealthy is, by node, the newest fingerprint other than m.FP the node ran healthy. A failed +// health check of m.FP on a node holds it: every node wanting m.FP is set back to its last healthy fingerprint, but a +// module holding data is not set back (in release 5 it is asked about from release 6 on: wants are only compared). A +// node that wants m but is no longer among its nodes gets its want written removed, never deleted (ADR 0408 §4). +func Wants(m Module, n Nodes, wants map[string]delivery.Want, lastHealthy map[string]string) Outcome { + out := Outcome{Wants: map[string]delivery.Want{}, Rules: map[string]Holding{}} + held := m.Held + if !held && m.FP != "" { + for _, node := range slices.Sorted(slices.Values(m.Nodes)) { + if h, ok := n.Haves[node][m.Name]; ok && h.FP == m.FP && h.Health == delivery.HealthFailed { + out.Held = &delivery.Held{Node: node, Words: h.Detail} + held = true + break + } + } + } + mm := m + mm.Held = held + for _, node := range m.Nodes { + if m.FP == "" { + break + } + before, had := wants[node] + var next delivery.Want + switch ok, rule, words := Holds(mm, node, n); { + case ok: + next = delivery.Want{FP: m.FP} + case held && had && before.FP == m.FP: + out.Rules[node] = Holding{Rule: rule, Words: words} + last := lastHealthy[node] + switch { + case keepsData(m.Manifest): + next = before + next.Words = fmt.Sprintf("%s holds data, so it is not set back from %s: the operator is asked from release 6", m.Name, short(m.FP)) + case last == "": + next = before + next.Words = fmt.Sprintf("%s is held at %s and %s ran no other fingerprint healthy to set it back to", m.Name, short(m.FP), node) + default: + next = delivery.Want{FP: last, Newest: m.FP, Rule: rule, Words: words} + } + default: + out.Rules[node] = Holding{Rule: rule, Words: words} + if !had || before.Removed { + continue + } + next = delivery.Want{FP: before.FP, Newest: m.FP, Rule: rule, Words: words} + if before.FP == m.FP { + next = delivery.Want{FP: m.FP} + } + } + if !had || !delivery.SameWant(normal(before), normal(next)) { + out.Wants[node] = next + } + } + for node, before := range wants { + if !slices.Contains(m.Nodes, node) && !before.Removed { + out.Wants[node] = delivery.Want{Removed: true, Words: fmt.Sprintf("%s is no longer assigned to %s", m.Name, node)} + } + } + return out +} + +// normal is a want without its version, as a decoded and a composed one are compared. +func normal(w delivery.Want) delivery.Want { + w.V = 0 + return w +} + +func short(fp string) string { + part, err := delivery.FingerprintKeyPart(fp) + if err != nil || len(part) < 12 { + return fp + } + return part[:12] +} diff --git a/modules/mesh-delivery-assigner/internal/assign/wants_test.go b/modules/mesh-delivery-assigner/internal/assign/wants_test.go new file mode 100644 index 0000000..c0ff7df --- /dev/null +++ b/modules/mesh-delivery-assigner/internal/assign/wants_test.go @@ -0,0 +1,139 @@ +package assign + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func fpOf(c byte) string { return delivery.SourcePrefix2 + strings.Repeat(string(c), 64) } + +var ( + fNew = fpOf('b') + fOld = fpOf('a') +) + +// ready is a module mail at fNew on ace and g14 for which every rule holds: built, not held, granted, its provider +// store healthy on ace and the canary ace running fNew healthy. +func ready() (Module, Nodes) { + m := Module{Name: "mail", FP: fNew, Built: true, Granted: true, Nodes: []string{"g14", "ace"}, + Manifest: catalogue.Manifest{Module: "mail", Requires: []string{"database"}}} + n := Nodes{ + Online: map[string]bool{"ace": true, "g14": true}, + Haves: map[string]map[string]delivery.Have{ + "ace": {"mail": {FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}, "store": {FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}, + "g14": {"mail": {FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}}, + }, + Providers: map[string]map[string][]string{"database": {"store": {"ace"}}}, + } + return m, n +} + +func wantedOld() map[string]delivery.Want { + return map[string]delivery.Want{"ace": {FP: fOld}, "g14": {FP: fOld}} +} + +// 1297/B4: with every rule holding, both nodes' wants move to the new fingerprint (the canary ace runs it healthy, so +// g14 follows at once: step 0, ADR 0528 clause 3). +func TestEveryRuleHoldsSoTheWantsMove(t *testing.T) { + m, n := ready() + out := Wants(m, n, wantedOld(), nil) + for _, node := range []string{"ace", "g14"} { + if out.Wants[node].FP != fNew || out.Wants[node].Rule != 0 { + t.Errorf("%s's want = %+v, want %s", node, out.Wants[node], fNew) + } + } +} + +// 1297/B4, one test per rule: given every rule but N holds, no want moves to F, and the rule N is named. +func TestEachRuleHoldsTheWantBack(t *testing.T) { + for rule, breakIt := range map[int]func(*Module, *Nodes){ + delivery.RuleBuildDone: func(m *Module, _ *Nodes) { m.Built = false }, + delivery.RuleNotHeld: func(m *Module, _ *Nodes) { m.Held = true }, + delivery.RuleCanary: func(_ *Module, n *Nodes) { + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + }, + delivery.RuleProviders: func(_ *Module, n *Nodes) { + n.Haves["ace"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthWaits} + }, + delivery.RuleGrant: func(m *Module, _ *Nodes) { m.Granted = false }, + } { + m, n := ready() + breakIt(&m, &n) + out := Wants(m, n, wantedOld(), nil) + if w, ok := out.Wants["g14"]; ok && w.FP == fNew { + t.Errorf("rule %d broken: g14's want moved to F", rule) + } + if got := out.Rules["g14"].Rule; got != rule { + t.Errorf("rule %d broken: g14 is held back by rule %d (%s)", rule, got, out.Rules["g14"].Words) + } + if w := out.Wants["g14"]; w.FP != "" && (w.FP != fOld || w.Newest != fNew || w.Rule != rule) { + t.Errorf("rule %d broken: g14's want is %+v", rule, w) + } + } +} + +// 1297/B4, ADR 0435 §3: a canary that is offline is passed over: the next node by name is the canary, and moves first. +func TestAnOfflineCanaryIsPassedOver(t *testing.T) { + m, n := ready() + n.Online["ace"] = false + n.Haves["ace"]["mail"] = delivery.Have{FP: fOld, Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + n.Providers["database"]["store"] = []string{"ace", "g14"} + n.Haves["g14"]["store"] = delivery.Have{FP: fpOf('s'), Phase: delivery.PhaseRunning, Health: delivery.HealthOK} + if c := Canary(m.Nodes, n.Online); c != "g14" { + t.Fatalf("canary = %q, want g14", c) + } + out := Wants(m, n, wantedOld(), nil) + if out.Wants["g14"].FP != fNew { + t.Errorf("the canary g14's want = %+v", out.Wants["g14"]) + } + if out.Rules["ace"].Rule != delivery.RuleCanary { + t.Errorf("ace, behind the canary g14, is held back by %+v", out.Rules["ace"]) + } +} + +// 1297/B4: a failed health check of F holds F and sets the nodes that want it back to their last healthy fingerprint. +func TestAFailedHealthHoldsAndSetsBack(t *testing.T) { + m, n := ready() + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed, Detail: "connection refused"} + out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}, "g14": {FP: fOld}}, map[string]string{"ace": fOld}) + if out.Held == nil || out.Held.Node != "ace" || out.Held.Words != "connection refused" { + t.Fatalf("held = %+v", out.Held) + } + if w := out.Wants["ace"]; w.FP != fOld || w.Newest != fNew || w.Rule != delivery.RuleNotHeld { + t.Errorf("ace's want = %+v, want set back to the last healthy fingerprint", w) + } + if w, ok := out.Wants["g14"]; ok && w.FP == fNew { + t.Errorf("g14 moved to a held fingerprint: %+v", w) + } +} + +// 1297/B4: a module holding data is not set back; its want stays and says why. +func TestADataModuleIsNotSetBack(t *testing.T) { + m, n := ready() + m.Manifest.Data = &catalogue.Data{Own: []catalogue.DataItem{{ID: "mail", Class: catalogue.ClassIrreplaceable}}} + n.Haves["ace"]["mail"] = delivery.Have{FP: fNew, Phase: delivery.PhaseRunning, Health: delivery.HealthFailed} + out := Wants(m, n, map[string]delivery.Want{"ace": {FP: fNew}}, map[string]string{"ace": fOld}) + if out.Held == nil { + t.Fatal("F was not held") + } + if w := out.Wants["ace"]; w.FP != fNew || !strings.Contains(w.Words, "holds data") { + t.Errorf("ace's want = %+v, want it left at F, saying it holds data", w) + } +} + +// 1297/B4, ADR 0408 §4: a node that leaves the module's nodes gets its want written removed, never deleted; a want that +// is already as the rules give it is not written again. +func TestALeavingNodeIsRemovedAndNothingIsWrittenTwice(t *testing.T) { + m, n := ready() + m.Nodes = []string{"ace"} + out := Wants(m, n, map[string]delivery.Want{"ace": {V: 1, FP: fNew}, "g14": {FP: fOld}}, nil) + if w := out.Wants["g14"]; !w.Removed { + t.Errorf("g14's want = %+v, want removed", w) + } + if _, ok := out.Wants["ace"]; ok { + t.Errorf("ace's want, unchanged, was written again: %+v", out.Wants["ace"]) + } +} diff --git a/modules/mesh-delivery-assigner/manifest_test.go b/modules/mesh-delivery-assigner/manifest_test.go new file mode 100644 index 0000000..03b879a --- /dev/null +++ b/modules/mesh-delivery-assigner/manifest_test.go @@ -0,0 +1,57 @@ +package manifest + +import ( + "os" + "reflect" + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +func load(t *testing.T) catalogue.Manifest { + t.Helper() + raw, err := os.ReadFile("module.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatalf("mesh-sdk's manifest reader refuses the manifest: %v", err) + } + return m +} + +// 1297/B2a: the controller's reader (mesh-sdk catalogue, which the controller reads manifests with) takes the +// manifest; it reads exactly the buckets the assigner opens (the program's list, held equal in its own test), keeps +// its three states, and holds a credential of its own. +func TestManifest(t *testing.T) { + m := load(t) + if m.Module != "mesh-delivery-assigner" { + t.Errorf("module = %q", m.Module) + } + reads := slices.Clone(m.Reads) + slices.Sort(reads) + want := []string{"mesh-controller.assignments", "mesh-controller.builds", "mesh-controller.grants", + "mesh-delivery-builder.builds", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", + "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"} + if !reflect.DeepEqual(reads, want) { + t.Errorf("reads = %v, want exactly %v", reads, want) + } + states := map[string]int{} + for _, s := range m.State { + states[s.Name] = s.History + } + if want := map[string]int{"asks": 5, "compositions": 5, "differences": 64, "wants": 20}; !reflect.DeepEqual(states, want) { + t.Errorf("state = %v, want %v", states, want) + } + if _, ok := m.OwnSecrets["broker"]; !ok || len(m.OwnSecrets) != 1 { + t.Errorf("own-secrets = %v, want exactly broker (or no credential is issued)", m.OwnSecrets) + } + if m.SecretsOwner != "mesh-delivery-assigner" { + t.Errorf("secrets-owner = %q", m.SecretsOwner) + } + if len(m.Claims) != 0 || len(m.Tools) != 0 || len(m.Invokes) != 0 { + t.Errorf("the assigner claims, serves or calls something: claims %v, tools %v, invokes %v", m.Claims, m.Tools, m.Invokes) + } +} diff --git a/modules/mesh-delivery-assigner/module.json b/modules/mesh-delivery-assigner/module.json new file mode 100644 index 0000000..418e8fb --- /dev/null +++ b/modules/mesh-delivery-assigner/module.json @@ -0,0 +1,71 @@ +{ + "module": "mesh-delivery-assigner", + "version": "1", + "slug": "assigner", + "reads": [ + "mesh-controller.assignments", + "mesh-controller.builds", + "mesh-controller.grants", + "mesh-delivery-builder.builds", + "mesh-delivery-indexer.candidates", + "mesh-delivery-indexer.manifests", + "mesh-delivery-indexer.modules", + "mesh-delivery-watchman.presence", + "node-engine.have" + ], + "state": [ + { "name": "asks", "history": 5 }, + { "name": "compositions", "history": 5 }, + { "name": "differences", "history": 64 }, + { "name": "wants", "history": 20 } + ], + "own-secrets": { + "broker": "${dir:state}/broker" + }, + "secrets-owner": "mesh-delivery-assigner", + "resources": [ + { + "id": "account", + "type": "user", + "name": "mesh-delivery-assigner", + "shell": "/usr/bin/nologin", + "home": "/var/lib/mesh-delivery-assigner" + }, + { + "id": "state", + "type": "directory", + "mode": "0700", + "owner": "mesh-delivery-assigner", + "place": ".", + "unknown": "program" + }, + { + "id": "assigner", + "type": "process", + "name": "mesh-delivery-assigner", + "artifact": "program", + "run": [ + "./mesh-delivery-assigner" + ], + "env": { + "MESH_BROKER_FILE": "${dir:state}/broker" + }, + "user": "mesh-delivery-assigner", + "health": { + "kind": "unit" + } + } + ], + "build": { + "artifacts": [ + { + "name": "program", + "kind": "bundle", + "language": "go", + "system": "arch", + "from": "cmd/mesh-delivery-assigner", + "binary": "mesh-delivery-assigner" + } + ] + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go index b456fbd..359b1e9 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/bus.go @@ -102,43 +102,6 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key } } -// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the -// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every -// change after. No timer: the caller bounds the wait with its context. -type stepsWaiter struct{ js bucketOpener } - -func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) { - kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module)) - if err != nil { - return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) - } - watch, err := kv.Watch(ctx, key) - if err != nil { - return delivery.Step{}, err - } - defer watch.Stop() - for { - select { - case <-ctx.Done(): - return delivery.Step{}, ctx.Err() - case e, ok := <-watch.Updates(): - if !ok { - return delivery.Step{}, errors.New("the watch of the step ended") - } - if e == nil || !opOf(e) { - continue - } - step, err := delivery.DecodeStep(e.Value()) - if err != nil { - return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err) - } - if step.State != delivery.StepRunning { - return step, nil - } - } - } -} - // manifestReader reads manifests. of the indexer and parses the manifest it holds. type manifestReader struct{ kv jetstream.KeyValue } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go new file mode 100644 index 0000000..e5493a4 --- /dev/null +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/held.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "fmt" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// heldOnTheBus says whether an ask a build key names is on its build seat's stream, or a build node wrote a step for it +// (issue 1910): read once, at the take-over of a key at start, never polled. +type heldOnTheBus struct{ js jetstream.JetStream } + +// Held is build.Builder's Held: a step key `.` in the build module's steps, or a message of the stream under +// the ask's subject whose Nats-Msg-Id is the ask's id. +func (h heldOnTheBus) Held(ctx context.Context, a delivery.StepAsk) (bool, error) { + module, err := delivery.BuildModuleOf(a.Language) + if err != nil { + return false, err + } + pattern, err := delivery.AskStepWatch(a.ID) + if err != nil { + return false, err + } + steps, err := h.js.KeyValue(ctx, delivery.StepsBucket(module)) + if err != nil { + return false, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) + } + keys, err := steps.ListKeysFiltered(ctx, pattern) + if err != nil { + return false, err + } + for range keys.Keys() { + keys.Stop() + return true, nil // a node took it + } + subject, err := delivery.AskSubject(a.Language, a.Kind) + if err != nil { + return false, err + } + // The seat's stream by its name (delivery.AskStream), as the broker grants the builder its information and its + // messages by sequence and nothing wider (issue 1910). + name, err := delivery.AskStream(a.Language) + if err != nil { + return false, err + } + stream, err := h.js.Stream(ctx, name) + if errors.Is(err, jetstream.ErrStreamNotFound) { + return false, nil + } + if err != nil { + return false, err + } + // A seat's stream is a work queue, which takes no second consumer over its subjects: its messages are read by sequence + // instead, each a direct read that consumes nothing. + info, err := stream.Info(ctx) + if err != nil { + return false, err + } + for seq := info.State.FirstSeq; seq != 0 && seq <= info.State.LastSeq; seq++ { + m, err := stream.GetMsg(ctx, seq) + if errors.Is(err, jetstream.ErrMsgNotFound) { + continue // taken and acknowledged, or removed + } + if err != nil { + return false, err + } + if m.Subject == subject && m.Header.Get(jetstream.MsgIDHeader) == a.ID { + return true, nil + } + } + return false, nil +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go index 182885b..51580bf 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/integration_test.go @@ -22,6 +22,7 @@ import ( "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -47,13 +48,12 @@ const ( fp = "src2:3333333333333333333333333333333333333333333333333333333333333333" ) -// fakeBuildModules serve the tools of mesh-build-go and mesh-build-node as the real ones do (ADR 0469 clause 2): a call on -// mesh.mod..call... answers at once with the step and its key, and the step's end is written to -// the key . of the module's steps bucket, some milliseconds later. +// fakeBuildModules are the build modules on the nodes as the real ones run since R4g (ADR 0525; issue 1887 A4): each pulls +// its build seat's asks on its nodes (buildstep.Puller over the seat's workers) and runs an ask's job with step bodies that +// answer as the real tools do, ending the step on . of its steps bucket. type fakeBuildModules struct { js jetstream.JetStream ctx context.Context - n atomic.Int64 failBuild atomic.Bool // refusePush makes every bundle and archive build fail as a push the artifact store refused (ADR 0498 clause 2). refusePush atomic.Bool @@ -61,6 +61,21 @@ type fakeBuildModules struct { calls map[string]int // by `.` fetches []map[string]any builds []map[string]any // the arguments of every build call + // closed, while set, keeps every node's capacity at 0: no ask is pulled until open is called (a test of the asks + // waiting in their streams). + closed atomic.Bool + pullers []*buildstep.Puller +} + +// open lets the nodes pull again and wakes their pullers. +func (f *fakeBuildModules) open() { + f.closed.Store(false) + f.mu.Lock() + ps := append([]*buildstep.Puller(nil), f.pullers...) + f.mu.Unlock() + for _, p := range ps { + p.Wake() + } } func (f *fakeBuildModules) count(tool string) int { @@ -75,76 +90,76 @@ func digestOf(artifact string) string { return "sha256:" + hex.EncodeToString(sum[:]) } -func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { - steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1}) +var prefixOf = map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_", "mesh-build-image": "build_image_"} + +// step is one tool's answer, as the real module's. +func (f *fakeBuildModules) step(module, name string, args map[string]any) (any, error) { + tool := prefixOf[module] + name + f.mu.Lock() + f.calls[module+"."+tool]++ + if name == "fetch" { + f.fetches = append(f.fetches, args) + } else { + f.builds = append(f.builds, args) + } + f.mu.Unlock() + switch { + case name == "fetch": + merge := "" + if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult) + merge = `,"merge":"` + strings.Repeat("9", 40) + `"` + } + return json.RawMessage(`{"workspace":"w` + buildstep.NewID() + `","commit":"` + commit + `"` + merge + `}`), nil + case f.failBuild.Load() && tool != "build_image_copy": // the build fails wherever it runs + return nil, fmt.Errorf("go build of %v: boom", args["artifact"]) + case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4) + d := digestOf(fmt.Sprint(args["artifact"])) + return json.RawMessage(`{"artifact":"` + fmt.Sprint(args["artifact"]) + `","digest":"` + d + `","reference":"registry.example/` + fmt.Sprint(args["module"]) + `/` + fmt.Sprint(args["artifact"]) + `@` + d + `","pushed":true}`), nil + case tool != "build_image_build" && f.refusePush.Load(): + return nil, fmt.Errorf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"]) + case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1) + d := digestOf(fmt.Sprint(args["artifact"])) + return json.RawMessage(`{"digest":"` + d + `","reference":"artifact-store://` + filepath.Base(fmt.Sprint(args["module"])) + `/` + fmt.Sprint(args["artifact"]) + `/blobs/` + d + `"}`), nil + case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference + d := digestOf(fmt.Sprint(args["artifact"])) + return json.RawMessage(`{"digest":"` + d + `","reference":"registry.example/imaged/server@` + d + `"}`), nil + default: + return json.RawMessage(`{"digest":"` + digestOf(fmt.Sprint(args["artifact"])) + `"}`), nil + } +} + +// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on nodes. +func (f *fakeBuildModules) serve(t *testing.T, module string, nodes ...string) { + kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1}) if err != nil { t.Fatal(err) } - sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) { - tokens := strings.Split(m.Subject, ".") // mesh mod call - tool, node := tokens[4], tokens[5] - f.mu.Lock() - f.calls[module+"."+tool]++ - f.mu.Unlock() - var args map[string]any - _ = json.Unmarshal(m.Data, &args) - if strings.HasSuffix(tool, "_fetch") { - f.mu.Lock() - f.fetches = append(f.fetches, args) - f.mu.Unlock() - } else { - f.mu.Lock() - f.builds = append(f.builds, args) - f.mu.Unlock() - } - id := fmt.Sprintf("s%d", f.n.Add(1)) - key, _ := delivery.StepKey(node, id) - put := func(s delivery.Step) { - data, err := delivery.EncodeStep(s) - if err == nil { - _, err = steps.Put(f.ctx, key, data) - } - if err != nil { - t.Errorf("writing step %s: %v", key, err) - } - } - step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()} - put(step) - answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node}) - _ = m.Respond(answer) - go func() { - time.Sleep(30 * time.Millisecond) - step.Ended = time.Now().UTC() - switch { - case strings.HasSuffix(tool, "_fetch"): - merge := "" - if args["base"] != nil { // a fetch with a base answers the merge commit (mesh-build-go FetchResult) - merge = `,"merge":"` + strings.Repeat("9", 40) + `"` + language := strings.TrimPrefix(module, "mesh-build-") + if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil { + t.Fatal(err) + } + put := func(key string, value any) (uint64, error) { + raw, _ := value.(json.RawMessage) + return kv.Put(f.ctx, key, raw) + } + call := func(name string) buildstep.StepFunc { + return func(ctx context.Context, args map[string]any) (any, error) { return f.step(module, name, args) } + } + steps := buildstep.AskSteps{Fetch: call("fetch"), Build: call("build"), Copy: call("copy")} + for _, node := range nodes { + p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: node, Put: put, + Capacity: func() int { + if f.closed.Load() { + return 0 } - step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"workspace":"w`+id+`","commit":"`+commit+`"`+merge+`}`) - case f.failBuild.Load() && tool != "build_image_copy": // every node's build fails: one node builds a fingerprint - step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("go build of %v: boom", args["artifact"]) - case tool == "build_image_copy": // the upstream copy answers the digest and the registry's reference (ADR 0498 clause 4) - d := digestOf(fmt.Sprint(args["artifact"])) - step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"artifact":"`+fmt.Sprint(args["artifact"])+`","digest":"`+d+`","reference":"registry.example/`+fmt.Sprint(args["module"])+`/`+fmt.Sprint(args["artifact"])+`@`+d+`","pushed":true}`) - case tool != "build_image_build" && f.refusePush.Load(): - step.State, step.Reason = delivery.StepFailed, fmt.Sprintf("%s%v: the artifact store answered 507 Insufficient Storage", delivery.NotStored, args["artifact"]) - case tool != "build_image_build": // a bundle or an archive is pushed and its reference answered (ADR 0498 clause 1) - d := digestOf(fmt.Sprint(args["artifact"])) - step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"artifact-store://`+filepath.Base(fmt.Sprint(args["module"]))+`/`+fmt.Sprint(args["artifact"])+`/blobs/`+d+`"}`) - case tool == "build_image_build" && args["publish"] == true: // a trunk image is pushed: the digest and the reference - d := digestOf(fmt.Sprint(args["artifact"])) - step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+d+`","reference":"registry.example/imaged/server@`+d+`"}`) - default: - step.State, step.Result = delivery.StepPassed, json.RawMessage(`{"digest":"`+digestOf(fmt.Sprint(args["artifact"]))+`"}`) - } - put(step) - }() - }) - if err != nil { - t.Fatal(err) + return 2 + }, PullWait: 200 * time.Millisecond, + Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }} + f.mu.Lock() + f.pullers = append(f.pullers, p) + f.mu.Unlock() + go func() { _ = p.Pull(f.ctx) }() } - t.Cleanup(func() { _ = sub.Unsubscribe() }) } type bed struct { @@ -154,8 +169,6 @@ type bed struct { candidates jetstream.KeyValue builds jetstream.KeyValue tools map[string]jetstream.KeyValue - presence jetstream.KeyValue - have jetstream.KeyValue instances jetstream.KeyValue stop func() // stops the builder started last fake *fakeBuildModules @@ -228,20 +241,11 @@ func startBed(t *testing.T) *bed { b.candidates = mk(delivery.CandidatesBucket, delivery.CandidatesHistory) mk(delivery.ManifestsBucket, delivery.ManifestsHistory) b.builds = mk(delivery.BuilderBuildsBucket, delivery.BuilderBuildsHistory) - presence := mk(delivery.PresenceBucket, delivery.PresenceHistory) - b.presence, b.have = presence, mk(delivery.HaveBucket, delivery.HaveHistory) b.instances = mk(instancesBucket, 1) - for _, node := range []string{"n1", "n2"} { - b.setOnline(t, node, true) - for _, module := range []string{"mesh-build-go", "mesh-build-node"} { - b.setHealth(t, node, module, delivery.HealthOK) - } - } b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} - b.fake.serve(t, ctl, "mesh-build-go") - b.fake.serve(t, ctl, "mesh-build-node") - b.fake.serve(t, ctl, "mesh-build-image") - b.setHealth(t, "n1", "mesh-build-image", delivery.HealthOK) // the image build module runs on n1 only + b.fake.serve(t, "mesh-build-go", "n1", "n2") + b.fake.serve(t, "mesh-build-node", "n1", "n2") + b.fake.serve(t, "mesh-build-image", "n1") // the image build module runs on n1 only b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1") b.report(t, "mesh-build-go", "n2", "go", "go1.27.1", "1") b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1") @@ -249,28 +253,6 @@ func startBed(t *testing.T) *bed { return b } -// setOnline writes the watchman's presence of a node. -func (b *bed) setOnline(t *testing.T, node string, online bool) { - t.Helper() - data, _ := delivery.EncodePresence(delivery.Presence{Online: online}) - if _, err := b.presence.Put(b.ctx, node, data); err != nil { - t.Fatal(err) - } -} - -// setHealth writes the node-engine's have of a build module on a node, running with this health. -func (b *bed) setHealth(t *testing.T, node, module string, health delivery.Health) { - t.Helper() - data, err := delivery.EncodeHave(delivery.Have{Phase: delivery.PhaseRunning, Health: health}) - if err != nil { - t.Fatal(err) - } - key, _ := delivery.HaveKey(node, module) - if _, err := b.have.Put(b.ctx, key, data); err != nil { - t.Fatal(err) - } -} - // report writes a node's toolchains key of a build module and says so on the module's stream, as the module does. func (b *bed) report(t *testing.T, module, node, language, version, recipe string) { t.Helper() @@ -433,20 +415,8 @@ func TestAnArchiveOnlyModuleNamingNoToolchainIsBuilt(t *testing.T) { } } -// TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed: the node-engine's have of mesh-build-go on n2 says failed, so only n1 builds. -func TestANodeWhoseBuildModuleIsUnhealthyIsNotUsed(t *testing.T) { - b := startBed(t) - b.setHealth(t, "n2", "mesh-build-go", delivery.HealthFailed) - b.archive(t, "ca-trust", "modules/ca-trust", commit) - b.start(t, "n1", nil) - v := b.await(t, "ca-trust", isState(delivery.BuildDone)) - if !strings.Contains(v.Step, "on n1") { - t.Errorf("build = %+v, want it built on n1 alone", v) - } -} - -// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails on one node, so the -// key is failed naming it; the indexer writing modules. again with the same fingerprint wakes the builder, which builds +// TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint: a build step fails, so the key is +// failed naming it; the indexer writing modules. again with the same fingerprint wakes the builder, which builds // the key again as a new revision of the same key, now done. func TestAFailedKeyIsBuiltAgainAsANewRevisionFromANewModulesValueWithTheSameFingerprint(t *testing.T) { // 1607/AC6, AC7 b := startBed(t) @@ -742,3 +712,106 @@ func TestABuilderWhoseBuildModulesAnsweredReferencesSaysItAndStoppedSaysNothing( t.Fatalf("the stopped builder still says: %v", err) } } + +// 1887/AC1, AC2 on the real bus: the builder puts the trunk fingerprint's build as asks on the build seats' delivery +// streams and calls no node's tool; the build modules that pulled them wrote the steps, and the key is done naming the +// references. +func TestTheBuilderAsksTheBuildSeatsAndCallsNoNode(t *testing.T) { + b := startBed(t) + b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") + sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) + if err != nil { + t.Fatal(err) + } + defer sub.Close() + var calls atomic.Int64 + s, err := sub.Subscribe("mesh.mod.*.call.>", func(*nats.Msg) { calls.Add(1) }) + if err != nil { + t.Fatal(err) + } + defer s.Unsubscribe() + asked := b.asksPut(t, sub) + b.start(t, "n1", nil) + v := b.await(t, "netcheck", isState(delivery.BuildDone)) + for _, a := range v.Artifacts { + if a.Reference == "" { + t.Errorf("artifact %s names no reference", a.Name) + } + } + if n := calls.Load(); n != 0 { + t.Errorf("%d tool calls reached a build module", n) + } + // 1887/AC1: one ask per build module the build needs (netcheck: a Go and a TypeScript bundle), each on its seat's + // delivery stream. + got := asked() + if len(got) != 2 { + t.Fatalf("asks put = %+v, want one Go and one Node delivery ask", got) + } + for _, a := range got { + if a.Kind != delivery.AskDelivery || a.Build == nil || a.Build.FP != fp { + t.Errorf("ask %+v, want a delivery ask of the fingerprint", a) + } + } +} + +// asksPut hears every ask put on a build seat's accept subjects (a core subscription beside the stream) and answers the +// asks heard so far. +func (b *bed) asksPut(t *testing.T, nc *nats.Conn) func() []delivery.StepAsk { + t.Helper() + var mu sync.Mutex + var asks []delivery.StepAsk + s, err := nc.Subscribe("mesh.seat.*.accept.>", func(m *nats.Msg) { + if a, err := delivery.DecodeStepAsk(m.Data); err == nil { + mu.Lock() + asks = append(asks, a) + mu.Unlock() + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Unsubscribe() }) + return func() []delivery.StepAsk { + mu.Lock() + defer mu.Unlock() + return append([]delivery.StepAsk(nil), asks...) + } +} + +// 1887/AC3: a builder that started again while its asks waited in the streams (no node had a free slot) puts no second +// ask: it waits for the asks its building key names, and the key is done once a node took them. +func TestARestartedBuilderPutsNoSecondAsk(t *testing.T) { + b := startBed(t) + b.fake.closed.Store(true) + b.module(t, "netcheck", "catalogue-a67f0196-netcheck.module.json", "modules/netcheck") + sub, err := nats.Connect(b.srvURL, nats.UserInfo("ctl", "pw")) + if err != nil { + t.Fatal(err) + } + defer sub.Close() + asked := b.asksPut(t, sub) + b.start(t, "n1", nil) + v := b.await(t, "netcheck", func(v delivery.BuilderBuild) bool { return v.State == delivery.BuildBuilding && len(v.Asks) == 2 }) + for deadline := time.Now().Add(5 * time.Second); len(asked()) < 2 && time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + } + b.stop() + b.start(t, "n1", nil) // the same builder, started again, while both asks wait + time.Sleep(500 * time.Millisecond) + b.fake.open() + done := b.await(t, "netcheck", isState(delivery.BuildDone)) + if got := asked(); len(got) != 2 { + t.Fatalf("asks put = %d (%+v), want the two the key named and none again", len(got), got) + } + for lang, id := range v.Asks { + found := false + for _, a := range asked() { + found = found || a.ID == id + } + if !found { + t.Errorf("the %s ask %s the key named was never put", lang, id) + } + } + if len(done.Artifacts) != 2 { + t.Errorf("done = %+v", done) + } +} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go index 822b325..6e9d5db 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/main.go @@ -26,14 +26,14 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/process" "mesh-delivery-builder/internal/build" ) const ( - // defaultStepBound is how long the builder waits for one step of a build module (a fetch downloads the Go modules, a - // build compiles). + // defaultStepBound is how long one ask's step may run once a build node took it (a fetch downloads the Go modules, a + // build compiles): from its start, never from its put, so an ask waiting in its stream for a free slot is not failed + // (buildrun.Asker.Bound; ADR 0525 §3, §6). defaultStepBound = 30 * time.Minute ) @@ -143,44 +143,28 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - // Calls to the build modules go out as this module's own account, naming the machine (mesh-sdk process.CallOn); the - // buckets are read and written on the connection above. - account, err := process.ReadAccount(path) - if err != nil { - return err - } - caller, err := process.Connect(account, nats.Name(cred.Node+"."+cred.Module+".calls"), nats.RetryOnFailedConnect(true)) - if err != nil { - return err - } - defer caller.Close() - bound := defaultStepBound if v := strings.TrimSpace(getenv("MESH_BUILDER_STEP_BOUND")); v != "" { if bound, err = time.ParseDuration(v); err != nil { return fmt.Errorf("MESH_BUILDER_STEP_BOUND: %w", err) } } - capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_BUILDER_SETTINGS")), getenv("MESH_BUILDER_CAPACITY")) - if err != nil { - return fmt.Errorf("the build capacity setting: %w", err) - } tools := &buildrun.ToolchainSet{} - presence := &buildrun.PresenceSet{} - have := &buildrun.HaveSet{} + // Each build module's part of a build is an ask on its build seat's stream, pulled by a build node with a free slot + // (novox/hq ADR 0525): the builder chooses no node, so it reads neither presence nor have and holds no capacity. + asks := buildrun.JetStreamAsks{JS: js} b := &build.Builder{ Claimer: build.Claimer{Store: kvStore{builds}, By: cred.Node + "." + cred.Module}, - Runner: buildrun.Runner{Caller: caller, Steps: stepsWaiter{js}, Bound: bound}, + Asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: bound}, + Held: heldOnTheBus{js}, Toolchains: tools, - Presence: presence, - Health: have, - Places: &buildrun.Places{Capacity: capacity}, Manifests: manifestReader{manifests}, Logf: logger.Printf, } // **Said once every build module answered a reference, taken away when stopped** (issue 1886, ADR 0522): the // controller delivers from this builder's keys only while an instance says it writes references. A bucket not there - // yet (the manifest's state not made) is said, and the controller keeps asking build-agent. + // yet (the manifest's state not made) is said, and the controller's plan waits for the keys, named (no build-agent + // is asked since R4g, ADR 0524 §7). switch kv, err := js.KeyValue(ctx, instancesBucket); { case errors.Is(err, jetstream.ErrBucketNotFound): logger.Printf("the bucket %s does not exist yet: the controller is not told that this builder writes references until it starts again", instancesBucket) @@ -219,48 +203,6 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } defer cc.Stop() } - // The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing - // builds; the builder says so and is started again with the watchman. - presenceErrs := make(chan error, 1) - switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no node counts as online until the builder starts again", delivery.PresenceBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.PresenceBucket, err) - default: - pw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { presence.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err - } - go func() { - presenceErrs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { - if presence.Put(e.Key(), e.Value(), !opOf(e)) { - b.NodesChanged(ctx) // a node came online or went: a build that waited for one may go on - } - }) - }() - } - // The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not - // chosen (ADR 0478 clause 3). Where its bucket is not there yet no build module counts as healthy, so nothing builds. - haveErrs := make(chan error, 1) - switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the builder starts again", delivery.HaveBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err) - default: - hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err - } - go func() { - haveErrs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) { - if have.Put(e.Key(), e.Value(), !opOf(e)) { - b.NodesChanged(ctx) // a build module became healthy or stopped being: a build that waited may go on - } - }) - }() - } each := func(e jetstream.KeyValueEntry) { if opOf(e) { b.Module(ctx, e.Key(), e.Value()) @@ -300,9 +242,5 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-watch.errs: return err - case err := <-presenceErrs: - return err - case err := <-haveErrs: - return err } } diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go index 8ff0302..2454b21 100644 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go +++ b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/says.go @@ -13,7 +13,7 @@ import ( // done key it writes names where its artifacts are (ADR 0498 §1). The references come from the build modules' own pushes, // so an instance says it only once a step of every build module it calls answered one since it started: a builder beside // a build module from before R4e says nothing. The controller delivers from the builder's keys only while an instance -// says so, and asks build-agent as before R4e otherwise. The bucket is read by the controller alone, so its name and shape +// says so; since R4g it asks no build-agent, and a module whose key is not yet done waits, named (ADR 0524 §7). The bucket is read by the controller alone, so its name and shape // are not in mesh-sdk. const ( instancesBucket = "mesh-delivery-builder_instances" diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go deleted file mode 100644 index e0255f1..0000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings.go +++ /dev/null @@ -1,28 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" - "os" -) - -// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `=` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a -// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch. -func capacityFrom(path, fallback string) (map[string]int, error) { - if path == "" { - return buildrun.ParseCapacity(fallback) - } - raw, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("the settings file: %w", err) - } - var s struct { - Capacity string `json:"capacity"` - } - if err := json.Unmarshal(raw, &s); err != nil { - return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err) - } - return buildrun.ParseCapacity(s.Capacity) -} diff --git a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings_test.go b/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings_test.go deleted file mode 100644 index 31b1acf..0000000 --- a/modules/mesh-delivery-builder/cmd/mesh-delivery-builder/settings_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import ( - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" - "os" - "path/filepath" - "testing" -) - -func settingsFile(t *testing.T, body string) string { - t.Helper() - p := filepath.Join(t.TempDir(), "settings.json") - if err := os.WriteFile(p, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - return p -} - -// 1661: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere. -func TestCapacityIsReadFromTheSettingsFile(t *testing.T) { - for body, want := range map[string]map[string]int{ - `{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2}, - `{"capacity": ""}`: {}, - `{}`: {}, - } { - got, err := capacityFrom(settingsFile(t, body), "") - if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] { - t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want) - } - } - p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))} - if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 { - t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace")) - } -} - -func mustCapacity(t *testing.T, path string) map[string]int { - t.Helper() - c, err := capacityFrom(path, "") - if err != nil { - t.Fatal(err) - } - return c -} - -func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) { - if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil { - t.Error("shanks=-1 accepted") - } - if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 { - t.Errorf("the starting value: %v, %v", c, err) - } - if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil { - t.Error("a file that is not JSON accepted") - } - if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil { - t.Error("a settings file that is not there accepted") - } -} - -// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's. -func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) { - got, err := capacityFrom("", "ace=2") - if err != nil || got["ace"] != 2 { - t.Errorf("capacity %v, %v", got, err) - } -} diff --git a/modules/mesh-delivery-builder/files/settings.json b/modules/mesh-delivery-builder/files/settings.json deleted file mode 100644 index 3175fe3..0000000 --- a/modules/mesh-delivery-builder/files/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "capacity": "shanks=3,ace=2,g14=0,novox=0" -} diff --git a/modules/mesh-delivery-builder/go.mod b/modules/mesh-delivery-builder/go.mod index 284143f..d5b8384 100644 --- a/modules/mesh-delivery-builder/go.mod +++ b/modules/mesh-delivery-builder/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-builder go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-builder/go.sum b/modules/mesh-delivery-builder/go.sum index db1ea18..2a3e9f3 100644 --- a/modules/mesh-delivery-builder/go.sum +++ b/modules/mesh-delivery-builder/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-builder/internal/build/fixtures_test.go b/modules/mesh-delivery-builder/internal/build/fixtures_test.go index b5a6566..4ecc3e0 100644 --- a/modules/mesh-delivery-builder/internal/build/fixtures_test.go +++ b/modules/mesh-delivery-builder/internal/build/fixtures_test.go @@ -1,10 +1,14 @@ package build import ( + "context" "encoding/json" "os" + "strings" "testing" + "time" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -25,17 +29,31 @@ func captured(t *testing.T, name string) json.RawMessage { return step.Result } +// The captured image steps, run as a pulled ask runs them (buildstep.RunAsk), answer what the builder reads: the trunk's +// build its digest and reference, a candidate's its digest and no reference (issue 1887 A4a). func TestCapturedImageStepsDecodeWithTheBuildersOwnTypes(t *testing.T) { // review r2, mesh-catalog 2; ADR 0487 clause 1 - var f fetchResult - if err := json.Unmarshal(captured(t, "fetch.json"), &f); err != nil || f.Workspace == "" { - t.Errorf("fetch = %+v, %v, want a workspace (the field the builder reads)", f, err) - } - var trunk buildResult - if err := json.Unmarshal(captured(t, "build-trunk.json"), &trunk); err != nil || trunk.Digest == "" || trunk.Reference == "" { - t.Errorf("trunk build = %+v, %v, want a digest and a reference", trunk, err) - } - var cand buildResult - if err := json.Unmarshal(captured(t, "build-candidate.json"), &cand); err != nil || cand.Digest == "" || cand.Reference != "" { - t.Errorf("candidate build = %+v, %v, want a digest and no reference", cand, err) + for _, tc := range []struct { + build string + publish bool + ref bool + }{{"build-trunk.json", true, true}, {"build-candidate.json", false, false}} { + steps := buildstep.AskSteps{ + Fetch: func(context.Context, map[string]any) (any, error) { + return json.RawMessage(captured(t, "fetch.json")), nil + }, + Build: func(context.Context, map[string]any) (any, error) { return json.RawMessage(captured(t, tc.build)), nil }, + } + ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: delivery.AskDelivery, Language: delivery.AskImage, Asker: delivery.BuilderModule, + At: time.Now(), Build: &delivery.BuildJob{Module: "m", Path: "modules/m", Publish: tc.publish, + Fetch: map[string]any{"repository": "novox/mesh-catalog", "commit": strings.Repeat("a", 40)}, + Artifacts: []delivery.AskArtifact{{Artifact: "server"}}}} + out, err := buildstep.RunAsk(context.Background(), steps, ask) + if err != nil { + t.Fatalf("%s: %v", tc.build, err) + } + a := out.(delivery.BuildAnswer) + if len(a.Artifacts) != 1 || a.Artifacts[0].Digest == "" || (a.Artifacts[0].Reference != "") != tc.ref { + t.Errorf("%s answered %+v, want a digest and a reference only for the trunk", tc.build, a) + } } } diff --git a/modules/mesh-delivery-builder/internal/build/r4e_test.go b/modules/mesh-delivery-builder/internal/build/r4e_test.go index d3d851c..fab0854 100644 --- a/modules/mesh-delivery-builder/internal/build/r4e_test.go +++ b/modules/mesh-delivery-builder/internal/build/r4e_test.go @@ -111,7 +111,7 @@ func TestADoneKeyWrittenBeforeR4eIsBuiltAgainOnceAsANewRevision(t *testing.T) { func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheReference(t *testing.T) { // B4, ADR 0498 clause 4 up := catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest} r := newRig(t, up) - r.b.Health = imageHave(t, "ace", "novox") + r.puller = "ace" v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 1 { t.Fatalf("build = %+v, want done with the copy", v) @@ -133,7 +133,7 @@ func TestAPinnedUpstreamIsCopiedOnceOnOneImageNodeWithNoFetchAndItsKeyNamesTheRe func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode(t *testing.T) { // B4 r := newRig(t, imageArt, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) - r.b.Health = imageHave(t, "ace") + r.puller = "ace" v := r.consider(t, "modules/m") if ok, missing := v.Referenced(); !ok { t.Fatalf("build = %+v, missing references %v", v, missing) @@ -145,7 +145,7 @@ func TestAnUpstreamBesideAnImageIsCopiedAndTheImageFetchedAndBuiltOnTheSameNode( func TestACopyThatFailsNotStoredFailsTheKeyAndIsBuiltAgain(t *testing.T) { // B4, ADR 0498 clause 2 r := newRig(t, catalogue.Artifact{Name: "up", Kind: "upstream", From: "nats:2.11.17-alpine@" + digest}) - r.b.Health = imageHave(t, "ace") + r.puller = "ace" r.bm.fails["ace/build_image_copy"] = delivery.NotStored + "up: the registry answered 500" v := r.consider(t, "modules/m") if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, delivery.NotStored) || delivery.FailedUntilChanged(v.Error) { diff --git a/modules/mesh-delivery-builder/internal/build/round4_test.go b/modules/mesh-delivery-builder/internal/build/round4_test.go index f2bcc51..e396831 100644 --- a/modules/mesh-delivery-builder/internal/build/round4_test.go +++ b/modules/mesh-delivery-builder/internal/build/round4_test.go @@ -3,7 +3,6 @@ package build import ( "context" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" - "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -39,26 +38,6 @@ func TestAnArchiveOnlyModuleIsBuiltAlthoughItsFingerprintNamesNoToolchain(t *tes } } -func TestAnArchiveOnlyModuleNeedsNoAgreedToolchainButNeedsNodesThatQualify(t *testing.T) { // review round 4, 1 - r := newRig(t, archiveA) - r.tools.Put(buildrun.GoBuildModule, "ace.go", tcValue(t, "go", "go1.26.0"), false) // the nodes disagree: a bundle would wait - r.b.Health = &buildrun.HaveSet{} // no build module is healthy anywhere - r.b.Module(context.Background(), "m", moduleValueWith(t, "modules/m", nil)) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") { - t.Errorf("build = %+v, want failed: no node's build module is healthy", v) - } - have := &buildrun.HaveSet{} - have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - r.b.Health = have - r.b.Reconsider(context.Background()) - r.b.Wait() - v := valueOf(t, r.st, r.key) - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { - t.Errorf("build = %+v, want done on novox, the one node that qualifies", v) - } -} - func TestABundleAndAnArchiveOfOneBuildModuleStillNeedTheAgreedToolchain(t *testing.T) { // review round 4, 1 p, err := PlanOf(catalogue.Manifest{Module: "m", Build: &catalogue.Build{Artifacts: []catalogue.Artifact{goBundle, archiveA}}}) if err != nil || !p.NeedsToolchain(buildrun.GoBuildModule) { @@ -80,115 +59,6 @@ func TestClaimLeavesAKeyFailedForDifferingDigestsAlone(t *testing.T) { // review } } -// 4. A step never waits on a full node while another qualifying node has room. -func TestChoosesMadeAtOnceSeeEachOthersReservationsSoNoLaneWaitsWhileANodeHasRoom(t *testing.T) { // review round 4, 4 - p := &buildrun.Places{Capacity: map[string]int{"ace": 2, "g14": 1, "novox": 1, "shanks": 3}} - entries := tcs("go", "go1.27.1", "ace", "g14", "novox", "shanks") - total := 2 + 1 + 1 + 3 - var waiting int - for i := 1; i <= 4; i++ { // four modules choose before any lane has started - room := 0 - for _, n := range []string{"ace", "g14", "novox", "shanks"} { - room += p.CapacityOf(n) - p.Running()[n] - p.Reserved()[n] - } - nodes, _, err := buildrun.ChooseNodes("go", entries, p, nil, 2) - if err != nil || len(nodes) != 2 { - t.Fatalf("choice %d: %v, %v", i, nodes, err) - } - if room >= 2 { - for _, n := range nodes { - if p.Reserved()[n] > p.CapacityOf(n) { - t.Errorf("choice %d: %v put a lane on full %s while %d places were free", i, nodes, n, room) - } - } - } else { - waiting += 2 - room - } - } - if want := 4*2 - total; waiting != want { - t.Errorf("%d lanes wait, want %d: the 8 lanes less the %d places", waiting, want, total) - } - loads := p.Reserved() - for n, c := range loads { - if c > p.CapacityOf(n) && n != "shanks" && n != "ace" && n != "g14" && n != "novox" { - t.Errorf("unknown node %s", n) - } - } - if loads["shanks"] != 3 || loads["ace"] != 2 || loads["g14"]+loads["novox"] != 3 { - t.Errorf("reservations = %v, want shanks 3, ace 2 and the single-place nodes taking the rest", loads) - } -} - -func TestACancelledReservationFreesTheChoiceAndAnAcquireTakesItsReservation(t *testing.T) { // review round 4, 4 - p := &buildrun.Places{} - nodes, _, _ := buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) - if len(nodes) != 1 || p.Reserved()["ace"] != 1 { - t.Fatalf("nodes %v, reserved %v", nodes, p.Reserved()) - } - release, err := p.Acquire(context.Background(), "ace") - if err != nil || p.Reserved()["ace"] != 0 || p.Running()["ace"] != 1 { - t.Fatalf("acquire: %v, reserved %v, running %v", err, p.Reserved(), p.Running()) - } - release() - buildrun.ChooseNodes("go", tcs("go", "go1.27.1", "ace"), p, nil, 1) - p.Cancel("ace") - if len(p.Reserved()) != 0 { - t.Errorf("reserved after Cancel = %v", p.Reserved()) - } -} - -// 5. A node whose build module is not healthy is not chosen. -func TestHaveSetKnowsOnlyBuildModulesRunningAndHealthy(t *testing.T) { // review round 4, 5 - var s buildrun.HaveSet - if s.Healthy("ace", buildrun.GoBuildModule) { - t.Error("a module with no have is healthy") - } - if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) || !s.Healthy("ace", buildrun.GoBuildModule) { - t.Error("a running, healthy module is not") - } - if s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) { - t.Error("the same state counted as a change") - } - if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) || s.Healthy("ace", buildrun.GoBuildModule) { - t.Error("a module whose health failed is healthy") - } - s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - if !s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseFailed, delivery.HealthOK), false) || s.Healthy("ace", buildrun.GoBuildModule) { - t.Error("a module whose phase failed is healthy") - } - s.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - if !s.Put("ace.mesh-build-go", nil, true) || s.Healthy("ace", buildrun.GoBuildModule) { - t.Error("a deleted have is healthy") - } - if s.Put("ace.node", []byte(`{}`), false) || s.Put("ace.mesh-build-go", []byte(`not json`), false) { - t.Error("a node's key or a non-have counted") - } -} - -func TestANodeWhoseBuildModuleIsNotHealthyIsNotChosen(t *testing.T) { // review round 4, 5 - r := newRig(t, goBundle) - have := &buildrun.HaveSet{} - have.Put("ace.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) // go is not the recipe's - have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - r.b.Health = have - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { - t.Fatalf("build = %+v, want done on novox alone", v) - } - for _, c := range r.bm.calls { - if c.node == "ace" { - t.Errorf("a step was called on ace, whose build module is unhealthy: %+v", c) - } - } - have.Put("novox.mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthFailed), false) - r.st.vals = map[string][]byte{} // a fresh key - r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) - r.b.Wait() - if v := valueOf(t, r.st, r.key); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "healthy") { - t.Errorf("build = %+v, want failed: no node has a healthy build module", v) - } -} - // 6. A key found building is taken over, whoever holds it. func TestABuildLeftBuildingByAnyBuilderIsTakenOverAtStart(t *testing.T) { // review round 4, 6; 1607/AC3, ADR 0478 clauses 5, 6 st := newMem() diff --git a/modules/mesh-delivery-builder/internal/build/run.go b/modules/mesh-delivery-builder/internal/build/run.go index 3021dd0..f4f2035 100644 --- a/modules/mesh-delivery-builder/internal/build/run.go +++ b/modules/mesh-delivery-builder/internal/build/run.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "maps" "sort" "strings" "sync" + "time" "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -20,29 +22,29 @@ type ToolchainEntries interface { Entries(language string) []delivery.NodeToolchain } -// toolPrefix is the start of each build module's tool names: fetch and build. -var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"} - // Builder builds, dark, every new-form fingerprint the indexer writes (novox/hq ADR 0453 clause 4, ADR 0478): it is // handed each modules value as the watch of the indexer's bucket sees it, and for each module with a fingerprint and an -// artifact to build claims the fingerprint's build key, builds it through the build modules on the nodes -// the toolchains choose, and writes the key done or failed. It holds no timer; a build starts from a modules value or a +// artifact to build claims the fingerprint's build key, builds it by asking the build seats (one ask per build module, +// pulled by whichever build node has a free slot: ADR 0525), and writes the key done or failed. It holds no timer; a build starts from a modules value or a // change of the toolchains. type Builder struct { - Claimer Claimer - Runner buildrun.Runner - Toolchains ToolchainEntries - // Presence says which nodes are online (the watchman's); a node that is not is not chosen. - Presence interface{ Online(node string) bool } - Manifests Manifests - // Health says whether a node's build module is running and healthy (the node-engine's have); a node whose build module - // is not is not chosen. Nil means every node's is. - Health interface { - Healthy(node, module string) bool + Claimer Claimer + // Asks puts each build module's part of a build as one ask on its build seat's stream and waits for its step (novox/hq + // ADR 0525 §1, §3, §6; issue 1887 A4): the builder chooses no node, whichever build node pulls the ask builds it. + Asks interface { + Put(ctx context.Context, a delivery.StepAsk) error + Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) } - // buildrun.Places holds a step back while its node is at its build capacity; a Builder with none runs one step at a time on each node. - Places *buildrun.Places - // Limit is how many builds run at once; four when 0. + // Held says whether an ask a key names is on its stream or a build node wrote a step for it (issue 1910): a key taken + // over at start whose ask is neither was left by a builder that stopped between naming the ask and putting it, and + // the ask is put then, under the same id. Nil takes every named ask as held. + Held interface { + Held(ctx context.Context, a delivery.StepAsk) (bool, error) + } + Toolchains ToolchainEntries + Manifests Manifests + // Limit is how many builds the builder has asked at once; their asks wait in the build seats' streams, which are the + // backlog (ADR 0525 §9), not here. 256 when 0. Limit int Logf func(format string, args ...any) // Answered is told the build module whose step answered a reference (issue 1886): what the builder says it writes @@ -75,15 +77,12 @@ func (b *Builder) init() { b.once.Do(func() { limit := b.Limit if limit <= 0 { - limit = 4 + limit = 256 } b.sem = make(chan struct{}, limit) b.cands, b.via, b.trunkFP = map[string]delivery.Candidate{}, map[string]via{}, map[string]bool{} b.known, b.inflight, b.again, b.plans, b.tried = map[string]delivery.Module{}, map[string]bool{}, map[string]delivery.Module{}, map[string]planned{}, map[string]bool{} b.published = map[string]bool{} - if b.Places == nil { - b.Places = &buildrun.Places{} - } }) } @@ -116,8 +115,7 @@ func (b *Builder) Module(ctx context.Context, key string, value []byte) { b.consider(ctx, key, m) } -// NodesChanged is Reconsider after a change of the nodes: one came online or went, a build module became healthy or -// stopped being, or a toolchain moved, so a build that waited for a node may go on. +// NodesChanged is Reconsider after a toolchain moved, so a build that waited for the nodes to agree on one may go on. func (b *Builder) NodesChanged(ctx context.Context) { b.Reconsider(ctx) } // Sweep reads, at start, every key of the bucket: it keeps the package versions done keys name as published (a key @@ -314,7 +312,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { first := !b.tried[key] b.tried[key] = true b.mu.Unlock() - cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: b.imageBuilderRuns, Trunk: trunk, Published: b.isPublished}) + cl, err := b.Claimer.Claim(ctx, module, m.FP2, m.Manifest, ClaimOptions{TakeOver: first, ImageBuilder: imageSeat, Trunk: trunk, Published: b.isPublished}) if err != nil { b.logf("%s: %v", module, err) return @@ -330,7 +328,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { if cl.Build.State == delivery.BuildDone { prior, only = &cl.Build, map[string]bool{} if trunk { - for _, bm := range publishable(cl.Build, b.imageBuilderRuns) { + for _, bm := range publishable(cl.Build, imageSeat) { only[bm] = true publishing = true } @@ -342,7 +340,7 @@ func (b *Builder) try(ctx context.Context, module string, m delivery.Module) { // any other, so the controller has a done key to register it from (issue 1613, ADR 0478 clause 1, ADR 0501 clause 1). result = delivery.BuilderBuild{State: delivery.BuildDone, Step: "nothing to build: the manifest names no artifact"} } else { - result = b.build(ctx, module, m, plan, planErr, prior, only) + result = b.build(ctx, module, m, plan, planErr, prior, only, b.notes(cl)) } if ctx.Err() != nil { return @@ -399,14 +397,15 @@ func failed(format string, args ...any) delivery.BuilderBuild { // lane is the build of a plan's tasks of one build module on one node. type lane struct { - module, node string + module, node string // node is the one that pulled the lane's ask tasks []Task + toolchain *delivery.ToolchainUsed digests map[string]string // artifact -> digest references map[string]string // artifact -> the reference an image was pushed to (a trunk build only) err error } -func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error, prior *delivery.BuilderBuild, only map[string]bool) delivery.BuilderBuild { +func (b *Builder) build(ctx context.Context, module string, m delivery.Module, plan Plan, planErr error, prior *delivery.BuilderBuild, only map[string]bool, asked *askNotes) delivery.BuilderBuild { switch { case planErr != nil: return failed("%s", planErr) @@ -417,76 +416,39 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p repository := delivery.RepositoryName(m.Repository) var lanes []*lane used := map[string]delivery.ToolchainUsed{} - var steps []string - // Every node chosen holds a reservation until its lane takes the place; a build that ends before its lanes start gives them up. - giveUp := func(nodes ...string) { - for _, n := range nodes { - b.Places.Cancel(n) - } - } - giveUpLanes := func() { - for _, l := range lanes { - b.Places.Cancel(l.node) - } - } var kept []string // build modules whose artifacts stay as the prior done key has them for _, bm := range plan.Modules() { if prior != nil && len(only) > 0 && !only[bm] { kept = append(kept, bm) continue } - language := buildrun.LanguageOf[bm] - entries := b.Toolchains.Entries(language) - qualifies := func(node string) bool { return b.qualifies(node, bm) } - var nodes []string - if bm == ImageBuildModule { - // An image is built on one node, chosen from the nodes whose image build module runs healthy (ADR 0487 clause 8): - // the module has no toolchain language, so no toolchain entry names its nodes, and a second build would only - // push the image again. - ns, err := buildrun.ChooseAnyNodes("image", b.imageEntries(), b.Places, qualifies, 1) - if err != nil { - giveUpLanes() - return failed("%s no node has %s, so the image cannot be built: the provision rootless-image-build is held by the module buildah; assign buildah and %s to a node (%v)", NoImageBuilderPrefix, ImageBuildModule, ImageBuildModule, err) - } - nodes = ns - } else if plan.NeedsToolchain(bm) { - ns, tc, err := buildrun.ChooseNodes(language, entries, b.Places, qualifies, 1) - if err != nil { - giveUpLanes() - return failed("%s", err) + var toolchain *delivery.ToolchainUsed + if bm != ImageBuildModule && plan.NeedsToolchain(bm) { + // The ask names the toolchain the fingerprint was computed with; a node building with another refuses it. + language := buildrun.LanguageOf[bm] + tc, none := delivery.AgreedToolchain(b.Toolchains.Entries(language)) + if none != "" { + return failed("the %s toolchain is not agreed: %s", language, none) } if want := m.Toolchains[language]; want.Version != tc.Version || want.Recipe != tc.Recipe { - giveUp(ns...) - giveUpLanes() return delivery.BuilderBuild{} // the agreed toolchain moved since try looked: nothing is written } - used[language] = delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} - nodes = ns - } else { - // An archive is packed with no toolchain (the fingerprint names none): any node of the build module that qualifies. - ns, err := buildrun.ChooseAnyNodes(language, entries, b.Places, qualifies, 1) - if err != nil { - giveUpLanes() - return failed("%s", err) - } - nodes = ns + u := delivery.ToolchainUsed{Version: tc.Version, Recipe: tc.Recipe} + used[language], toolchain = u, &u } - // One node builds each fingerprint (ADR 0478 clause 4, from R4e). - steps = append(steps, fmt.Sprintf("%s on %s", bm, nodes[0])) var tasks []Task for _, t := range plan.Tasks { if t.Module == bm { tasks = append(tasks, t) } } - for _, n := range nodes { - lanes = append(lanes, &lane{module: bm, node: n, tasks: tasks, digests: map[string]string{}, references: map[string]string{}}) - } + // One ask builds each fingerprint's part of a build module, on whichever node pulls it (ADR 0525 §3). + lanes = append(lanes, &lane{module: bm, tasks: tasks, toolchain: toolchain, digests: map[string]string{}, references: map[string]string{}}) } var wg sync.WaitGroup for _, l := range lanes { wg.Add(1) - go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module) }() + go func() { defer wg.Done(); b.runLane(ctx, l, repository, m, module, asked) }() } wg.Wait() var errs []string @@ -548,6 +510,10 @@ func (b *Builder) build(ctx context.Context, module string, m delivery.Module, p } artifacts = append(artifacts, built) } + var steps []string + for _, l := range lanes { + steps = append(steps, fmt.Sprintf("%s on %s", l.module, l.node)) + } for _, bm := range kept { steps = append(steps, bm+" kept") for lang, u := range prior.Toolchains { @@ -617,73 +583,56 @@ func keyError(errs []string) string { return strings.Join(errs, "; ") } -func (b *Builder) online(node string) bool { return b.Presence == nil || b.Presence.Online(node) } +// imageSeat says an image build is asked of the image build seat, which takes it when a node pulls it: a key failed for +// want of an image builder before R4g is taken again (ADR 0525 §1). +func imageSeat() bool { return true } -// qualifies is whether the node can build with the build module now: it is online and the build module on it is healthy. -func (b *Builder) qualifies(node, bm string) bool { - return b.Places.CapacityOf(node) > 0 && b.online(node) && (b.Health == nil || b.Health.Healthy(node, bm)) +// askLanguage is the build seat language of each build module (delivery.AskSubject). +var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode, ImageBuildModule: delivery.AskImage} + +// askNotes writes a claimed key's asks into it, building, before each is put (ADR 0525 §1; issue 1887 A4b), and holds +// the asks a key taken over at start already names, so they are waited for and never put again. +type askNotes struct { + mu sync.Mutex + b *Builder + cl *Claim + resumed map[string]string } -// imageEntries are the nodes an image may be built on, as toolchain entries for the chooser: every node the builder has a -// capacity for or a build module toolchain from. Whether the image build module runs there is qualifies', from the have. -func (b *Builder) imageEntries() []delivery.NodeToolchain { - seen := map[string]bool{} - var entries []delivery.NodeToolchain - add := func(n string) { - if !seen[n] { - seen[n] = true - entries = append(entries, delivery.NodeToolchain{Node: n}) - } +func (b *Builder) notes(cl *Claim) *askNotes { + n := &askNotes{b: b, cl: cl} + if cl.Build.State == delivery.BuildBuilding && len(cl.Build.Asks) > 0 { + n.resumed = maps.Clone(cl.Build.Asks) } - for n := range b.Places.Capacity { - add(n) + return n +} + +// resume is the id of the ask a taken-over key names for language, or "". +func (n *askNotes) resume(language string) string { + n.mu.Lock() + defer n.mu.Unlock() + return n.resumed[language] +} + +// note writes the ask's id into the key, building, by compare-and-set at the claim's revision. +func (n *askNotes) note(ctx context.Context, language, id, step string) error { + n.mu.Lock() + defer n.mu.Unlock() + v := n.cl.Build + v.State, v.By, v.Step, v.Error, v.Artifacts = delivery.BuildBuilding, n.b.Claimer.By, step, "", nil + v.Asks = maps.Clone(v.Asks) + if v.Asks == nil { + v.Asks = map[string]string{} } - for _, language := range []string{"go", "typescript"} { - for _, e := range b.Toolchains.Entries(language) { - add(e.Node) - } - } - sort.Slice(entries, func(i, j int) bool { return entries[i].Node < entries[j].Node }) - return entries + v.Asks[language] = id + return n.b.Claimer.Finish(ctx, n.cl, v) } -// imageBuilderRuns is whether a node can build an image now. -func (b *Builder) imageBuilderRuns() bool { - for _, e := range b.imageEntries() { - if b.qualifies(e.Node, ImageBuildModule) { - return true - } - } - return false -} - -// step runs one step of a build module; the lane that runs it holds its node's place. -func (b *Builder) step(ctx context.Context, bm, node, tool string, args map[string]any) (json.RawMessage, error) { - return b.Runner.Run(ctx, bm, node, toolPrefix[bm]+tool, args) -} - -// fetchResult and buildResult are the results of a build module's fetch and build steps as the builder reads them. -type fetchResult struct { - Workspace string `json:"workspace"` - Merge string `json:"merge"` - Conflicts []string `json:"conflicts"` -} - -type buildResult struct { - Digest string `json:"digest"` - Reference string `json:"reference"` -} - -// runLane takes a place on the node, fetches the commit there and builds each of the lane's artifacts, one step at a time. -// An upstream artifact is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches -// nothing. -func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string) { - release, err := b.Places.Acquire(ctx, l.node) - if err != nil { - l.err = err - return - } - defer release() +// runLane puts the lane's part of the build as one ask on its build seat's stream (the delivery stream for a trunk key, the +// check stream for a candidate's) and reads what the node that pulled it built (ADR 0525 §1, §3, §6). An upstream artifact +// is copied by its pinned reference with no fetch (ADR 0498 clause 4); a lane of copies alone fetches nothing. A lane of a +// key taken over at start waits for the ask the key names and puts none. +func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m delivery.Module, module string, asked *askNotes) { req := buildstep.FetchRequest{Repository: repository, Commit: m.Commit} if key, err := delivery.BuilderBuildKey(module, m.FP2); err == nil { b.mu.Lock() @@ -702,57 +651,85 @@ func (b *Builder) runLane(ctx context.Context, l *lane, repository string, m del } sort.Strings(req.Paths) } - fetchArgs := req.Args() candidate := req.Base != "" - var fetched fetchResult + language := askLanguage[l.module] + kind := delivery.AskDelivery + if candidate { + kind = delivery.AskCheck + } + job := &delivery.BuildJob{Module: module, Path: m.Path, FP: m.FP2, Toolchain: l.toolchain, + // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, + // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). + Publish: !candidate} + for _, t := range l.tasks { + a := delivery.AskArtifact{Artifact: t.Artifact} + if t.Kind == catalogue.ArtifactUpstream { + a.From = t.From // the copy runs no code of a change: the module's name, the artifact and its pin + } + job.Artifacts = append(job.Artifacts, a) + } if needsFetch(l.tasks) { - raw, err := b.step(ctx, l.module, l.node, "fetch", fetchArgs) - if err != nil { + job.Fetch = req.Args() + } + ask := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: language, Asker: delivery.BuilderModule, At: time.Now(), Build: job} + if id := asked.resume(language); id != "" { + ask.ID = id // put before this builder started again: the stream holds it, or a node already ran it + if b.Held != nil { + held, err := b.Held.Held(ctx, ask) + if err != nil { + // Said, and the key is not failed: the ask may well be on its stream and its build finish (issue 1910). + b.logf("whether the ask %s of %s the key names is on its stream cannot be read, so it is waited for as put: %v", ask.ID, l.module, err) + held = true + } + if !held { + // The builder that named it stopped before it put it (issue 1910): put it now, under the same id. + b.logf("the ask %s of %s that the key names was never put: it is put now", ask.ID, l.module) + if err := b.Asks.Put(ctx, ask); err != nil { + l.err = err + return + } + } + } + } else { + if err := asked.note(ctx, language, ask.ID, "asked "+l.module); err != nil { + l.err = fmt.Errorf("the ask of %s could not be written to the key: %w", l.module, err) + return + } + if err := b.Asks.Put(ctx, ask); err != nil { l.err = err return } - if err := json.Unmarshal(raw, &fetched); err != nil || fetched.Workspace == "" { - l.err = fmt.Errorf("the fetch on %s answered no workspace: %s", l.node, raw) - return - } - // A candidate is built from its head merged onto the trunk. A head that does not merge ends the fetch passed with the tree at - // the trunk, and building that would store the trunk's build under the candidate's fingerprint. - if candidate { - if len(fetched.Conflicts) > 0 { - l.err = fmt.Errorf("the head does not merge into the trunk on %s: files in conflict: %s", l.node, strings.Join(fetched.Conflicts, ", ")) - return - } - if fetched.Merge == "" { - l.err = fmt.Errorf("the fetch with a base on %s answered no merge: %s", l.node, raw) - return - } - } + } + step, err := b.Asks.Wait(ctx, ask) + if err != nil { + l.err = fmt.Errorf("%s: %w", l.module, err) + return + } + l.node = step.Node + if step.State != delivery.StepPassed { + l.err = fmt.Errorf("%s on %s: the ask %s %s: %s", l.module, step.Node, ask.ID, step.State, step.Reason) + return + } + var answer delivery.BuildAnswer + if err := json.Unmarshal(step.Result, &answer); err != nil { + l.err = fmt.Errorf("the ask of %s on %s answered what is not a build: %s", l.module, step.Node, step.Result) + return + } + built := map[string]delivery.AskBuilt{} + for _, a := range answer.Artifacts { + built[a.Artifact] = a } for _, t := range l.tasks { - tool, args := "build", map[string]any{"workspace": fetched.Workspace, "module": m.Path, "artifact": t.Artifact} - if t.Kind == catalogue.ArtifactUpstream { - // The copy runs no code of a change: the module's name (the reference's path), the artifact and its pin. - tool, args = "copy", map[string]any{"module": module, "artifact": t.Artifact, "from": t.From} - } else if !candidate { - // Only a trunk key's build pushes an image or publishes a package; a candidate's never does (ADR 0487 clause 12, - // ADR 0498 clauses 5 and 6). A bundle or an archive is pushed either way (clause 3). - args["publish"] = true - } - raw, err := b.step(ctx, l.module, l.node, tool, args) - if err != nil { - l.err = err + a, ok := built[t.Artifact] + if !ok || a.Digest == "" { + l.err = fmt.Errorf("the build of %s of %s on %s answered no digest: %s", t.Artifact, module, step.Node, step.Result) return } - var built buildResult - if err := json.Unmarshal(raw, &built); err != nil || built.Digest == "" { - l.err = fmt.Errorf("the %s of %s of %s on %s answered no digest: %s", tool, t.Artifact, module, l.node, raw) - return + l.digests[t.Artifact] = a.Digest + if a.Reference != "" && (stored(t.Kind) || !candidate) { + l.references[t.Artifact] = a.Reference } - l.digests[t.Artifact] = built.Digest - if built.Reference != "" && (stored(t.Kind) || !candidate) { - l.references[t.Artifact] = built.Reference - } - if built.Reference != "" && b.Answered != nil { + if a.Reference != "" && b.Answered != nil { b.Answered(l.module) } } diff --git a/modules/mesh-delivery-builder/internal/build/run_test.go b/modules/mesh-delivery-builder/internal/build/run_test.go index 4a8524c..f3e82b8 100644 --- a/modules/mesh-delivery-builder/internal/build/run_test.go +++ b/modules/mesh-delivery-builder/internal/build/run_test.go @@ -3,6 +3,7 @@ package build import ( "context" "encoding/json" + "errors" "fmt" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" "strings" @@ -10,6 +11,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -124,22 +126,56 @@ type rig struct { st *memStore bm *buildModules tools *buildrun.ToolchainSet - pres *buildrun.PresenceSet mf *fakeManifests key string + // asks are the build seats' streams; puller is the node that pulls every ask ("ace" when empty). + asks *buildrun.FakeAsks + puller string +} + +// toolPrefix is the start of each build module's tool names, as the fake build modules answer them. +var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_", ImageBuildModule: "build_image_"} + +// answer runs an ask as the build module that pulls it does (buildstep.RunAsk), with the fake build modules' tools on the +// rig's pulling node, and answers the step that node writes. +func (r *rig) answer(a delivery.StepAsk) delivery.Step { + node := r.puller + if node == "" { + node = "ace" + } + module, _ := delivery.BuildModuleOf(a.Language) + runner := buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second} + tool := func(name string) buildstep.StepFunc { + return func(ctx context.Context, args map[string]any) (any, error) { + raw, err := runner.Run(ctx, module, node, toolPrefix[module]+name, args) + if err != nil { + return nil, err + } + return raw, nil + } + } + steps := buildstep.AskSteps{Fetch: tool("fetch"), Build: tool("build"), Copy: tool("copy")} + out, err := buildstep.RunAsk(context.Background(), steps, a) + step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed} + if err != nil { + step.State, step.Reason = delivery.StepFailed, err.Error() + return step + } + step.Result, _ = json.Marshal(out) + return step } func newRig(t *testing.T, artifacts ...catalogue.Artifact) *rig { t.Helper() - r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, pres: &buildrun.PresenceSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} + r := &rig{st: newMem(), bm: newBuildModules(), tools: &buildrun.ToolchainSet{}, mf: &fakeManifests{byDigest: map[string]catalogue.Manifest{mdigest('a'): {Module: "m", Build: &catalogue.Build{Artifacts: artifacts}}}}} for _, n := range []string{"ace", "novox"} { r.tools.Put(buildrun.GoBuildModule, n+".go", tcValue(t, "go", "go1.27.1"), false) r.tools.Put(buildrun.NodeBuildModule, n+".typescript", tcValue(t, "typescript", "tsc5"), false) - r.pres.Put(n, presenceValue(t, true), false) } + r.asks = &buildrun.FakeAsks{Answer: r.answer} r.b = &Builder{ - Claimer: Claimer{Store: r.st, By: "novox.builder"}, Runner: buildrun.Runner{Caller: r.bm, Steps: r.bm.steps, Bound: time.Second}, - Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf, + Claimer: Claimer{Store: r.st, By: "novox.builder"}, Asks: buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 2 * time.Second}, + Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf, } var err error if r.key, err = delivery.BuilderBuildKey("m", fp1); err != nil { @@ -220,15 +256,53 @@ func TestABundleAndAnArchiveAreBuiltOnOneNodeAndDone(t *testing.T) { // 1607/AC7 } } -func TestOneQualifyingNodeBuildsOnce(t *testing.T) { // 1607/AC4, ADR 0478 clause 4 +// 1887/AC1, AC2: a trunk fingerprint with no key is one ask on the Go seat's delivery stream, named in the building key +// before it is put, and the builder calls no node; the step a node wrote for it makes the key done naming its reference. +func TestATrunkFingerprintIsOneAskOnTheDeliveryStream(t *testing.T) { r := newRig(t, goBundle) - r.tools.Put(buildrun.GoBuildModule, "ace.go", nil, true) - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on novox") { - t.Errorf("build = %+v, want done with its step saying novox", v) + r.asks.Answer = nil // no node has pulled it yet + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + var asks []delivery.StepAsk + for deadline := time.Now().Add(2 * time.Second); len(asks) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no ask was put") + } + asks = r.asks.Asks() } - if n := len(r.bm.callsOf("build_go_build")); n != 1 { - t.Errorf("%d builds, want one", n) + a := asks[0] + if len(asks) != 1 || a.Kind != delivery.AskDelivery || a.Language != delivery.AskGo || a.Build == nil || a.Build.FP != fp1 { + t.Fatalf("asks = %+v, want one Go delivery ask of the fingerprint", asks) + } + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding || v.Asks[delivery.AskGo] != a.ID { + t.Fatalf("the key = %+v, want building naming the ask %s", v, a.ID) + } + if n := len(r.bm.callsOf("build_go_build")) + len(r.bm.callsOf("build_go_fetch")); n != 0 { + t.Fatalf("the builder called a node %d times", n) + } + r.asks.End(r.answer(a)) // a node pulled it and ran it + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || !strings.Contains(v.Step, "on ace") || v.Artifacts[0].Reference == "" || len(v.Asks) != 0 { + t.Fatalf("the key = %+v, want done on ace naming the reference", v) + } +} + +// 1887/AC3: a builder that starts again while its ask waits puts no second ask: it waits for the step of the ask the +// building key names. +func TestARestartedBuilderWaitsForTheAskTheKeyNames(t *testing.T) { + r := newRig(t, goBundle) + const id = "mvl0frv18d1957" + data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) + r.st.vals[r.key] = data + r.asks.Answer = nil + r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, + Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) + v := r.consider(t, "modules/m") + if n := len(r.asks.Asks()); n != 0 { + t.Fatalf("%d asks put again", n) + } + if v.State != delivery.BuildDone || !strings.Contains(v.Step, "on shanks") { + t.Fatalf("the key = %+v, want done from the ask's step on shanks", v) } } @@ -247,20 +321,9 @@ func TestAFailingStepFailsTheBuildWithItsReasonAndNodeAndIsTriedAgainAsANewRevis } } -// imageHave marks mesh-build-image running and healthy on these nodes. -func imageHave(t *testing.T, nodes ...string) *buildrun.HaveSet { - t.Helper() - have := &buildrun.HaveSet{} - for _, n := range nodes { - have.Put(n+".mesh-build-go", haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - have.Put(n+"."+ImageBuildModule, haveValue(t, delivery.PhaseRunning, delivery.HealthOK), false) - } - return have -} - func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { // 1705a, ADR 0487 clause 5 r := newRig(t, imageArt) - r.b.Health = imageHave(t, "ace", "novox") // two nodes qualify; an image still builds on one + r.puller = "ace" // two nodes qualify; an image still builds on one v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 1 || v.Artifacts[0].Kind != "image" || v.Artifacts[0].Digest != digest { t.Fatalf("build = %+v, want done with one image artifact and its digest", v) @@ -278,19 +341,9 @@ func TestAnImageGoesToOneNodeAndGetsOneDoneKeyWithItsReference(t *testing.T) { / } } -func TestAnImageIsBuiltOnlyWhereMeshBuildImageIsHealthy(t *testing.T) { // 1705a - r := newRig(t, imageArt) - r.b.Health = imageHave(t, "novox") - r.consider(t, "modules/m") - c := r.bm.callsOf("build_image_build") - if len(c) != 1 || c[0].node != "novox" { - t.Errorf("calls = %+v, want one on novox", c) - } -} - func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a r := newRig(t, goBundle, imageArt) - r.b.Health = imageHave(t, "ace", "novox") + r.puller = "ace" v := r.consider(t, "modules/m") if v.State != delivery.BuildDone || len(v.Artifacts) != 2 { t.Fatalf("build = %+v", v) @@ -303,33 +356,9 @@ func TestABundleAndAnImageOfOneModuleBuildOnOneNodeEach(t *testing.T) { // 1705a } } -func TestWithNoImageNodeTheKeyFailsNamingTheProvisionAndIsNotRetakenUntilANodeReportsTheModule(t *testing.T) { // 1705b, lead's decision - r := newRig(t, imageArt) - r.b.Health = imageHave(t) // mesh-build-image runs nowhere - v := r.consider(t, "modules/m") - if v.State != delivery.BuildFailed || !strings.HasPrefix(v.Error, NoImageBuilderPrefix) || !strings.Contains(v.Error, "rootless-image-build") || !strings.Contains(v.Error, "buildah") || !strings.Contains(v.Error, ImageBuildModule) { - t.Fatalf("build = %+v, want failed naming rootless-image-build, buildah and mesh-build-image", v) - } - if len(r.bm.calls) != 0 { - t.Errorf("%d calls, want none", len(r.bm.calls)) - } - writes := r.st.writes - r.consider(t, "modules/m") - r.consider(t, "modules/m") - if r.st.writes != writes { - t.Errorf("the failed key was written again (%d writes, was %d)", r.st.writes, writes) - } - // A node reports the module: the key is taken again. - r.b.Health = imageHave(t, "ace") - v = r.consider(t, "modules/m") - if v.State != delivery.BuildDone { - t.Errorf("build = %+v, want done once a node reports mesh-build-image", v) - } -} - func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) { // 1705b, lead's decision r := newRig(t, imageArt) - r.b.Health = imageHave(t, "ace") + r.puller = "ace" trunkModule(t, r) r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() @@ -348,7 +377,7 @@ func TestACandidatesImageCallCarriesNoPublishAndItsKeyNoReference(t *testing.T) func TestATrunkImageCallCarriesPublish(t *testing.T) { // 1705b r := newRig(t, imageArt) - r.b.Health = imageHave(t, "ace") + r.puller = "ace" r.consider(t, "modules/m") calls := r.bm.callsOf("build_image_build") if len(calls) != 1 || calls[0].body.(map[string]any)["publish"] != true { @@ -439,7 +468,7 @@ func TestAKeyTheBuilderLeftBuildingIsBuiltAtStartAndDoneKeysAreNotBuiltTwice(t * // whose compare-and-set comes first, and the other's write is refused, never overwriting it. func TestTwoInstancesStartingTogetherEndWithOneDoneKey(t *testing.T) { // 1607/AC7, review round 4 r := newRig(t, goBundle) - other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Runner: r.b.Runner, Toolchains: r.tools, Presence: r.pres, Manifests: r.mf, Logf: t.Logf} + other := &Builder{Claimer: Claimer{Store: r.st, By: "ace.builder"}, Asks: r.b.Asks, Toolchains: r.tools, Manifests: r.mf, Logf: t.Logf} value := moduleValue(t, "modules/m") var wg sync.WaitGroup for _, b := range []*Builder{r.b, other} { @@ -522,36 +551,9 @@ func TestARootModulesPathIsDotAndIsGivenToTheBuildTools(t *testing.T) { // 1615/ } } -func TestANodeThatIsNotOnlineIsNotChosenAndComingBackWakesTheBuild(t *testing.T) { // 1607 review 3 - r := newRig(t, goBundle) - r.tools.Put(buildrun.GoBuildModule, "g14.go", tcValue(t, "go", "go1.27.1"), false) // a laptop that reports its toolchain and sleeps - v := r.consider(t, "modules/m") - if v.State != delivery.BuildDone { - t.Fatalf("build = %+v", v) - } - for _, c := range r.bm.calls { - if c.node == "g14" { - t.Errorf("a call went to g14, which is not online: %+v", c) - } - } - // With no node online the key fails naming why; a node coming online builds it as a new revision. - r2 := newRig(t, goBundle) - r2.pres.Put("ace", nil, true) - r2.pres.Put("novox", nil, true) - if v := r2.consider(t, "modules/m"); v.State != delivery.BuildFailed || !strings.Contains(v.Error, "online") { - t.Fatalf("build = %+v, want failed because no node is online", v) - } - r2.pres.Put("ace", presenceValue(t, true), false) - r2.b.Reconsider(context.Background()) - r2.b.Wait() - if v := valueOf(t, r2.st, r2.key); v.State != delivery.BuildDone { - t.Errorf("build = %+v, want done once a node is online", v) - } -} - func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testing.T) { // review r1, mesh-delivery 1; ADR 0487 clause 5 r := newRig(t, imageArt) - r.b.Health = imageHave(t, "ace") + r.puller = "ace" trunkModule(t, r) // the trunk carries another fingerprint; fp1 is the candidate's r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() @@ -590,7 +592,7 @@ func TestACandidatesImageIsPublishedOnceWhenTheTrunkNamesItsFingerprint(t *testi func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // review r2, mesh-delivery 1; ADR 0487 clause 5 r := newRig(t, imageArt) - r.b.Health = imageHave(t, "ace") + r.puller = "ace" trunkModule(t, r) r.b.Candidate(context.Background(), "feat%2Fx", candidateValue(t, "feat/x", 7, headX)) r.b.Wait() @@ -612,3 +614,86 @@ func TestAFailedPublishOfACandidatesImageIsWrittenToTheKey(t *testing.T) { // re t.Errorf("build = %+v, want done with a reference", v) } } + +// Review round 2 item 1: a build ask that waits in its stream longer than the step bound is not failed and not asked +// again; the key is done when a node takes it. +func TestABuildAskWaitingInItsStreamIsNotBounded(t *testing.T) { + r := newRig(t, goBundle) + r.asks.Answer = nil + r.b.Asks = buildrun.Asker{Publisher: r.asks, Waiter: r.asks, Bound: 20 * time.Millisecond} + r.b.Module(context.Background(), "m", moduleValue(t, "modules/m")) + for deadline := time.Now().Add(2 * time.Second); len(r.asks.Asks()) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no ask was put") + } + } + time.Sleep(100 * time.Millisecond) + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildBuilding { + t.Fatalf("the key = %+v while its ask waits, want building", v) + } + r.asks.End(r.answer(r.asks.Asks()[0])) + r.b.Wait() + if v := valueOf(t, r.st, r.key); v.State != delivery.BuildDone || len(r.asks.Asks()) != 1 { + t.Fatalf("the key = %+v after %d asks, want done from the one ask", v, len(r.asks.Asks())) + } +} + +// heldAsks says an ask is held when its id is among held. +type heldAsks map[string]bool + +func (h heldAsks) Held(_ context.Context, a delivery.StepAsk) (bool, error) { return h[a.ID], nil } + +// 1910: a builder that stopped between naming the ask in its key and putting it left a key naming an ask neither the +// stream holds nor any node ran: started again, the builder puts that ask, under the same id, once, and the key is +// built from its step; an ask the stream holds is not put again. +func TestAnAskTheKeyNamesAndNobodyPutIsPutAtStart(t *testing.T) { + for _, held := range []bool{false, true} { + r := newRig(t, goBundle) + const id = "mvl0frv18d1958" + data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) + r.st.vals[r.key] = data + r.b.Held = heldAsks{id: held} + if held { + r.asks.Answer = nil + r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, + Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) + } + v := r.consider(t, "modules/m") + asks := r.asks.Asks() + switch { + case held && len(asks) != 0: + t.Errorf("an ask the stream holds was put again: %+v", asks) + case !held && (len(asks) != 1 || asks[0].ID != id): + t.Errorf("the ask nobody put: asks %+v, want one under %s", asks, id) + } + if v.State != delivery.BuildDone { + t.Errorf("held %v: the key = %+v, want done", held, v) + } + } +} + +// unreadableHeld cannot tell whether an ask is held. +type unreadableHeld struct{} + +func (unreadableHeld) Held(context.Context, delivery.StepAsk) (bool, error) { + return false, errors.New("permissions violation for publish to $JS.API.STREAM.INFO.SEAT_NODE_BUILD_GO") +} + +// 1910: a take-over that cannot read whether the ask its key names is on its stream says so and waits for the ask's step: +// the key is not failed, and the build finishes when a node ran the ask. +func TestATakeOverThatCannotReadTheStreamWaitsForTheAsk(t *testing.T) { + r := newRig(t, goBundle) + const id = "mvl0frv18d1959" + data, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "novox.builder", Manifest: manifestD, + Step: "asked mesh-build-go", Asks: map[string]string{delivery.AskGo: id}}) + r.st.vals[r.key] = data + r.b.Held = unreadableHeld{} + r.asks.Answer = nil + r.asks.End(delivery.Step{ID: id, Node: "shanks", State: delivery.StepPassed, + Result: []byte(`{"artifacts":[{"artifact":"tool","digest":"` + digest + `","reference":"artifact-store://m/tool/blobs/` + digest + `"}]}`)}) + v := r.consider(t, "modules/m") + if v.State != delivery.BuildDone || len(r.asks.Asks()) != 0 { + t.Fatalf("the key = %+v after %d asks, want done from the ask's step, none put", v, len(r.asks.Asks())) + } +} diff --git a/modules/mesh-delivery-builder/manifest_test.go b/modules/mesh-delivery-builder/manifest_test.go index 6f13144..930b0d8 100644 --- a/modules/mesh-delivery-builder/manifest_test.go +++ b/modules/mesh-delivery-builder/manifest_test.go @@ -38,6 +38,7 @@ type manifest struct { Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` + Uses []string `json:"uses"` Invokes []string `json:"invokes"` Config map[string]struct { Type string `json:"type"` @@ -86,12 +87,17 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if want := []state{{Name: "builds", History: 5}, {Name: "instances", History: 1}}; !reflect.DeepEqual(m.State, want) { // delivery.BuilderBuildsHistory t.Errorf("state = %+v, want %+v", m.State, want) } + // 1887/A4c: the builder chooses no node, so it reads neither presence nor have, invokes no build module's tool and + // holds no capacity; it puts asks on the three build seats and reads their steps (ADR 0525 §1, §4, §5). if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-image.steps", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { + "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"mesh-build-go.build_go_build", "mesh-build-go.build_go_fetch", "mesh-build-image.build_image_build", "mesh-build-image.build_image_copy", "mesh-build-image.build_image_fetch", "mesh-build-node.build_node_build", "mesh-build-node.build_node_fetch"}; !reflect.DeepEqual(sorted(m.Invokes), want) { - t.Errorf("invokes = %v, want only the build modules' fetch and build (go, image, node) and the image module's copy (ADR 0498 clause 4)", m.Invokes) + if want := []string{"node-build-go", "node-build-image", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { + t.Errorf("uses = %v, want the three build seats", m.Uses) + } + if len(m.Invokes) != 0 || len(m.Config) != 0 { + t.Errorf("invokes = %v, config = %v, want none: no tool call from the builder and no capacity", m.Invokes, m.Config) } // The builder is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { @@ -103,7 +109,7 @@ func TestManifestDeclarations(t *testing.T) { // 1607/AC1 if m.SecretsOwner != "mesh-delivery-builder" { t.Errorf("secrets-owner = %q", m.SecretsOwner) } - for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "tools", "seat", "seats", "bus"} { + for _, k := range []string{"streams", "logs", "emits", "claims", "provides", "requires", "defines-seats", "tools", "seat", "seats", "bus", "invokes", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the builder has none", k) } @@ -129,35 +135,19 @@ func TestManifestProcess(t *testing.T) { // 1607/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - // The build capacity of each node is the module's setting `capacity` (issue 1661, ADR 0478 clause 3), laid into the - // settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity. - if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_BUILDER_SETTINGS"] != "${dir:state}/settings.json" { - t.Errorf("env = %v", p["env"]) + if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" { + t.Errorf("env = %v, want the broker file alone", p["env"]) } - if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) { - t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"]) - } - c, ok := m.Config["capacity"] - if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "shanks=3,ace=2,g14=0,novox=0" || c.Description == "" { - t.Errorf("config = %+v, want one string setting, capacity, shanks=3,ace=2,g14=0,novox=0 to start", m.Config) - } - var settings map[string]any for _, r := range m.Resources { if r["id"] == "settings" { - settings = r + t.Errorf("a settings resource %v: the builder holds no setting since the capacity is the build modules' (ADR 0525 §5)", r) } } - if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" { - t.Errorf("settings resource = %v", settings) - } - if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": "shanks=3,ace=2,g14=0,novox=0"`) { - t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err) - } if _, bus := p["bus"]; bus { t.Errorf("the process declares a bus: it runs on its own broker credential") } - if len(m.Build.Artifacts) != 2 || m.Build.Artifacts[1].Name != "files" || m.Build.Artifacts[1].Kind != "archive" || m.Build.Artifacts[1].From != "files" { - t.Fatalf("artifacts = %+v, want the program and the files", m.Build.Artifacts) + if len(m.Build.Artifacts) != 1 { + t.Fatalf("artifacts = %+v, want the program alone", m.Build.Artifacts) } a := m.Build.Artifacts[0] if st, err := os.Stat(a.From); err != nil || !st.IsDir() || a.Binary != "mesh-delivery-builder" || !strings.HasSuffix(a.From, a.Binary) || a.Kind != "bundle" || a.Language != "go" { diff --git a/modules/mesh-delivery-builder/module.json b/modules/mesh-delivery-builder/module.json index f18e708..798cb79 100644 --- a/modules/mesh-delivery-builder/module.json +++ b/modules/mesh-delivery-builder/module.json @@ -18,27 +18,14 @@ "mesh-build-node.toolchains", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", - "mesh-delivery-indexer.modules", - "mesh-delivery-watchman.presence", - "node-engine.have" + "mesh-delivery-indexer.modules" ], - "invokes": [ - "mesh-build-go.build_go_build", - "mesh-build-go.build_go_fetch", - "mesh-build-image.build_image_build", - "mesh-build-image.build_image_copy", - "mesh-build-image.build_image_fetch", - "mesh-build-node.build_node_build", - "mesh-build-node.build_node_fetch" + "uses": [ + "node-build-go", + "node-build-image", + "node-build-node" ], - "config": { - "capacity": { - "type": "string", - "description": "How many build steps the builder runs at once on each node, as node=steps pairs separated by commas, such as shanks=3,ace=2 (ADR 0478 clause 3); a node set to 0 never builds, a node not named runs one at a time, and empty is one everywhere. The builder builds on shanks and ace only (the operator, 2026-10-21).", - "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", - "default": "shanks=3,ace=2,g14=0,novox=0" - } - }, + "own-secrets": { "broker": "${dir:state}/broker" }, @@ -59,17 +46,6 @@ "place": ".", "unknown": "program" }, - { - "id": "settings", - "trusted": false, - "type": "file", - "path": "${dir:state}/settings.json", - "mode": "0600", - "owner": "mesh-delivery-builder", - "takes-config": ["capacity"], - "merge": "json", - "template": "files/settings.json" - }, { "id": "builder", "type": "process", @@ -79,12 +55,8 @@ "./mesh-delivery-builder" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker", - "MESH_BUILDER_SETTINGS": "${dir:state}/settings.json" + "MESH_BROKER_FILE": "${dir:state}/broker" }, - "restart-on": [ - "settings" - ], "user": "mesh-delivery-builder", "health": { "kind": "unit" @@ -100,11 +72,6 @@ "system": "arch", "from": "cmd/mesh-delivery-builder", "binary": "mesh-delivery-builder" - }, - { - "name": "files", - "kind": "archive", - "from": "files" } ] } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go index acac211..c43f8cc 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker-tools/main.go @@ -50,7 +50,7 @@ func TryTools(tr Trying) []stdio.Tool { Description: "Try a pushed head before its pull request is opened: the checker judges it as it judges a pull request's head, merged onto its trunk's head, " + "its unit and integration levels through the build modules, and owes the forge nothing. Answers the try's id at once; " + check.TryReadTool + " reads " + "where it stands. A try of the same head that waits or runs is the answer to a second ask; at most 4 wait or run at once, mesh-wide. A repository " + - "the checker checks no level of (hq, one whose trunk the gate does not name) is refused.", + "the checker checks no level of (one whose trunk the gate does not name) is refused.", Input: map[string]any{"type": "object", "required": []string{"repository", "head"}, "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "/"}, "head": map[string]any{"type": "string", "description": "the full commit to try, 40 or 64 hex digits"}, @@ -104,8 +104,7 @@ func Tools(r check.Reader) []stdio.Tool { return map[string]any{"verdicts": vs}, err }}, {Name: check.PullReadTool, - Description: "The verdict that judges one pull request and the full report of its status mesh/check; for an hq pull request, that hq is judged by " + - "merge-check.sh until issue 1296. Read-only.", + Description: "The verdict that judges one pull request and the full report of its status mesh/check. Read-only.", Input: map[string]any{"type": "object", "required": []string{"repository", "number"}, "properties": map[string]any{ "repository": map[string]any{"type": "string", "description": "/"}, "number": map[string]any{"type": "integer", "description": "the pull request's number"}}}, @@ -143,32 +142,16 @@ func Tools(r check.Reader) []stdio.Tool { return map[string]any{"previews": ps, "unreadable": bad}, err }}, {Name: check.StatusReadTool, - Description: "What holds the checker back and what runs: the capacity per node (the setting, the default one step per node), whether the controller's " + - "delivery plans in their build phase hold a key (no new check starts while one does), the steps in flight on the build modules by node, and the " + - "candidates without a verdict for their heads (a group of hq's pull requests alone is left out: merge-check.sh judges it), and the " + - "candidates whose check waits by kind (waiting_by_kind: new, nodes, and trunk, which start after the other two). A key a build " + - "module left running (started past the step bound, or its build module not running on its node) is listed apart under left_running " + - "with why, and not counted. The places the checker holds are in its process, not in a bucket, and the answer says so. Read-only.", + Description: "What runs: the steps in flight on the build modules by node, the candidates without a verdict for their heads, " + + "and the candidates whose check is owed by kind (waiting_by_kind: new, nodes, and trunk, a re-judge pulled after the other two). A key a " + + "build module left running (started past the step bound, or its build module not running on its node) is listed apart under " + + "left_running with why, and not counted. Nothing holds the checker and it has no capacity of its own: a check is an ask on the " + + "build seats' streams, whose backlog the delivery view shows (novox/hq ADR 0525). Read-only.", Input: map[string]any{"type": "object", "properties": map[string]any{}}, Run: func(map[string]any) (any, error) { return r.Status() }}, } } -func capacity() (string, error) { - path := os.Getenv("MESH_CHECKER_SETTINGS") - if path == "" { - return os.Getenv("MESH_CHECKER_CAPACITY"), nil - } - raw, err := os.ReadFile(path) - if err != nil { - return "", err - } - var s struct { - Capacity string `json:"capacity"` - } - return s.Capacity, json.Unmarshal(raw, &s) -} - // stepBound is the checker's step bound as the checker reads it (MESH_CHECKER_STEP_BOUND), its default when unset or not a // duration. func stepBound() time.Duration { @@ -180,9 +163,9 @@ func stepBound() time.Duration { func main() { r := check.Reader{VerdictState: read(stdio.State("verdicts")), KeptState: read(stdio.State("kept")), CandidateState: read(stdio.State("mesh-delivery-indexer.candidates")), - PreviewState: read(stdio.State("previews")), PlansState: read(stdio.State("mesh-controller.plans-building")), - StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, - Capacity: capacity, HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()} + PreviewState: read(stdio.State("previews")), + StepStates: map[string]check.StateRead{"mesh-build-go": read(stdio.State("mesh-build-go.steps")), "mesh-build-node": read(stdio.State("mesh-build-node.steps"))}, + HaveState: read(stdio.State("node-engine.have")), StepBound: stepBound()} tries := stdio.State(check.TriesState) source := stdio.State("mesh-delivery-gate.source") trunks := func() (map[string]string, error) { diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go index 6415687..448979d 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/bus.go @@ -105,43 +105,6 @@ func follow(ctx context.Context, w jetstream.KeyWatcher, each func(jetstream.Key } } -// stepsWaiter waits for a build module's step by watching its key in the module's steps bucket (ADR 0469 clause 2): the -// watch hands over the value the key holds now, so a step that ended before the watch began is read at once, and every -// change after. No timer: the caller bounds the wait with its context. -type stepsWaiter struct{ js bucketOpener } - -func (w stepsWaiter) Wait(ctx context.Context, module, key string) (delivery.Step, error) { - kv, err := w.js.KeyValue(ctx, delivery.StepsBucket(module)) - if err != nil { - return delivery.Step{}, fmt.Errorf("the bucket %s: %w", delivery.StepsBucket(module), err) - } - watch, err := kv.Watch(ctx, key) - if err != nil { - return delivery.Step{}, err - } - defer watch.Stop() - for { - select { - case <-ctx.Done(): - return delivery.Step{}, ctx.Err() - case e, ok := <-watch.Updates(): - if !ok { - return delivery.Step{}, errors.New("the watch of the step ended") - } - if e == nil || !opOf(e) { - continue - } - step, err := delivery.DecodeStep(e.Value()) - if err != nil { - return delivery.Step{}, fmt.Errorf("the step %s is not a step: %w", key, err) - } - if step.State != delivery.StepRunning { - return step, nil - } - } - } -} - // manifestReader reads manifests. of the indexer and parses the manifest it holds. type manifestReader struct{ kv jetstream.KeyValue } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/composition_line_integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/composition_line_integration_test.go new file mode 100644 index 0000000..e41c96c --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/composition_line_integration_test.go @@ -0,0 +1,63 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// assignerWrote raises the assigner's compositions bucket, the assigner online in the watchman's presence, and its verdict +// under the check's key, as the assigner leaves them on the bus. +func (b *bed) assignerWrote(t *testing.T, key string, v delivery.CompositionVerdict) { + t.Helper() + kv, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory}) + if err != nil { + t.Fatal(err) + } + pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox") + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + if _, err := b.buckets[delivery.PresenceBucket].Put(b.ctx, pkey, on); err != nil { + t.Fatal(err) + } + data, err := delivery.EncodeCompositionVerdict(v) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(b.ctx, key, data); err != nil { + t.Fatal(err) + } +} + +// 1297/AC3: started whole with the assigner's verdict on the bus, the checker's line says the composition's verdict, not +// "not yet run": a pass names its machine, a fail fails the check with the assigner's words. +func TestTheCheckersLineSaysTheAssignersComposition(t *testing.T) { + for _, c := range []struct { + state, owed, words string + }{ + {delivery.CompositionPass, "success", "composition passed"}, + {delivery.CompositionFail, "failure", "g14 runs web, which the change removes from its source"}, + } { + b := startBed(t) + key := b.candidate(t, "feat/x", pull(repo, 7, head1)) + group, heads, _ := delivery.ParseCheckerVerdictKey(key) + v := delivery.CompositionVerdict{Group: group, Heads: heads, State: c.state, SDK: "v0.1.14-x", Machines: []string{"g14"}} + if c.state == delivery.CompositionFail { + v.Machine, v.Words = "g14", c.words + } + b.assignerWrote(t, key, v) + b.start(t, nil) + got := b.verdict(t, key, ended) + if got.Composition == nil || got.Composition.State != c.state { + t.Fatalf("%s: the verdict's composition is %+v", c.state, got.Composition) + } + o := b.owed(t, 7, head1, c.owed) + if !strings.Contains(o.Text, c.words) || strings.Contains(o.Text, "not yet run") { + t.Errorf("%s: the line says %q", c.state, o.Text) + } + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions.go new file mode 100644 index 0000000..c5a3fe3 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions.go @@ -0,0 +1,131 @@ +package main + +import ( + "context" + "errors" + "sync" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-checker/internal/check" +) + +// compositionWatch reads the assigner's composition of a check (ADR 0528 clause 5) by watching its key and the +// assigner's presence: no poll and no timer. With no assigner online it answers check.ErrAssignerNotRunning at once. +type compositionWatch struct { + kv jetstream.KeyValue + // presence is the watchman's presence bucket; nil (not there) counts as no assigner online. + presence jetstream.KeyValue +} + +// Await answers the composition under key once the assigner wrote one that does not wait (check.AwaitComposition). +func (c compositionWatch) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + comps := make(chan check.CompositionEvent) + presence := make(chan check.PresenceEvent) + cw, err := c.kv.Watch(ctx, key) + if err != nil { + return delivery.CompositionVerdict{}, err + } + defer cw.Stop() + go func() { + for e := range cw.Updates() { + ev := check.CompositionEvent{Settled: e == nil} + if e != nil && e.Operation() == jetstream.KeyValuePut { + ev.Value = e.Value() + } + select { + case comps <- ev: + case <-ctx.Done(): + return + } + } + }() + if c.presence == nil { + go func() { + select { + case presence <- check.PresenceEvent{Settled: true}: + case <-ctx.Done(): + } + }() + } else { + pw, err := c.presence.Watch(ctx, "part."+delivery.AssignerModule+".*") + if err != nil { + return delivery.CompositionVerdict{}, err + } + defer pw.Stop() + go func() { + for e := range pw.Updates() { + ev := check.PresenceEvent{Settled: e == nil} + if e != nil { + ev.Key = e.Key() + if e.Operation() == jetstream.KeyValuePut { + if p, err := delivery.DecodePresence(e.Value()); err == nil { + ev.Online = p.Online + } + } + } + select { + case presence <- ev: + case <-ctx.Done(): + return + } + } + }() + } + return check.AwaitComposition(ctx, presence, comps) +} + +// followCompositions hands the checker each composition the assigner writes from now on, so a verdict that said its +// composition was not judged takes it in when the assigner starts (event-driven). +func followCompositions(ctx context.Context, kv jetstream.KeyValue, ch *check.Checker) error { + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return err + } + go func() { + defer w.Stop() + for e := range w.Updates() { + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + if comp, err := delivery.DecodeCompositionVerdict(e.Value()); err == nil { + ch.Composed(ctx, comp) + } + } + }() + return nil +} + +// lazyCompositions opens the assigner's compositions when a check first asks for one and the bucket is there: until it +// is, no assigner ever ran, so the check says at once that its composition is not judged (check.ErrAssignerNotRunning), +// and the next check looks again. No timer: each look is a check asking. +type lazyCompositions struct { + open func(ctx context.Context) (check.CompositionReader, error) + + mu sync.Mutex + reader check.CompositionReader +} + +// Await is the opened reader's Await, opening it first when it is not yet. +func (l *lazyCompositions) Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) { + l.mu.Lock() + r := l.reader + if r == nil { + opened, err := l.open(ctx) + switch { + case errors.Is(err, jetstream.ErrBucketNotFound): + l.mu.Unlock() + return delivery.CompositionVerdict{}, check.ErrAssignerNotRunning + case err != nil: + l.mu.Unlock() + return delivery.CompositionVerdict{}, err + } + l.reader, r = opened, opened + } + l.mu.Unlock() + return r.Await(ctx, key) +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_integration_test.go new file mode 100644 index 0000000..0325a35 --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_integration_test.go @@ -0,0 +1,106 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-checker/internal/check" +) + +// 1297/AC3 (its bus half): on the real bus, the checker's read of the assigner's composition waits while the key is +// absent and while the assigner says it waits, and answers the verdict the assigner then writes under the check's key. +func TestTheCheckerAwaitsTheAssignersComposition(t *testing.T) { + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "comp") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { os.RemoveAll(dir) }) + srv, err := server.NewServer(&server.Options{Host: "127.0.0.1", Port: -1, JetStream: true, StoreDir: filepath.Join(dir, "js"), NoLog: true, NoSigs: true}) + if err != nil { + t.Fatal(err) + } + go srv.Start() + if !srv.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + t.Cleanup(srv.Shutdown) + nc, err := nats.Connect(srv.ClientURL()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(nc.Close) + js, _ := jetstream.New(nc) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + kv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.AssignerCompositionsBucket, History: delivery.AssignerCompositionsHistory}) + if err != nil { + t.Fatal(err) + } + pkv, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.PresenceBucket, History: 5}) + if err != nil { + t.Fatal(err) + } + pkey, _ := delivery.PresencePartKey(delivery.AssignerModule, "novox") + on, _ := delivery.EncodePresence(delivery.Presence{Online: true}) + if _, err := pkv.Put(ctx, pkey, on); err != nil { + t.Fatal(err) + } + // With no assigner online the wait answers at once (a second key, before the assigner of this test is online). + off, _ := delivery.EncodePresence(delivery.Presence{Online: false}) + if _, err := pkv.Put(ctx, pkey, off); err != nil { + t.Fatal(err) + } + other, _ := delivery.CompositionKey("feat/y", "0123456789ab") + if _, err := (compositionWatch{kv: kv, presence: pkv}).Await(ctx, other); !errors.Is(err, check.ErrAssignerNotRunning) { + t.Fatalf("with no assigner online the wait answered %v", err) + } + if _, err := pkv.Put(ctx, pkey, on); err != nil { + t.Fatal(err) + } + key, _ := delivery.CompositionKey("feat/x", "0123456789ab") + got := make(chan delivery.CompositionVerdict, 1) + go func() { + v, err := compositionWatch{kv: kv, presence: pkv}.Await(ctx, key) + if err != nil { + t.Error(err) + } + got <- v + }() + put := func(v delivery.CompositionVerdict) { + data, err := delivery.EncodeCompositionVerdict(v) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put(ctx, key, data); err != nil { + t.Fatal(err) + } + } + time.Sleep(200 * time.Millisecond) + put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "the indexer holds no candidates yet"}) + select { + case v := <-got: + t.Fatalf("a waiting composition was answered: %+v", v) + case <-time.After(300 * time.Millisecond): + } + put(delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}}) + select { + case v := <-got: + if v.State != delivery.CompositionPass || v.Machines[0] != "g14" { + t.Errorf("answered %+v", v) + } + case <-time.After(5 * time.Second): + t.Fatal("the composition written was not answered") + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_test.go new file mode 100644 index 0000000..64afd0b --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/compositions_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "context" + "errors" + "testing" + + "github.com/nats-io/nats.go/jetstream" + + "git.novox.be/novox/mesh-sdk/go/delivery" + + "mesh-delivery-checker/internal/check" +) + +type answering struct{ v delivery.CompositionVerdict } + +func (a answering) Await(context.Context, string) (delivery.CompositionVerdict, error) { + return a.v, nil +} + +// 1297 review item 3: the compositions bucket missing when the checker starts is looked up again at the next check: while +// it is missing a check says at once the assigner is not running; once it is there the check reads its composition, and +// it is opened once. +func TestTheCompositionsBucketIsLookedUpAtEachCheck(t *testing.T) { + opens, there := 0, false + l := &lazyCompositions{open: func(context.Context) (check.CompositionReader, error) { + opens++ + if !there { + return nil, jetstream.ErrBucketNotFound + } + return answering{delivery.CompositionVerdict{State: delivery.CompositionPass, SDK: "x"}}, nil + }} + if _, err := l.Await(context.Background(), "k"); !errors.Is(err, check.ErrAssignerNotRunning) { + t.Fatalf("without the bucket: %v", err) + } + there = true + for range 2 { + v, err := l.Await(context.Background(), "k") + if err != nil || v.State != delivery.CompositionPass { + t.Fatalf("with the bucket: %+v, %v", v, err) + } + } + if opens != 2 { + t.Errorf("opened %d times, want twice (missing, then once)", opens) + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/heard.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/heard.go deleted file mode 100644 index e186fce..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/heard.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - - "github.com/nats-io/nats.go" - - "mesh-delivery-checker/internal/check" -) - -// heardCaller is the build runner's caller with the calls that reached nobody marked: no responders, or none answering within -// the call's bound. The step never began there, so the check may give it to another node (issue 1820). A call that was -// answered, with a failure or not, is left as it is. -type heardCaller struct { - call func(node, key string, body any) (json.RawMessage, error) -} - -func (h heardCaller) CallOn(node, key string, body any) (json.RawMessage, error) { - raw, err := h.call(node, key, body) - if errors.Is(err, nats.ErrNoResponders) || errors.Is(err, nats.ErrTimeout) || errors.Is(err, context.DeadlineExceeded) { - return nil, &check.Unheard{Err: err} - } - return raw, err -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/heard_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/heard_test.go deleted file mode 100644 index d660385..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/heard_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "testing" - - "github.com/nats-io/nats.go" - - "mesh-delivery-checker/internal/check" -) - -func TestACallThatReachedNobodyIsMarkedAndAnAnsweredFailureIsNot(t *testing.T) { // 1820/AC6 - for _, err := range []error{nats.ErrNoResponders, nats.ErrTimeout, context.DeadlineExceeded} { - h := heardCaller{call: func(string, string, any) (json.RawMessage, error) { return nil, fmt.Errorf("call: %w", err) }} - _, got := h.CallOn("g14", "m.t", nil) - var u *check.Unheard - if !errors.As(got, &u) { - t.Errorf("%v: %v is not marked unheard", err, got) - } - } - h := heardCaller{call: func(string, string, any) (json.RawMessage, error) { return nil, errors.New("the module refused") }} - _, got := h.CallOn("g14", "m.t", nil) - var u *check.Unheard - if errors.As(got, &u) { - t.Errorf("an answered failure %v is marked unheard", got) - } -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go deleted file mode 100644 index 1518bd7..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold.go +++ /dev/null @@ -1,101 +0,0 @@ -package main - -import ( - "context" - "errors" - "log" - "sync" - - "github.com/nats-io/nats.go/jetstream" - - "mesh-delivery-checker/internal/check" -) - -// holdWatch holds the checker back while a delivery builds (ADR 0484 clause 12): whether one does is the list of the keys of -// the controller's bucket of the delivery plans in their build phase, read each time a check is about to start. Its keys age -// out (a time to live, and no delete marker), which a watch never hears, so a watch cannot decide it (mesh-controller -// internal/link/holds.go lists the keys too); the watch is only the wake-up, for a plan that ended by a delete. The bucket -// is read as empty where it is absent (issue 1652 makes it) or cannot be read, saying so once. It may come into being after -// the checker started and the checker holds no timer (ADR 0437 §8), so each time a check is about to start and the bucket is -// not known yet, the checker looks for it. -type holdWatch struct { - ctx context.Context - js bucketOpener - keys func(ctx context.Context) ([]string, error) - changed func(ctx context.Context) // a plan key changed: the checks that waited may start - logger *log.Logger - errs chan error - mu sync.Mutex - watched bool - said bool -} - -func newHoldWatch(ctx context.Context, js bucketOpener, changed func(context.Context), logger *log.Logger) *holdWatch { - return &holdWatch{ctx: ctx, js: js, changed: changed, logger: logger, errs: make(chan error, 1)} -} - -// Held says whether a delivery builds now. -func (w *holdWatch) Held() bool { - w.ensure() - w.mu.Lock() - keys := w.keys - w.mu.Unlock() - if keys == nil { - return false - } - list, err := keys(w.ctx) - if err != nil { - w.say("the keys of %s cannot be listed, so it is read as empty: %v", check.PlansBuildingBucket, err) - return false - } - return len(list) > 0 -} - -func (w *holdWatch) say(format string, args ...any) { - w.mu.Lock() - defer w.mu.Unlock() - if !w.said { - w.said = true - w.logger.Printf(format, args...) - } -} - -// ensure finds the bucket if it is not known and exists, and starts the wake-up watch. -func (w *holdWatch) ensure() { - w.mu.Lock() - if w.watched || w.js == nil { - w.mu.Unlock() - return - } - w.mu.Unlock() - kv, err := w.js.KeyValue(w.ctx, check.PlansBuildingBucket) - switch { - case errors.Is(err, jetstream.ErrBucketNotFound): - w.say("the bucket %s does not exist yet: it is read as empty, so no check is held back", check.PlansBuildingBucket) - return - case err != nil: - w.say("the bucket %s cannot be read, so it is read as empty: %v", check.PlansBuildingBucket, err) - return - } - w.mu.Lock() - defer w.mu.Unlock() - if w.watched { - return - } - w.watched, w.said = true, false - w.keys = func(ctx context.Context) ([]string, error) { - list, err := kv.Keys(ctx) - if errors.Is(err, jetstream.ErrNoKeysFound) { - return nil, nil - } - return list, err - } - go func() { - wake, err := kv.WatchAll(w.ctx) - if err != nil { - w.errs <- err - return - } - w.errs <- follow(w.ctx, wake, func(jetstream.KeyValueEntry) { w.changed(w.ctx) }) - }() -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go deleted file mode 100644 index 3f15b31..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/hold_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "context" - "errors" - "io" - "log" - "testing" -) - -// The controller's plans-building keys age out of the bucket (a time to live, no delete marker), which a watch never hears: -// whether a delivery builds is the list of the keys now, and the watch is only a wake-up (review D3). -func TestADeliveryHoldsWhileItsKeyIsListedAndNotWhenItAgedOutWithNoEvent(t *testing.T) { - listed := []string{"plan-1"} - w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return listed, nil }} - if !w.Held() { - t.Fatal("a plan key is listed and nothing holds") - } - listed = nil // the key aged out: no delete reached the watch - if w.Held() { - t.Error("the key is gone from the bucket and the checker is still held") - } -} - -func TestABucketThatCannotBeListedIsReadAsEmpty(t *testing.T) { - w := &holdWatch{watched: true, logger: log.New(io.Discard, "", 0), keys: func(context.Context) ([]string, error) { return []string{"plan-1"}, errors.New("the bus is down") }} - if w.Held() { - t.Error("an unreadable bucket held the checker") - } - var absent holdWatch - absent.logger = log.New(io.Discard, "", 0) - if absent.Held() { - t.Error("a watch with no bucket held the checker") - } -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go index e2f4c15..b7611e1 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/integration_test.go @@ -5,6 +5,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -20,6 +21,7 @@ import ( "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" "mesh-delivery-checker/internal/check" @@ -75,7 +77,6 @@ func readCaptured(t *testing.T, name string) map[string]any { type fakeBuildModules struct { js jetstream.JetStream ctx context.Context - n atomic.Int64 conflict atomic.Bool mu sync.Mutex calls map[string]int // by `.` @@ -112,45 +113,55 @@ func (f *fakeBuildModules) result(t *testing.T, tool string, args map[string]any return data } -func (f *fakeBuildModules) serve(t *testing.T, nc *nats.Conn, module string) { - steps, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1}) +// serve makes module's steps bucket and its build seat's stream and workers, and pulls its asks on n1 as the real module +// does since R4g (ADR 0525; issue 1887 A5): each ask's job runs with step bodies answering the captured results, and a +// check takes the kept verdicts of the checker's kept bucket by their digests. +func (f *fakeBuildModules) serve(t *testing.T, module string, kept jetstream.KeyValue) { + kv, err := f.js.CreateKeyValue(f.ctx, jetstream.KeyValueConfig{Bucket: delivery.StepsBucket(module), History: 1}) if err != nil { t.Fatal(err) } - sub, err := nc.Subscribe("mesh.mod."+module+".call.>", func(m *nats.Msg) { - tokens := strings.Split(m.Subject, ".") // mesh mod call - tool, node := tokens[4], tokens[5] - f.mu.Lock() - f.calls[module+"."+tool]++ - f.mu.Unlock() - var args map[string]any - _ = json.Unmarshal(m.Data, &args) - id := fmt.Sprintf("s%d", f.n.Add(1)) - key, _ := delivery.StepKey(node, id) - put := func(s delivery.Step) { - data, err := delivery.EncodeStep(s) - if err == nil { - _, err = steps.Put(f.ctx, key, data) + language := strings.TrimPrefix(module, "mesh-build-") + if err := buildstep.DeclareAskWorkers(f.ctx, f.js, language, 30*time.Second); err != nil { + t.Fatal(err) + } + prefix := map[string]string{"mesh-build-go": "build_go_", "mesh-build-node": "build_node_"}[module] + call := func(name string) buildstep.StepFunc { + return func(ctx context.Context, args map[string]any) (any, error) { + tool := prefix + name + f.mu.Lock() + f.calls[module+"."+tool]++ + f.mu.Unlock() + return f.result(t, tool, args), nil + } + } + steps := buildstep.AskSteps{Fetch: call("fetch"), Test: call("test"), + Kept: func(ctx context.Context, level, digest string) (int, bool, error) { + key, err := delivery.KeptKey(level, digest) + if err != nil { + return 0, false, err + } + e, err := kept.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return 0, false, nil } if err != nil { - t.Errorf("writing step %s: %v", key, err) + return 0, false, err } - } - step := delivery.Step{ID: id, Tool: tool, Node: node, State: delivery.StepRunning, Started: time.Now().UTC()} - put(step) - answer, _ := json.Marshal(map[string]any{"result": map[string]any{"step": id, "key": key, "state": "running"}, "error": "", "node": node}) - _ = m.Respond(answer) - go func() { - time.Sleep(20 * time.Millisecond) - step.Ended = time.Now().UTC() - step.State, step.Result = delivery.StepPassed, f.result(t, tool, args) - put(step) - }() - }) - if err != nil { - t.Fatal(err) + v, err := delivery.DecodeKept(e.Value()) + return v.Tests, err == nil, err + }} + if module == "mesh-build-go" { + steps.Digest = call("digest") } - t.Cleanup(func() { _ = sub.Unsubscribe() }) + put := func(key string, value any) (uint64, error) { + raw, _ := value.(json.RawMessage) + return kv.Put(f.ctx, key, raw) + } + p := &buildstep.Puller{Source: &buildstep.JetStreamAsks{JS: f.js, Language: language}, Language: language, Node: "n1", Put: put, + Capacity: func() int { return 2 }, PullWait: 200 * time.Millisecond, + Run: func(ctx context.Context, a delivery.StepAsk) (any, error) { return buildstep.RunAsk(ctx, steps, a) }} + go func() { _ = p.Pull(f.ctx) }() } // serveForge answers the forge's two reads as its tools do, with the answers the forge gave (internal/check/testdata/captured): @@ -283,8 +294,8 @@ func startBed(t *testing.T) *bed { } b.serveForge(t, ctl) b.fake = &fakeBuildModules{js: js, ctx: ctx, calls: map[string]int{}} - b.fake.serve(t, ctl, "mesh-build-go") - b.fake.serve(t, ctl, "mesh-build-node") + b.fake.serve(t, "mesh-build-go", b.buckets[delivery.CheckerKeptBucket]) + b.fake.serve(t, "mesh-build-node", b.buckets[delivery.CheckerKeptBucket]) b.report(t, "mesh-build-go", "n1", "go", "go1.27.1", "1") b.report(t, "mesh-build-node", "n1", "typescript", "tsc5", "1") b.setTrunk(t, map[string]string{repo: trunk}) @@ -520,14 +531,25 @@ func TestAHeadThatDoesNotMergeFailsNamingTheFilesAndRunsNoTest(t *testing.T) { / // TestAnHqPullRequestGetsNoVerdictAndAnotherDoes: hq's pull request is judged by no one (ADR 0484 clause 8), the other one in // the same run is. -func TestAnHqPullRequestGetsNoVerdictAndAnotherDoes(t *testing.T) { // ADR 0484 clause 8 +// 1887/A10, ADR 0527 §2: a pull request into hq gets a verdict and a mesh/check from the checker, as any repository's. +func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) { b := startBed(t) + b.setTrunk(t, map[string]string{repo: trunk, "novox/hq": trunk}) hq := b.candidate(t, "feat/hq", pull("novox/hq", 3, head1)) - other := b.candidate(t, "feat/x", pull(repo, 7, head2)) b.start(t, nil) - b.verdict(t, other, ended) - if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, hq); err == nil { - t.Error("hq's pull request has a verdict") + if v := b.verdict(t, hq, ended); v.State != delivery.VerdictPass { + t.Fatalf("hq's verdict = %+v", v) + } + key, _ := delivery.CheckerOwedKey("novox/hq", 3, head1) + for deadline := time.Now().Add(20 * time.Second); ; time.Sleep(50 * time.Millisecond) { + if e, err := b.buckets[delivery.CheckerOwedBucket].Get(b.ctx, key); err == nil { + if o, err := delivery.DecodeCheckerOwed(e.Value()); err == nil && o.State == "success" { + return + } + } + if time.Now().After(deadline) { + t.Fatalf("hq#3 owes no success within 20 s") + } } } @@ -550,34 +572,6 @@ func TestACheckLeftRunningIsTakenOverWhenTheCheckerStarts(t *testing.T) { // N6/ } } -// TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds: the controller's bucket of the plans in their build phase -// holds a plan's key; the candidate waits, no step is called; the key deleted, the check starts and passes (ADR 0484 clause 12). -func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1 - b := startBed(t) - plans, err := b.js.CreateKeyValue(b.ctx, jetstream.KeyValueConfig{Bucket: "mesh-controller_plans-building", History: 1}) - if err != nil { - t.Fatal(err) - } - if _, err := plans.Put(b.ctx, "plan-1", []byte(`{"plan":"plan-1","since":"2026-10-21T19:00:00Z","seen":1}`)); err != nil { - t.Fatal(err) - } - key := b.candidate(t, "feat/x", pull(repo, 7, head1)) - b.start(t, nil) - time.Sleep(1500 * time.Millisecond) // a wait for nothing to happen: there is no event to wait on - if _, err := b.buckets[delivery.CheckerVerdictsBucket].Get(b.ctx, key); err == nil { - t.Fatal("a check started while a delivery builds") - } - if n := b.fake.count("mesh-build-go.build_go_fetch"); n != 0 { - t.Fatalf("%d fetches while a delivery builds", n) - } - if err := plans.Delete(b.ctx, "plan-1"); err != nil { - t.Fatal(err) - } - if v := b.verdict(t, key, ended); v.State != delivery.VerdictPass { - t.Errorf("verdict = %+v", v) - } -} - // TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit: the go.mod the forge answers names a commit that main does not // hold; the verdict fails naming the member and the commit and no test runs (ADR 0484 clause 9). func TestAMemberNamingAMeshSDKCommitOffTheTrunkFailsNamingTheCommit(t *testing.T) { // 1624/AC2 diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go index ed8ff96..cea8b84 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge.go @@ -15,11 +15,8 @@ import ( // judgeDeps are the parts the judge of a check is made of. type judgeDeps struct { - runner buildrun.Runner + asks check.Asks tools *buildrun.ToolchainSet - presence *buildrun.PresenceSet - have *buildrun.HaveSet - places *buildrun.Places trunks *check.Trunks kept check.KV previews check.KV @@ -27,11 +24,13 @@ type judgeDeps struct { forge check.Forge announce check.Announcer builds check.BuildStates - logf func(format string, args ...any) + // compositions is the assigner's bucket of compositions; nil leaves every composition not run. + compositions check.CompositionReader + logf func(format string, args ...any) } func newJudge(d judgeDeps) check.Judge { - return &check.Examiner{Steps: d.runner, Toolchains: d.tools, Presence: d.presence, Health: d.have, Places: d.places, + return &check.Examiner{Compositions: d.compositions, Asks: d.asks, Toolchains: d.tools, Kept: check.Kept{Store: d.kept}, Previews: d.previews, Modules: d.repos, Packages: d.repos, Forge: d.forge, Runs: d.repos, Builds: d.builds, Announce: d.announce, Trunks: d.trunks, Logf: d.logf} } @@ -142,17 +141,28 @@ func (r *repoModules) PackagePaths(repository string) []string { return out } -// BuildModules are the build modules that judge a repository, sorted. +// BuildModules are the build modules that judge a repository, sorted: the Go build module always, and the Node build +// module when a module of the repository builds TypeScript. A repository of which the mesh knows no module (mesh-lab, hq) +// is judged by both, each finding its own directories in the tree (go.mod files; package.json files), so a repository +// with tests in either language and no manifest is judged whole (issue 1887 A12). func (r *repoModules) BuildModules(repository string) []string { out := []string{buildrun.GoBuildModule} r.mu.Lock() defer r.mu.Unlock() + known := false for module, repo := range r.repoOf { - if repo == repository && r.langs[r.typed[module]]["typescript"] { + if repo != repository { + continue + } + known = true + if r.langs[r.typed[module]]["typescript"] { out = append(out, buildrun.NodeBuildModule) break } } + if !known { + out = append(out, buildrun.NodeBuildModule) + } sort.Strings(out) return out } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go index 8afa3c1..126b348 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/judge_test.go @@ -69,8 +69,10 @@ func TestARepositoryWithATypeScriptModuleIsJudgedByTheNodeBuildModuleToo(t *test if got := r.CommitOf("mesh-delivery-checker"); got != strings.Repeat("a", 40) || r.CommitOf("gone") != "" { t.Errorf("the commit the mesh runs = %q", got) } - if got := r.BuildModules("novox/mesh-sdk"); strings.Join(got, ",") != buildrun.GoBuildModule { - t.Errorf("a repository with no module: %v, want the Go build module", got) + // 1887/A12: a repository with no module the mesh knows (mesh-lab: nested go.mod files and a root package.json) is + // judged by both build modules. + if got := r.BuildModules("novox/mesh-lab"); strings.Join(got, ",") != buildrun.GoBuildModule+","+buildrun.NodeBuildModule { + t.Errorf("a repository with no module: %v, want both build modules", got) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go index e19c92c..c2f5f1e 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/kept_test.go @@ -37,7 +37,8 @@ func TestTheCheckerAnnouncesEveryStateItKeepsAndOwedReadable(t *testing.T) { // if !equal(manifest, announced) { t.Errorf("announced %v, the manifest keeps %v", announced, manifest) } - if want := []string{"owed", "previews", "verdicts"}; !equal(readable, want) { + // kept is read by the Go build module for a check ask (issue 1887 design limit 1). + if want := []string{"kept", "owed", "previews", "verdicts"}; !equal(readable, want) { t.Errorf("readable %v, want %v", readable, want) } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go index 41377e8..34db5d7 100644 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/main.go @@ -114,7 +114,9 @@ func durationEnv(getenv func(string) string, name string, def time.Duration) (ti // keptStates is every state the checker keeps and which others may read (ADR 0495 §2-§3): owed by the forge module, // which writes mesh/check from it (ADR 0521), and by the forge bridge; previews by the indexer; verdicts by the gate. var keptStates = []announce.State{ - {Name: "kept"}, + // The kept verdicts are read by the Go build module, which takes them for a check ask's unchanged packages (issue 1887's + // design limit 1); the checker alone writes them (ADR 0444 clause 5). + {Name: "kept", Readable: true}, {Name: "owed", Readable: true}, {Name: "previews", Readable: true}, {Name: "tries"}, @@ -189,20 +191,16 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err != nil { return err } - capacity, err := capacityFrom(strings.TrimSpace(getenv("MESH_CHECKER_SETTINGS")), getenv("MESH_CHECKER_CAPACITY")) - if err != nil { - return fmt.Errorf("the capacity setting: %w", err) - } link := strings.TrimSpace(getenv("MESH_CHECKER_LINK")) if link == "" { link = defaultLink } tools := &buildrun.ToolchainSet{} presence := &buildrun.PresenceSet{} - have := &buildrun.HaveSet{} trunks := &check.Trunks{} - places := &buildrun.Places{Capacity: capacity} - runner := buildrun.Runner{Caller: heardCaller{caller.CallOn}, Steps: stepsWaiter{js}, Bound: stepBound} + // A check is one ask per pull request and build module on the build seats' check stream, pulled by a build node with a + // free slot (novox/hq ADR 0525 §1, §3): the checker chooses no node and starts no step. + asks := buildrun.JetStreamAsks{JS: js} ch := &check.Checker{ Verdicts: kvStore{buckets[delivery.CheckerVerdictsBucket]}, Owed: kvStore{buckets[delivery.CheckerOwedBucket]}, @@ -222,11 +220,25 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro default: builds = buildReader{kvStore{kv}} } + // The assigner's compositions (ADR 0528 clause 5, issue 1297): their bucket is looked up at each check until it is + // there, so an assigner delivered after the checker started is read at the next check, and followed from then on. + compositions := &lazyCompositions{open: func(at context.Context) (check.CompositionReader, error) { + kv, err := js.KeyValue(at, delivery.AssignerCompositionsBucket) + if err != nil { + return nil, err + } + cw := compositionWatch{kv: kv} + if pkv, err := js.KeyValue(at, delivery.PresenceBucket); err == nil { + cw.presence = pkv + } + if err := followCompositions(ctx, kv, ch); err != nil { + return nil, fmt.Errorf("the bucket %s cannot be watched: %w", delivery.AssignerCompositionsBucket, err) + } + logger.Printf("the assigner's compositions are read from %s", delivery.AssignerCompositionsBucket) + return cw, nil + }} // The tries (ADR 0501 §5) are judged by the same judge, held by the same deliveries. tryer := &check.Tryer{Tries: kvStore{buckets[check.TriesBucket]}, Bound: checkBound, Logf: logger.Printf} - holds := newHoldWatch(ctx, js, func(ctx context.Context) { ch.Reconsider(ctx); tryer.Reconsider(ctx) }, logger) - ch.Hold = holds.Held - tryer.Hold = holds.Held announce := func(ctx context.Context, event, key string) { body, _ := json.Marshal(map[string]string{"key": key}) if _, err := js.Publish(ctx, eventSubject(event), body); err != nil { @@ -235,10 +247,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro } repos := &repoModules{manifests: kvStore{buckets[delivery.ManifestsBucket]}, logf: logger.Printf} ch.Judge = newJudge(judgeDeps{ - runner: runner, tools: tools, presence: presence, have: have, places: places, trunks: trunks, + asks: buildrun.Asker{Publisher: asks, Waiter: asks, Bound: stepBound}, tools: tools, trunks: trunks, kept: kvStore{buckets[delivery.CheckerKeptBucket]}, previews: kvStore{buckets[delivery.PreviewsBucket]}, repos: repos, builds: builds, forge: forgeReads{call: caller.Call}, logf: logger.Printf, - announce: announce, + announce: announce, compositions: compositions, }) ch.Announce = announce tryer.Judge = ch.Judge @@ -266,8 +278,10 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro defer cc.Stop() } errs := make(chan error, 8) - // The watchman's presence says which nodes are asleep. Where its bucket is not there yet no node is online, so nothing - // is checked; the checker says so and is started again with the watchman. + // The watchman's presence: a node coming back online asks again a check whose forge was not heard (UnheardWords), and + // the assigner's own presence says whether a composition is being made (compositionWatch, ADR 0537 §4). A check is no + // longer held for a node: its asks wait in the build seats' streams, which redeliver a lost node's ask (ADR 0525 §6, + // issue 1912), so the node-engine's have is not watched here. switch kv, err := js.KeyValue(ctx, delivery.PresenceBucket); { case errors.Is(err, jetstream.ErrBucketNotFound): logger.Printf("the bucket %s does not exist yet: no node counts as online until the checker starts again", delivery.PresenceBucket) @@ -281,27 +295,7 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro go func() { errs <- follow(ctx, pw, func(e jetstream.KeyValueEntry) { if presence.Put(e.Key(), e.Value(), !opOf(e)) { - ch.NodesChanged(ctx) // a node came online or went: a check that waited for one may go on - } - }) - }() - } - // The node-engine's have says whether a node's build module runs and is healthy: a node whose build module is not is not - // chosen. Where its bucket is not there yet no build module counts as healthy, so nothing is checked. - switch kv, err := js.KeyValue(ctx, delivery.HaveBucket); { - case errors.Is(err, jetstream.ErrBucketNotFound): - logger.Printf("the bucket %s does not exist yet: no build module counts as healthy until the checker starts again", delivery.HaveBucket) - case err != nil: - return fmt.Errorf("the bucket %s: %w", delivery.HaveBucket, err) - default: - hw, err := settle(ctx, kv, func(e jetstream.KeyValueEntry) { have.Put(e.Key(), e.Value(), !opOf(e)) }) - if err != nil { - return err - } - go func() { - errs <- follow(ctx, hw, func(e jetstream.KeyValueEntry) { - if have.Put(e.Key(), e.Value(), !opOf(e)) { - ch.NodesChanged(ctx) + ch.NodesChanged(ctx) // a node came online or went: a check whose forge was not heard may now be heard } }) }() @@ -377,7 +371,5 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err case err := <-watch.errs: return err - case err := <-holds.errs: - return err } } diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/nodes_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/nodes_test.go new file mode 100644 index 0000000..2a7b78d --- /dev/null +++ b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/nodes_test.go @@ -0,0 +1,21 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +// 1912: with checks as asks on the build seats' streams, the stream redelivers a lost node's ask (ADR 0525 §6), so the +// checker program opens no node-engine have to start again a check that failed for want of a node. +func TestTheCheckerWatchesNoHave(t *testing.T) { + raw, err := os.ReadFile("main.go") + if err != nil { + t.Fatal(err) + } + for _, s := range []string{"delivery.HaveBucket", "buildrun.HaveSet"} { + if strings.Contains(string(raw), s) { + t.Errorf("the checker program still reads %s", s) + } + } +} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go deleted file mode 100644 index d32d39f..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "os" - - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -// capacityFrom is the build capacity of each node (ADR 0478 clause 3): the module's setting `capacity`, `=` pairs separated by commas, laid into the settings file the controller composes (issue 1661); empty is one step at a -// time on every node. With no settings file named, fallback is read instead, for a run outside the module's launch. -func capacityFrom(path, fallback string) (map[string]int, error) { - if path == "" { - return buildrun.ParseCapacity(fallback) - } - raw, err := os.ReadFile(path) - if err != nil { - return nil, fmt.Errorf("the settings file: %w", err) - } - var s struct { - Capacity string `json:"capacity"` - } - if err := json.Unmarshal(raw, &s); err != nil { - return nil, fmt.Errorf("the settings file %s is not JSON: %w", path, err) - } - return buildrun.ParseCapacity(s.Capacity) -} diff --git a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go b/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go deleted file mode 100644 index 8543fe7..0000000 --- a/modules/mesh-delivery-checker/cmd/mesh-delivery-checker/settings_test.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "os" - "path/filepath" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" -) - -func settingsFile(t *testing.T, body string) string { - t.Helper() - p := filepath.Join(t.TempDir(), "settings.json") - if err := os.WriteFile(p, []byte(body), 0o600); err != nil { - t.Fatal(err) - } - return p -} - -// ADR 0484 clause 12: the build capacity is the module's setting `capacity`, laid into the settings file: empty is one step at a time everywhere. -func TestCapacityIsReadFromTheSettingsFile(t *testing.T) { - for body, want := range map[string]map[string]int{ - `{"capacity": "shanks=3,ace=2"}`: {"shanks": 3, "ace": 2}, - `{"capacity": ""}`: {}, - `{}`: {}, - } { - got, err := capacityFrom(settingsFile(t, body), "") - if err != nil || len(got) != len(want) || got["shanks"] != want["shanks"] || got["ace"] != want["ace"] { - t.Errorf("%s: capacity %v, %v, want %v", body, got, err, want) - } - } - p := &buildrun.Places{Capacity: mustCapacity(t, settingsFile(t, `{"capacity": ""}`))} - if p.CapacityOf("shanks") != 1 || p.CapacityOf("ace") != 1 { - t.Errorf("an empty setting gives %d and %d, want 1 everywhere", p.CapacityOf("shanks"), p.CapacityOf("ace")) - } -} - -func mustCapacity(t *testing.T, path string) map[string]int { - t.Helper() - c, err := capacityFrom(path, "") - if err != nil { - t.Fatal(err) - } - return c -} - -func TestACapacitySettingThatIsNotOneIsRefusedNamingIt(t *testing.T) { - if _, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=-1"}`), ""); err == nil { - t.Error("shanks=-1 accepted") - } - if c, err := capacityFrom(settingsFile(t, `{"capacity": "shanks=3,ace=2,g14=0,novox=0"}`), ""); err != nil || c["g14"] != 0 || len(c) != 4 { - t.Errorf("the starting value: %v, %v", c, err) - } - if _, err := capacityFrom(settingsFile(t, `not json`), ""); err == nil { - t.Error("a file that is not JSON accepted") - } - if _, err := capacityFrom(filepath.Join(t.TempDir(), "none"), ""); err == nil { - t.Error("a settings file that is not there accepted") - } -} - -// With no settings file named (a run outside the module's launch, as the tests do), the capacity is the environment's. -func TestWithNoSettingsFileTheCapacityIsTheEnvironments(t *testing.T) { - got, err := capacityFrom("", "ace=2") - if err != nil || got["ace"] != 2 { - t.Errorf("capacity %v, %v", got, err) - } -} diff --git a/modules/mesh-delivery-checker/files/settings.json b/modules/mesh-delivery-checker/files/settings.json deleted file mode 100644 index 1354ea4..0000000 --- a/modules/mesh-delivery-checker/files/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "capacity": "" -} diff --git a/modules/mesh-delivery-checker/go.mod b/modules/mesh-delivery-checker/go.mod index 0de4a55..c9e6d4a 100644 --- a/modules/mesh-delivery-checker/go.mod +++ b/modules/mesh-delivery-checker/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-checker go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-checker/go.sum b/modules/mesh-delivery-checker/go.sum index db1ea18..2a3e9f3 100644 --- a/modules/mesh-delivery-checker/go.sum +++ b/modules/mesh-delivery-checker/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-checker/internal/check/awaiting.go b/modules/mesh-delivery-checker/internal/check/awaiting.go new file mode 100644 index 0000000..1fbae2c --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/awaiting.go @@ -0,0 +1,141 @@ +package check + +import ( + "context" + "errors" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// ErrAssignerNotRunning is what a wait for a composition answers when no instance of the assigner is online: nobody is +// making it, so the check does not wait for it (issue 1297). The verdict says the composition is not judged, and when the +// assigner starts and writes it, the checker takes it in then (Checker.Composed). +var ErrAssignerNotRunning = errors.New(delivery.AssignerNotRunning) + +// PresenceEvent is a change of the watchman's presence bucket, or Settled once what stands was handed. +type PresenceEvent struct { + Key string + Online bool + Settled bool +} + +// CompositionEvent is a change of the composition's key (its value, nil for a delete), or Settled once what stands was +// handed. +type CompositionEvent struct { + Value []byte + Settled bool +} + +// AwaitComposition answers the assigner's composition once one is written that does not wait, while an instance of the +// assigner is online by the watchman's presence (`part.mesh-delivery-assigner.`). Once both watches handed +// what stands, no instance online is ErrAssignerNotRunning at once, and so is the last instance going offline during the +// wait. No timer: every answer follows an event; ctx ending is ctx's error. +func AwaitComposition(ctx context.Context, presence <-chan PresenceEvent, comps <-chan CompositionEvent) (delivery.CompositionVerdict, error) { + online := map[string]bool{} + presenceSettled, compSettled := false, false + for { + select { + case <-ctx.Done(): + return delivery.CompositionVerdict{}, ctx.Err() + case e, ok := <-presence: + if !ok { + return delivery.CompositionVerdict{}, errors.New("the watch of the assigner's presence ended") + } + if e.Settled { + presenceSettled = true + break + } + if _, part, instance, err := delivery.ParsePresenceKey(e.Key); err == nil && part == delivery.AssignerModule { + if e.Online { + online[instance] = true + } else { + delete(online, instance) + } + } + case e, ok := <-comps: + if !ok { + return delivery.CompositionVerdict{}, errors.New("the watch of the composition ended") + } + if e.Settled { + compSettled = true + break + } + if e.Value == nil { + break + } + v, err := delivery.DecodeCompositionVerdict(e.Value) + if err != nil { + return delivery.CompositionVerdict{}, fmt.Errorf("the composition cannot be read: %w", err) + } + if v.State != delivery.CompositionWaits { + return v, nil + } + } + if presenceSettled && compSettled && len(online) == 0 { + return delivery.CompositionVerdict{}, ErrAssignerNotRunning + } + } +} + +// withComposition is v with the assigner's composition taken in: a failed one fails the check, naming its machine. +func withComposition(v *delivery.Verdict, comp delivery.CompositionVerdict) { + v.Composition = comp.Composition() + v.Took.Composition = comp.Took + if comp.State == delivery.CompositionFail { + machine := comp.Machine + if machine == "" { + machine = "the composition" + } + v.State, v.Failed = delivery.VerdictFail, delivery.FailedFirst{Machine: machine, Words: comp.Words} + } +} + +// notJudged says whether a verdict's composition was not judged because the assigner was not running. +func notJudged(v delivery.Verdict) bool { + return v.Composition != nil && v.Composition.State == delivery.CompositionNotRun && v.Composition.Words == delivery.AssignerNotRunning +} + +// Composed takes in a composition the assigner wrote after the check's verdict said it was not judged because the +// assigner was not running (the assigner started since): the verdict and its status are written again with it. A +// composition of heads no candidate holds now, or of a verdict that judged its composition, is left. +func (ch *Checker) Composed(ctx context.Context, comp delivery.CompositionVerdict) { + ch.init() + if comp.State == delivery.CompositionWaits { + return + } + ckey, err := delivery.CandidatesKey(comp.Group) + if err != nil { + return + } + ch.mu.Lock() + c, ok := ch.known[ckey] + ch.mu.Unlock() + if !ok { + return + } + cs := CaseOf(c, ch.heads()) + if cs == nil || cs.HeadsHash != comp.Heads { + return + } + cur, rev, found, err := ch.Verdicts.Get(ctx, cs.Key) + if err != nil || !found { + return + } + v, err := delivery.DecodeVerdict(cur) + if err != nil || v.State != delivery.VerdictPass || !notJudged(v) { + return + } + withComposition(&v, comp) + data, err := delivery.EncodeVerdict(v) + if err != nil { + ch.logf("%s: the verdict with its composition cannot be written: %v", cs.Key, err) + return + } + if _, err := ch.Verdicts.Update(ctx, cs.Key, data, rev); err != nil { + ch.logf("%s: the verdict with its composition could not be written: %v", cs.Key, err) + return + } + ch.writeOwed(ctx, cs, v, Outcome{Verdict: v}) + ch.logf("%s: the composition came after the verdict: %s", cs.Key, comp.State) +} diff --git a/modules/mesh-delivery-checker/internal/check/awaiting_test.go b/modules/mesh-delivery-checker/internal/check/awaiting_test.go new file mode 100644 index 0000000..c775c49 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/awaiting_test.go @@ -0,0 +1,149 @@ +package check + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +func assignerOn(instance string, online bool) PresenceEvent { + key, _ := delivery.PresencePartKey(delivery.AssignerModule, instance) + return PresenceEvent{Key: key, Online: online} +} + +func compEvent(t *testing.T, v delivery.CompositionVerdict) CompositionEvent { + t.Helper() + data, err := delivery.EncodeCompositionVerdict(v) + if err != nil { + t.Fatal(err) + } + return CompositionEvent{Value: data} +} + +var passed = delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "x", Machines: []string{"g14"}} + +// 1297/B3d: with no assigner online, the wait answers ErrAssignerNotRunning as soon as both watches handed what stands. +func TestNoAssignerAnswersAtOnce(t *testing.T) { + presence, comps := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4) + presence <- assignerOn("novox", false) + presence <- PresenceEvent{Settled: true} + comps <- CompositionEvent{Settled: true} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := AwaitComposition(ctx, presence, comps); !errors.Is(err, ErrAssignerNotRunning) { + t.Fatalf("answered %v, want the assigner not running", err) + } +} + +// 1297/B3d: with the assigner online it waits for the key, past a composition that waits, and answers the verdict. +func TestAnOnlineAssignerIsWaitedFor(t *testing.T) { + presence, comps := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4) + presence <- assignerOn("novox", true) + presence <- PresenceEvent{Settled: true} + comps <- CompositionEvent{Settled: true} + done := make(chan error, 1) + var got delivery.CompositionVerdict + go func() { + var err error + got, err = AwaitComposition(context.Background(), presence, comps) + done <- err + }() + comps <- compEvent(t, delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionWaits, Words: "w"}) + select { + case err := <-done: + t.Fatalf("answered before the composition was written: %v", err) + case <-time.After(50 * time.Millisecond): + } + comps <- compEvent(t, passed) + if err := <-done; err != nil || got.State != delivery.CompositionPass { + t.Fatalf("answered %+v, %v", got, err) + } + // And the last instance going offline during a wait ends it as not running. + presence2, comps2 := make(chan PresenceEvent, 4), make(chan CompositionEvent, 4) + presence2 <- assignerOn("novox", true) + presence2 <- PresenceEvent{Settled: true} + comps2 <- CompositionEvent{Settled: true} + presence2 <- assignerOn("novox", false) + if _, err := AwaitComposition(context.Background(), presence2, comps2); !errors.Is(err, ErrAssignerNotRunning) { + t.Errorf("the assigner going offline answered %v", err) + } +} + +// 1297/B3d: a passing check with no assigner says at once, loud, that its composition is not judged; never a pass of it. +func TestAPassingCheckWithNoAssignerSaysTheCompositionIsNotJudged(t *testing.T) { + b := newBench(t) + b.ex.Compositions = &fakeCompositions{err: ErrAssignerNotRunning} + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if v.Composition == nil || v.Composition.State != delivery.CompositionNotRun || v.Composition.Words != delivery.AssignerNotRunning { + t.Fatalf("composition = %+v", v.Composition) + } + if words := compositionWords(v.Composition, 0); words != "composition not judged: the assigner is not running" { + t.Errorf("the line says %q", words) + } +} + +// 1297/B3d: the assigner starting writes the composition later, and the checker takes it into the verdict that said it +// was not judged, writing the status again; a failed one fails the check. +func TestTheAssignerStartingBringsTheComposition(t *testing.T) { + for _, comp := range []delivery.CompositionVerdict{passed, + {Group: "feat/x", Heads: "", State: delivery.CompositionFail, SDK: "x", Machine: "g14", Words: "g14 runs web, which the change removes"}} { + ch, cs := checkerWithNotJudged(t) + comp.Heads = cs.HeadsHash + ch.Composed(context.Background(), comp) + raw, _, _, _ := ch.Verdicts.Get(context.Background(), cs.Key) + v, err := delivery.DecodeVerdict(raw) + if err != nil || v.Composition == nil || v.Composition.State != comp.State { + t.Fatalf("after the composition came, the verdict is %+v, %v", v, err) + } + if comp.State == delivery.CompositionFail && (v.State != delivery.VerdictFail || v.Failed.Machine != "g14") { + t.Errorf("a failed composition left the verdict %+v", v) + } + owed := ch.Owed.(*memKV) + keys, _ := owed.Keys(context.Background()) + if len(keys) == 0 { + t.Fatal("no status was written again") + } + data, _, _, _ := owed.Get(context.Background(), keys[0]) + o, _ := delivery.DecodeCheckerOwed(data) + if !strings.Contains(o.Text, "composition passed") && !strings.Contains(o.Text, "g14 runs web") { + t.Errorf("the status's line says %q", o.Text) + } + } +} + +// checkerWithNotJudged is a checker holding a candidate whose passing verdict said its composition was not judged. +func checkerWithNotJudged(t *testing.T) (*Checker, *Case) { + t.Helper() + cs := caseOf(t, pullOf(repoA, 7, head1)) + raw := candidateOf(t, cs.Group, pullOf(repoA, 7, head1)) + ch := &Checker{Verdicts: newMemKV(), Owed: newMemKV(), Trunks: fixedTrunks{cs.Trunk}} + ckey, _ := delivery.CandidatesKey(cs.Group) + ch.init() + c, err := delivery.DecodeCandidate(raw) + if err != nil { + t.Fatal(err) + } + ch.known[ckey] = c + if got := CaseOf(c, ch.heads()); got == nil || got.Key != cs.Key { + t.Fatalf("the case is %+v, want %s", got, cs.Key) + } + v := delivery.Verdict{Group: cs.Group, Heads: cs.Heads, State: delivery.VerdictPass, + Levels: map[string]delivery.LevelResult{delivery.LevelUnit: {Run: 1}, delivery.LevelIntegration: {Run: 1}}, + Composition: &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning}} + data, err := delivery.EncodeVerdict(v) + if err != nil { + t.Fatal(err) + } + if _, err := ch.Verdicts.Create(context.Background(), cs.Key, data); err != nil { + t.Fatal(err) + } + return ch, cs +} + +type fixedTrunks struct{ heads map[string]string } + +func (f fixedTrunks) Heads() map[string]string { return f.heads } diff --git a/modules/mesh-delivery-checker/internal/check/beside.go b/modules/mesh-delivery-checker/internal/check/beside.go index 32ba99a..2453319 100644 --- a/modules/mesh-delivery-checker/internal/check/beside.go +++ b/modules/mesh-delivery-checker/internal/check/beside.go @@ -26,7 +26,7 @@ const ( // checks/deps). Which commit of each is besideCommit's rule. var besideRules = map[string][]string{ controllerRepo: {catalogueRepository, nodeEngineRepo, SDKRepository}, - nodeEngineRepo: {catalogueRepository, SDKRepository}, + nodeEngineRepo: {catalogueRepository, SDKRepository, controllerRepo}, SDKRepository: {catalogueRepository, nodeEngineRepo}, catalogueRepository: {SDKRepository, controllerRepo}, "novox/mesh-delivery": {SDKRepository, catalogueRepository}, @@ -35,13 +35,18 @@ var besideRules = map[string][]string{ // besideFor are the siblings a pull request's tests read (ADR 0484 clause 4): the catalogue at its trunk, always, even in a // group holding a catalogue pull request; the node-engine at the commit the mesh runs (the commit of the indexer's module // mesh-host, its trunk head when none is known); mesh-sdk, for the controller's tests, at the commit the controller's go.mod -// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk. +// pins; any other sibling at the head of the group's pull request for it when the group holds one, else at its trunk. And +// first, the change's own repository at the trunk it is merged onto, so a test comparing a shrink-only list with the +// trunk's copy reads it at $MESH_CHECK_BESIDE/ (issue 1887 A12, ADR 0529 §2). func (j *Examiner) besideFor(ctx context.Context, c *Case, p Pull) ([]BesideRef, error) { var heads map[string]string if j.Trunks != nil { heads = j.Trunks.Heads() } var out []BesideRef + if p.Trunk != "" { + out = append(out, BesideRef{Repository: p.Repository, Commit: p.Trunk}) + } for _, repo := range besideRules[p.Repository] { switch { case repo == catalogueRepository: diff --git a/modules/mesh-delivery-checker/internal/check/case.go b/modules/mesh-delivery-checker/internal/check/case.go index 4240ed4..22850d4 100644 --- a/modules/mesh-delivery-checker/internal/check/case.go +++ b/modules/mesh-delivery-checker/internal/check/case.go @@ -11,13 +11,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// HQRepository is the repository the dark checker writes no status for (ADR 0484 clause 8): hq builds no module and its -// check is a script no build module can run, so it is judged by its merge-check.sh until issue 1296 settles how. -const HQRepository = "novox/hq" - -// HQNote is what the read tools say of an hq pull request (ADR 0484 clause 8). -const HQNote = "hq: judged by merge-check.sh until issue 1296" - // Pull is one pull request the checker judges: its head, the trunk head of its repository it is merged onto and the // modules the indexer says it touches, with the fingerprints the indexer wrote for them. type Pull struct { @@ -29,8 +22,8 @@ type Pull struct { Fps map[string]delivery.CandidateModule } -// Case is one check: a candidate's pull requests that are judged (those of hq and of a repository whose trunk is not known -// are left out), the trunk head of each repository and the key of the verdict. +// Case is one check: a candidate's pull requests that are judged (those of a repository whose trunk is not known are left +// out; hq is judged as any repository since R4g, its checks Go tests: ADR 0527), the trunk head of each repository and the key of the verdict. type Case struct { // Key is the verdict key, `.`. Key string @@ -41,7 +34,7 @@ type Case struct { Pulls []Pull // Trunk is the trunk head of each repository of Pulls. Trunk map[string]string - // Left says why pull requests of the candidate are not judged (hq's, a repository with no known trunk); empty when all are. + // Left says why pull requests of the candidate are not judged (a repository with no known trunk); empty when all are. Left []string // Queued is when the checker took the case up, for the time of its queue. Queued time.Time @@ -49,6 +42,9 @@ type Case struct { Try *TryOptions // Progress says what the check is doing, so a check that ran past its bound names the step it was stuck at. Progress *Progress + // Kind is what asked the check (KindNew, KindNodes, KindTrunk): a re-judge after a trunk move is asked on the build + // seats' recheck stream, every other check on check (ADR 0536 §4). + Kind string } // Progress is the step a check is at, written by the check and read when its bound passes. @@ -83,8 +79,6 @@ func CaseOf(c delivery.Candidate, trunk map[string]string) *Case { cs := &Case{Key: key, Group: c.Group, HeadsHash: c.HeadsHash, Heads: GateHeads(c), Trunk: map[string]string{}, Progress: &Progress{}} for _, p := range c.Pulls { switch head, known := trunk[p.Repository]; { - case p.Repository == HQRepository: - cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: %s", p.Repository, p.Number, HQNote)) case !known || head == "": cs.Left = append(cs.Left, fmt.Sprintf("%s#%d: the trunk head of %s is not known", p.Repository, p.Number, p.Repository)) default: diff --git a/modules/mesh-delivery-checker/internal/check/checker.go b/modules/mesh-delivery-checker/internal/check/checker.go index 33f795e..d116005 100644 --- a/modules/mesh-delivery-checker/internal/check/checker.go +++ b/modules/mesh-delivery-checker/internal/check/checker.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "slices" "strings" "sync" "time" @@ -12,12 +11,11 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678). -const DefaultBound = 30 * time.Minute - -// NodesWords is in the Words of a failure that is the build nodes' (none qualifies, the toolchains disagree): such a verdict -// is judged again when the nodes change, by a watch and never a timer (ADR 0437 §8). -const NodesWords = "build node qualifies" +// DefaultBound is how long a check may run before it is ended failed naming the step it was stuck at (ADR 0484, issue 1678): +// none since R4g. A check's asks wait in the build seats' streams as long as the backlog is (ADR 0525 §9), and each ask's +// step is bounded from its start on the node that took it (buildrun.Asker.Bound, DefaultStepBound); a bound set by +// MESH_CHECKER_BOUND still ends a check that ran past it. +const DefaultBound time.Duration = 0 // UnheardWords end the Words of a failure because a part of the mesh the check reads was not heard (the forge, issue 1624): // like a failure of the nodes, it is not the change's, and is judged again when the nodes change. @@ -67,12 +65,8 @@ type Checker struct { Owed KV Judge Judge Trunks interface{ Heads() map[string]string } - // Bound is how long one check may run; DefaultBound when 0. + // Bound is how long one check may run; none when 0 (DefaultBound). Bound time.Duration - // Limit is how many checks run at once; two when 0. A freed place goes to the first waiting check by rank (ADR 0536 §4). - Limit int - // Hold says whether a delivery builds, so no new check starts (ADR 0484 clause 12); nil never holds. - Hold func() bool // Link is the base of the address a status points at; the verdict key follows. Link string // Announce says a status was written (EventOwedWritten); nil says nothing. @@ -81,10 +75,6 @@ type Checker struct { Logf func(format string, args ...any) once sync.Once - limit int - running int // checks holding a place - waiting []*waiter // checks waiting for a place, in the order they asked - seq uint64 wg sync.WaitGroup mu sync.Mutex known map[string]delivery.Candidate // by candidates key @@ -97,11 +87,6 @@ type Checker struct { func (ch *Checker) init() { ch.once.Do(func() { - limit := ch.Limit - if limit <= 0 { - limit = 2 - } - ch.limit = limit ch.known, ch.inflight, ch.again, ch.tried = map[string]delivery.Candidate{}, map[string]bool{}, map[string]bool{}, map[string]bool{} ch.nodesFail = map[string]uint64{} }) @@ -188,11 +173,7 @@ func (ch *Checker) heads() map[string]string { func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidate) { cs := CaseOf(c, ch.heads()) if cs == nil { - return // nothing to judge: hq's pull requests, or no trunk known - } - if ch.Hold != nil && ch.Hold() { - ch.logf("%s: not started, a delivery builds", cs.Key) - return // started when the plan bucket empties (Reconsider) + return // nothing to judge: no trunk known } cs.Queued = ch.now() ch.mu.Lock() @@ -207,9 +188,11 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat go func() { defer ch.wg.Done() for { - ch.admit(cs, ch.kindOf(ctx, cs)) + // Every check is asked at once: the build seats' streams hold the backlog, a new head's check and a check after a + // failure of the nodes on check, a re-judge after a trunk move on recheck, pulled after them (ADR 0525 §4, §9; + // ADR 0536 §4). The checker holds no place and no capacity of its own. + cs.Kind = ch.kindOf(ctx, cs) ch.try(ctx, key, cs) - ch.leave() ch.mu.Lock() more := ch.again[cs.Key] delete(ch.again, cs.Key) @@ -222,62 +205,19 @@ func (ch *Checker) consider(ctx context.Context, key string, c delivery.Candidat if next := CaseOf(ch.candidate(key, c), ch.heads()); next != nil { cs = next } - cs.Queued = ch.now() // asked again: it waits for a place by its rank like any other + cs.Queued = ch.now() } }() } -// The kinds of a check waiting for a place, by the rank they start in (ADR 0536 §4): a head never judged and a head asked -// again after a failure of the nodes first, then a head asked again only because its trunk moved. +// The kinds of a check (ADR 0536 §4): a head never judged and a head asked again after a failure of the nodes are asked on +// the build seats' check stream, a head asked again only because its trunk moved on the recheck stream, pulled after. const ( KindNew = "new" KindNodes = "nodes" KindTrunk = "trunk" ) -var kindRank = map[string]int{KindNew: 0, KindNodes: 0, KindTrunk: 1} - -// waiter is a check waiting for a place. -type waiter struct { - group, key, kind string - queued time.Time - seq uint64 - ready chan struct{} -} - -// Waiting names a check waiting for a place: its group, its verdict key, its kind and when it was asked. -type Waiting struct { - Group string `json:"group"` - Key string `json:"key"` - Kind string `json:"kind"` - Queued time.Time `json:"queued"` -} - -// Waiting are the checks waiting for a place, in the order they would start. -func (ch *Checker) Waiting() []Waiting { - ch.init() - ch.mu.Lock() - defer ch.mu.Unlock() - ws := slices.Clone(ch.waiting) - slices.SortStableFunc(ws, before) - out := make([]Waiting, len(ws)) - for i, w := range ws { - out[i] = Waiting{Group: w.group, Key: w.key, Kind: w.kind, Queued: w.queued} - } - return out -} - -// before orders waiting checks: the lower rank, then the one asked first, then the one that asked for a place first. -func before(a, b *waiter) int { - if ra, rb := kindRank[a.kind], kindRank[b.kind]; ra != rb { - return ra - rb - } - if c := a.queued.Compare(b.queued); c != 0 { - return c - } - return int(a.seq) - int(b.seq) -} - // kindOf is the kind of a case's check, read from its verdict key: none (or one left running) is a head never judged; a // failure of the nodes is asked again after the nodes changed; a verdict on other trunk heads is a re-judge after a trunk // move. A verdict that stands is ranked with the re-judges: its claim finds nothing to do. @@ -296,47 +236,6 @@ func (ch *Checker) kindOf(ctx context.Context, cs *Case) string { return KindTrunk } -// admit returns when the check holds a place: at once when one is free and no check waits, else when a check's end hands its -// place to this one as the first by rank. No timer: a check's end wakes the next (leave). -func (ch *Checker) admit(cs *Case, kind string) { - ch.mu.Lock() - if ch.running < ch.limit && len(ch.waiting) == 0 { - ch.running++ - ch.mu.Unlock() - return - } - ch.seq++ - w := &waiter{group: cs.Group, key: cs.Key, kind: kind, queued: cs.Queued, seq: ch.seq, ready: make(chan struct{})} - ch.waiting = append(ch.waiting, w) - ch.handOn() - ch.mu.Unlock() - <-w.ready -} - -// leave frees the check's place and hands it to the first waiting check by rank. -func (ch *Checker) leave() { - ch.mu.Lock() - ch.running-- - ch.handOn() - ch.mu.Unlock() -} - -// handOn gives each free place to the first waiting check by rank; it is called with mu held. -func (ch *Checker) handOn() { - for ch.running < ch.limit && len(ch.waiting) > 0 { - first := 0 - for i := range ch.waiting { - if before(ch.waiting[i], ch.waiting[first]) < 0 { - first = i - } - } - w := ch.waiting[first] - ch.waiting = slices.Delete(ch.waiting, first, first+1) - ch.running++ - close(w.ready) - } -} - func (ch *Checker) candidate(key string, fallback delivery.Candidate) delivery.Candidate { ch.mu.Lock() defer ch.mu.Unlock() @@ -407,9 +306,11 @@ func (ch *Checker) claim(ctx context.Context, cs *Case, first bool, epoch uint64 return &claimed{rev}, nil } -// IsNodesFailure says whether a verdict failed because no build node could take the check. +// IsNodesFailure says whether a verdict failed because a part of the mesh the check reads was not heard (UnheardWords): +// it is judged again when the nodes change. No check fails for want of a build node any more: its asks wait in the build +// seats' streams (ADR 0525 §6, issue 1912). func IsNodesFailure(v delivery.Verdict) bool { - return v.State == delivery.VerdictFail && (strings.Contains(v.Failed.Words, NodesWords) || strings.Contains(v.Failed.Words, UnheardWords)) + return v.State == delivery.VerdictFail && strings.Contains(v.Failed.Words, UnheardWords) } // try claims the check, judges it within its bound and writes the verdict and the owed statuses. A check that ends diff --git a/modules/mesh-delivery-checker/internal/check/checker_test.go b/modules/mesh-delivery-checker/internal/check/checker_test.go index 295a9ff..e8126c9 100644 --- a/modules/mesh-delivery-checker/internal/check/checker_test.go +++ b/modules/mesh-delivery-checker/internal/check/checker_test.go @@ -11,6 +11,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) +const hqRepo = "novox/hq" + const ( repoA = "novox/mesh-delivery" repoSDK = "novox/mesh-sdk" @@ -206,11 +208,13 @@ func TestACheckPastItsBoundFailsNamingTheStuckStep(t *testing.T) { // N6/AC3 } } -func TestNoQualifyingNodeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4 +// A check whose forge was not heard is judged again when the nodes change (issue 1912: no check fails for want of a build +// node any more; its asks wait in the streams). +func TestAnUnheardForgeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t *testing.T) { // N6/AC4 ctx := context.Background() j := &judgeFunc{f: func(_ context.Context, _ *Case, n int) (Outcome, error) { if n == 1 { - return Outcome{Verdict: failure("shanks", "no go "+NodesWords+": none is online")}, nil + return Outcome{Verdict: failure("the forge", "novox/mesh-delivery#7: go.mod cannot be read: no responders (the forge was "+UnheardWords+")")}, nil } return Outcome{Verdict: passing()}, nil }} @@ -234,10 +238,9 @@ func TestNoQualifyingNodeFailsWithTheReasonAndIsJudgedAgainWhenTheNodesChange(t } } -func TestHqAndARepositoryWithNoTrunkGetNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clauses 8 and 11 +func TestARepositoryWithNoTrunkGetsNoVerdict(t *testing.T) { // N6/AC6, ADR 0484 clause 11 ctx, j := context.Background(), passJudge() r := newRig(t, j) - r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(HQRepository, 3, head1))) r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf("novox/unknown", 4, head1))) r.ch.Wait() if j.count() != 0 || len(r.verdicts.vals) != 0 || len(r.owed.vals) != 0 { @@ -245,17 +248,24 @@ func TestHqAndARepositoryWithNoTrunkGetNoVerdict(t *testing.T) { // N6/AC6, ADR } } -func TestAGroupWithHqJudgesItsOtherMembersAndWritesNoStatusForHq(t *testing.T) { // ADR 0484 clause 8 +// 1887/A10, ADR 0527 §2: hq is judged as any repository, its checks Go tests, and its pull request gets a verdict and a +// mesh/check; in a group, hq's member is judged with the others. +func TestAnHqPullRequestGetsAVerdictAndAStatus(t *testing.T) { ctx, j := context.Background(), passJudge() r := newRig(t, j) - r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(HQRepository, 3, head1), pullOf(repoA, 7, head2))) + r.setTrunks(t, map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA2}) + r.ch.Candidate(ctx, "feat/hq", candidateOf(t, "feat/hq", pullOf(hqRepo, 3, head1))) r.ch.Wait() - if j.count() != 1 { - t.Fatalf("judged %d times, want once", j.count()) + if j.count() != 1 || len(r.verdicts.vals) != 1 { + t.Fatalf("judged %d times, %d verdicts, want one", j.count(), len(r.verdicts.vals)) } - keys, _ := r.owed.Keys(ctx) - if len(keys) != 1 || !strings.HasPrefix(keys[0], "novox/mesh-delivery.7.") && !strings.Contains(keys[0], "mesh-delivery") { - t.Errorf("statuses %v, want one for mesh-delivery#7 only", keys) + if s := owedState(t, r, hqRepo, 3, head1); s != "success" { + t.Errorf("hq#3 owes %q, want success", s) + } + r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(hqRepo, 4, head2), pullOf(repoA, 7, head2))) + r.ch.Wait() + if s := owedState(t, r, hqRepo, 4, head2); s != "success" { + t.Errorf("hq#4 in a group owes %q, want success", s) } } @@ -298,24 +308,6 @@ func TestATrunkMoveRewritesTheSameVerdictKeyAndTheHeadsStatus(t *testing.T) { // } } -func TestNoCheckStartsWhileADeliveryBuildsAndOneStartsWhenItEnds(t *testing.T) { // N8/AC1 - ctx, j := context.Background(), passJudge() - r := newRig(t, j) - held := true - r.ch.Hold = func() bool { return held } - r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) - r.ch.Wait() - if j.count() != 0 { - t.Fatalf("judged %d times while held", j.count()) - } - held = false - r.ch.Reconsider(ctx) - r.ch.Wait() - if j.count() != 1 { - t.Fatalf("judged %d times after the hold ended, want once", j.count()) - } -} - func TestTheStatusIsPendingWhileTheCheckRunsAndEndsSuccessOrFailure(t *testing.T) { // 1300/AC5 (unit part) ctx := context.Background() var seen []string diff --git a/modules/mesh-delivery-checker/internal/check/composition_test.go b/modules/mesh-delivery-checker/internal/check/composition_test.go new file mode 100644 index 0000000..19bb6b6 --- /dev/null +++ b/modules/mesh-delivery-checker/internal/check/composition_test.go @@ -0,0 +1,80 @@ +package check + +import ( + "context" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// fakeCompositions answers the assigner's composition of a check from memory, recording the keys asked. +type fakeCompositions struct { + v delivery.CompositionVerdict + err error + asked []string +} + +func (f *fakeCompositions) Await(_ context.Context, key string) (delivery.CompositionVerdict, error) { + f.asked = append(f.asked, key) + return f.v, f.err +} + +// 1297/AC3 (its unit half): a check whose tests pass takes the assigner's composition of it, by the check's own key, +// into its verdict: a pass says it passed with the mesh-sdk and machines it composed, and is no longer "not run". +func TestAPassingCheckTakesTheAssignersComposition(t *testing.T) { + b := newBench(t) + f := &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionPass, SDK: "v0.1.14-x", Machines: []string{"g14"}, Took: 40}} + b.ex.Compositions = f + cs := caseOf(t, pullOf(repoA, 7, head1)) + v := b.judge(t, cs).Verdict + if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionPass || v.Composition.SDK != "v0.1.14-x" || v.Took.Composition != 40 { + t.Fatalf("verdict = %+v, composition %+v", v, v.Composition) + } + if len(f.asked) != 1 || f.asked[0] != cs.Key { + t.Errorf("asked %v, want the check's key %s", f.asked, cs.Key) + } + if !compositionMentions(v, "passed") { + t.Errorf("the line says %q", compositionWords(v.Composition, v.Took.Composition)) + } +} + +func compositionMentions(v delivery.Verdict, word string) bool { + return strings.Contains(compositionWords(v.Composition, v.Took.Composition), word) +} + +// 1297/AC2 at the checker: a failed composition fails the check, naming the machine it failed at and its words. +func TestAFailedCompositionFailsTheCheck(t *testing.T) { + b := newBench(t) + b.ex.Compositions = &fakeCompositions{v: delivery.CompositionVerdict{Group: "feat/x", Heads: "0123456789ab", State: delivery.CompositionFail, SDK: "x", + Machine: "g14", Words: "g14 runs web, which the change removes from its source"}} + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if v.State != delivery.VerdictFail || v.Failed.Machine != "g14" || v.Composition == nil || v.Composition.State != delivery.CompositionFail { + t.Fatalf("verdict = %+v", v) + } + v.Group, v.Heads = "feat/x", "0123456789ab" // the checker names them when it writes the verdict + if _, err := delivery.EncodeVerdict(v); err != nil { + t.Errorf("the verdict cannot be written: %v", err) + } +} + +// 1297/B3d: a composition that cannot be read leaves it not run; a try is not composed. +func TestACompositionNotHadIsNotRun(t *testing.T) { + b := newBench(t) + f := &fakeCompositions{err: errors.New("the check's bound passed")} + b.ex.Compositions = f + v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if v.State != delivery.VerdictPass || v.Composition == nil || v.Composition.State != delivery.CompositionNotRun { + t.Fatalf("verdict = %+v", v) + } + b2 := newBench(t) + f2 := &fakeCompositions{v: delivery.CompositionVerdict{State: delivery.CompositionFail, Words: "w", SDK: "x"}} + b2.ex.Compositions = f2 + cs := caseOf(t, pullOf(repoA, 7, head1)) + cs.Try = &TryOptions{} + b2.judge(t, cs) + if len(f2.asked) != 0 { + t.Errorf("a try asked for a composition: %v", f2.asked) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/examiner_test.go b/modules/mesh-delivery-checker/internal/check/examiner_test.go index 1c4ee01..c8c4e17 100644 --- a/modules/mesh-delivery-checker/internal/check/examiner_test.go +++ b/modules/mesh-delivery-checker/internal/check/examiner_test.go @@ -15,6 +15,7 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) @@ -108,6 +109,7 @@ const digD = "dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44dd44" type bench struct { ex *Examiner + asks *buildrun.FakeAsks steps *fakeSteps kept *memKV prev *memKV @@ -116,6 +118,50 @@ type bench struct { test func(c stepCall) (any, error) } +// refsOf reads the siblings of a fetch's arguments, as they arrive in an ask (JSON) or as the checker gave them. +func refsOf(v any) []BesideRef { + raw, _ := json.Marshal(v) + var refs []BesideRef + _ = json.Unmarshal(raw, &refs) + return refs +} + +// toolPrefix is the start of each build module's tool names, as the fake build modules answer them. +var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"} + +// answer runs a check ask as the build module that pulls it does (buildstep.RunAsk), with the fake tools on shanks and the +// bench's kept verdicts, and answers the step it writes. +func (b *bench) answer(a delivery.StepAsk) delivery.Step { + module, _ := delivery.BuildModuleOf(a.Language) + const node = "shanks" + tool := func(name string) buildstep.StepFunc { + return func(ctx context.Context, args map[string]any) (any, error) { + raw, err := b.steps.Run(ctx, module, node, toolPrefix[module]+name, args) + if err != nil { + return nil, err + } + return raw, nil + } + } + steps := buildstep.AskSteps{Fetch: tool("fetch"), Test: tool("test"), + Kept: func(ctx context.Context, level, digest string) (int, bool, error) { + k, found, err := (Kept{Store: b.kept}).Lookup(ctx, level, digest) + return k.Tests, found, err + }} + if module == buildrun.GoBuildModule { + steps.Digest = tool("digest") + } + out, err := buildstep.RunAsk(context.Background(), steps, a) + began := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) + step := delivery.Step{ID: a.ID, Tool: a.Kind + " ask", Node: node, State: delivery.StepPassed, Started: began, Ended: began.Add(2 * time.Second)} + if err != nil { + step.State, step.Reason = delivery.StepFailed, err.Error() + return step + } + step.Result, _ = json.Marshal(out) + return step +} + // newBench is an examiner over fake build modules that merge cleanly, digest the captured fixture's packages and pass // the captured slow fixture's result at both levels, on shanks. func newBench(t testing.TB) *bench { @@ -137,12 +183,13 @@ func newBench(t testing.TB) *bench { b.steps.handlers = map[string]func(stepCall) (any, error){ "build_go_fetch": func(stepCall) (any, error) { return b.fetch, nil }, "build_node_fetch": func(stepCall) (any, error) { return b.fetch, nil }, - "build_go_digest": func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil }, + "build_go_digest": func(c stepCall) (any, error) { return echoBeside(b, b.digest[c.args["level"].(string)]), nil }, "build_go_test": func(c stepCall) (any, error) { return b.test(c) }, "build_node_test": func(c stepCall) (any, error) { return b.test(c) }, } now := time.Date(2026, 10, 21, 19, 0, 0, 0, time.UTC) - b.ex = &Examiner{Steps: b.steps, Toolchains: goEntries("shanks"), Places: &buildrun.Places{}, Kept: Kept{Store: b.kept}, Previews: b.prev, + b.asks = &buildrun.FakeAsks{Answer: b.answer} + b.ex = &Examiner{Asks: buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 5 * time.Second}, Toolchains: goEntries("shanks"), Kept: Kept{Store: b.kept}, Previews: b.prev, Modules: langs{}, Trunks: &Trunks{}, Now: func() time.Time { now = now.Add(50 * time.Millisecond); return now }} return b } @@ -353,33 +400,6 @@ func TestAHeadThatDoesNotMergeEndsFailedNamingTheFilesAndRunsNoTest(t *testing.T } } -func TestNoQualifyingNodeIsAFailureOfTheNodes(t *testing.T) { // N6/AC4 - b := newBench(t) - b.ex.Toolchains = entries{} - out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) - if !IsNodesFailure(out.Verdict) { - t.Fatalf("verdict = %+v, want a failure of the nodes", out.Verdict) - } - b.ex.Toolchains = goEntries("shanks") - b.ex.Presence = offline{} - if out = b.judge(t, caseOf(t, pullOf(repoA, 7, head1))); !IsNodesFailure(out.Verdict) { - t.Fatalf("an offline node: verdict = %+v", out.Verdict) - } -} - -type offline struct{} - -func (offline) Online(string) bool { return false } - -func TestAStepThatFailsIsAVerdictNamingTheNode(t *testing.T) { - b := newBench(t) - b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { return nil, fmt.Errorf("the step failed: ssh refused") } - out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) - if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "ssh refused") { - t.Fatalf("verdict = %+v", out.Verdict) - } -} - func TestALevelWhoseDigestIsNotNamedRunsWholeAndKeepsNothing(t *testing.T) { // 1287/AC4 b := newBench(t) b.steps.handlers["build_go_digest"] = func(stepCall) (any, error) { return nil, fmt.Errorf("go list failed") } @@ -441,7 +461,7 @@ func TestAGroupIsOneVerdictAndEachMembersTestsReadTheOthersAtTheirHeads(t *testi var forA []BesideRef for _, f := range fetch { if f.args["repository"] == repoA { - forA, _ = f.args["beside"].([]BesideRef) + forA, _ = refsOf(f.args["beside"]), true } } found := false @@ -571,6 +591,8 @@ func trunksOf(t *testing.T, heads map[string]string) *Trunks { func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversThem(t *testing.T) { // review D1, ADR 0444, issue 1287 b := newBench(t) + // A build module whose digest does not echo the siblings (before DigestVersion 3). + b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) { return b.digest[c.args["level"].(string)], nil } b.ex.Trunks = trunksOf(t, map[string]string{repoSDK: trunkS, "novox/mesh-catalog": trunkA}) b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) if got := len(b.steps.tool("fetch")); got != 1 || b.steps.tool("fetch")[0].args["beside"] == nil { @@ -592,12 +614,27 @@ func TestNothingIsKeptForAWorkspaceGivenSiblingsWhenTheDigestDoesNotSayItCoversT // besideDigest stands for the Go build module's digest over a workspace with siblings (DigestVersion 3, lead decision L5): // each package's digest folds the siblings' commits in, and the answer echoes the lines it folded. +// echoBeside is d with the siblings the fetches placed, as the Go build module's digest echoes them (its DigestResult's +// Beside): every check now places the change's own repository at its trunk (issue 1887 A12). +func echoBeside(b *bench, d Digested) Digested { + var lines []string + for _, f := range b.steps.tool("fetch") { + refs, _ := refsOf(f.args["beside"]), true + for _, r := range refs { + lines = append(lines, r.Repository+" "+r.Commit) + } + } + sort.Strings(lines) + d.Beside = lines + return d +} + func besideDigest(b *bench) func(c stepCall) (any, error) { return func(c stepCall) (any, error) { level := c.args["level"].(string) var lines []string for _, f := range b.steps.tool("fetch") { - refs, _ := f.args["beside"].([]BesideRef) + refs, _ := refsOf(f.args["beside"]), true for _, r := range refs { lines = append(lines, r.Repository+" "+r.Commit) } @@ -724,40 +761,85 @@ func TestNoStepOfACheckCarriesPublish(t *testing.T) { // ADR 0500 §4, 1309 (the } } -// keyCaller is a build module that starts every long step and answers its key on the node it was asked on. -type keyCaller struct{} - -func (keyCaller) CallOn(node, key string, _ any) (json.RawMessage, error) { - return json.Marshal(map[string]string{"step": "lost1", "key": node + ".lost1", "state": delivery.StepRunning}) -} - -// lostSteps is a steps state whose key never ends: the build module stopped, restarted or was unassigned with the step -// running, and nothing writes its end. -type lostSteps struct{} - -func (lostSteps) Wait(ctx context.Context, _, _ string) (delivery.Step, error) { - <-ctx.Done() - return delivery.Step{}, ctx.Err() -} - -func TestAStepThatNeverEndsEndsTheCheckLoudlyAndFreesItsNodesPlace(t *testing.T) { // 1890/AC1 +// 1890/AC1, 1887/A5a: an ask whose step never ends (the build node stopped with it running, and nothing redelivered it in +// time) ends the check loudly at the asker's bound; the checker holds no place to free. +func TestAnAskWhoseStepNeverEndsEndsTheCheckLoudly(t *testing.T) { b := newBench(t) - places := &buildrun.Places{Capacity: map[string]int{"shanks": 1}} - b.ex.Places = places - b.ex.Steps = buildrun.Runner{Caller: keyCaller{}, Steps: lostSteps{}, Bound: 20 * time.Millisecond} + // A node took the ask and its step stays running: the bound runs from its start (review round 2 item 1). + b.asks.Answer = func(a delivery.StepAsk) delivery.Step { + return delivery.Step{ID: a.ID, Tool: "check ask", Node: "shanks", State: delivery.StepRunning, Started: time.Now()} + } + b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond} out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) - if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || f.Machine != "shanks" || !strings.Contains(f.Words, "shanks.lost1 did not end within") { + if f := out.Verdict.Failed; out.Verdict.State != delivery.VerdictFail || !strings.Contains(f.Words, "on shanks did not end within") { t.Fatalf("verdict = %+v", out.Verdict) } - if r, res := places.Running(), places.Reserved(); len(r) != 0 || len(res) != 0 { - t.Fatalf("the lost step still holds shanks: running %v, reserved %v", r, res) + if n := len(b.asks.Asks()); n != 1 || b.asks.Asks()[0].Kind != delivery.AskCheck || b.asks.Asks()[0].Check == nil { + t.Fatalf("asks = %+v, want one check ask", b.asks.Asks()) + } +} + +// 1887/A5a, ADR 0536 §4: a check is one ask per pull request and build module on the check stream, a re-judge after a +// trunk move one on the recheck stream, each carrying both levels with the kept verdicts taken by the build module. +func TestACheckIsOneAskAndAReJudgeGoesOnTheRecheckStream(t *testing.T) { + b := newBench(t) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + asks := b.asks.Asks() + if len(asks) != 1 || asks[0].Kind != delivery.AskCheck || asks[0].Language != delivery.AskGo || asks[0].Asker != delivery.CheckerModule { + t.Fatalf("asks = %+v, want one Go check ask", asks) + } + if j := asks[0].Check; j == nil || !j.Kept || strings.Join(j.Levels, ",") != "unit,integration" || j.Fetch["repository"] != repoA { + t.Fatalf("the job = %+v", asks[0].Check) + } + cs := caseOf(t, pullOf(repoA, 7, head1)) + cs.Kind = KindTrunk + b.judge(t, cs) + if asks := b.asks.Asks(); len(asks) != 2 || asks[1].Kind != delivery.AskRecheck { + t.Fatalf("asks = %+v, want the re-judge on recheck", asks) + } +} + +// Review round 2 item 1: a check ask that waits in its stream longer than the step bound (no node has a free slot during a +// large rebuild) is not failed: the check goes on when a node takes it, and no second ask is put. +func TestACheckAskWaitingInItsStreamIsNotBounded(t *testing.T) { + b := newBench(t) + answer := b.asks.Answer + b.asks.Answer = nil + b.ex.Asks = buildrun.Asker{Publisher: b.asks, Waiter: b.asks, Bound: 20 * time.Millisecond} + done := make(chan Outcome, 1) + go func() { done <- b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) }() + for deadline := time.Now().Add(2 * time.Second); len(b.asks.Asks()) == 0; time.Sleep(time.Millisecond) { + if time.Now().After(deadline) { + t.Fatal("no ask was put") + } + } + time.Sleep(100 * time.Millisecond) // five bounds in the stream + select { + case out := <-done: + t.Fatalf("the check ended while its ask waited: %+v", out.Verdict) + default: + } + b.asks.End(answer(b.asks.Asks()[0])) // a node took it and ran it + if out := <-done; out.Verdict.State != delivery.VerdictPass || len(b.asks.Asks()) != 1 { + t.Fatalf("verdict %+v after %d asks, want a pass from the one ask", out.Verdict, len(b.asks.Asks())) + } +} + +// 1917: the digest is taken inside the check ask, as the build module's own digest step answers it (no step key is asked +// of it), so a second check of the same heads keeps every package the first one passed: "N kept", nothing run again. +func TestASecondCheckOfTheSameHeadsKeepsWhatTheFirstPassed(t *testing.T) { + b := newBench(t) + first := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + u := first.Levels[delivery.LevelUnit] + if first.State != delivery.VerdictPass || u.Run == 0 || u.Kept != 0 { + t.Fatalf("the first check = %+v", first) + } + b.steps.calls = nil + second := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict + if s := second.Levels[delivery.LevelUnit]; second.State != delivery.VerdictPass || s.Kept != u.Run || s.Run != 0 { + t.Fatalf("the second check's unit level = %+v, want %d kept and none run", s, u.Run) + } + if n := len(b.steps.tool("test")); n != 0 { + t.Errorf("%d test steps ran on the second check", n) } - // The node's one place is free: the next check takes it at once. - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - release, err := places.Acquire(ctx, "shanks") - if err != nil { - t.Fatalf("shanks's place was not freed: %v", err) - } - release() } diff --git a/modules/mesh-delivery-checker/internal/check/gate_test.go b/modules/mesh-delivery-checker/internal/check/gate_test.go index 6c709e0..c66168e 100644 --- a/modules/mesh-delivery-checker/internal/check/gate_test.go +++ b/modules/mesh-delivery-checker/internal/check/gate_test.go @@ -4,6 +4,7 @@ import ( "context" "strings" "testing" + "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -46,11 +47,11 @@ func TestTheVerdictsHeadsAreTheGatesHashOverLowercaseHeads(t *testing.T) { // ga func TestAGroupsVerdictHeadsCoverEveryMember(t *testing.T) { // gate (b): hq's member too, as the gate's ask covers it ctx := context.Background() r := newRig(t, passJudge()) - cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(HQRepository, 3, head2)) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(repoSDK, 12, head2), pullOf(hqRepo, 3, head2)) r.ch.Candidate(ctx, "feat%2Fx", cand) r.ch.Wait() want, _ := delivery.HeadsHash([]delivery.Head{{Repository: repoA, Number: 7, Head: head1}, {Repository: repoSDK, Number: 12, Head: head2}, - {Repository: HQRepository, Number: 3, Head: head2}}) + {Repository: hqRepo, Number: 3, Head: head2}}) if v := r.verdict(t, keyOf(t, "feat/x", cand)); v.Heads != want { t.Errorf("heads = %q, want %q", v.Heads, want) } @@ -71,11 +72,12 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) { t.Fatalf("%s#%d is %q before the move", p.repo, p.n, s) } } - // A delivery builds, so no check starts; mesh-delivery's trunk moves. - r.ch.Hold = func() bool { return true } + // mesh-delivery's trunk moves; the re-judges it asks are held in their judge (in the stream, since R4g) meanwhile. + release := make(chan struct{}) + r.ch.Judge = &judgeFunc{f: func(context.Context, *Case, int) (Outcome, error) { <-release; return Outcome{Verdict: passing()}, nil }} + defer func() { close(release); r.ch.Wait() }() r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) r.ch.TrunkMoved(ctx) - r.ch.Wait() if s := owedState(t, r, repoA, 7, head1); s != "pending" { t.Errorf("mesh-delivery#7, into the moved trunk, is %q, want pending", s) } @@ -90,17 +92,25 @@ func TestATrunkMoveMarksEveryOpenPullRequestIntoItPendingAtOnce(t *testing.T) { func TestACheckThatEndsAfterItsTrunkMovedOwesPendingNotItsVerdict(t *testing.T) { // gate (a): a pass on an old trunk is never shown ctx := context.Background() var r *rig + release, judged := make(chan struct{}), make(chan struct{}) j := &judgeFunc{f: func(ctx context.Context, c *Case, call int) (Outcome, error) { if call == 1 { r.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) // the trunk moves while it runs - r.ch.Hold = func() bool { return true } // and a delivery holds the next check + defer close(judged) + } else { + <-release // the check asked again for the new trunk waits in the stream meanwhile } return Outcome{Verdict: passing()}, nil }} r = newRig(t, j) r.ch.Candidate(ctx, "feat%2Fx", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) - r.ch.Wait() + <-judged + for deadline := time.Now().Add(2 * time.Second); owedState(t, r, repoA, 7, head1) == "" && time.Now().Before(deadline); time.Sleep(time.Millisecond) { + } + time.Sleep(20 * time.Millisecond) if s := owedState(t, r, repoA, 7, head1); s != "pending" { t.Errorf("the status is %q, want pending: the pass was judged on a trunk that moved", s) } + close(release) + r.ch.Wait() } diff --git a/modules/mesh-delivery-checker/internal/check/heard.go b/modules/mesh-delivery-checker/internal/check/heard.go index d97c17f..b10bd09 100644 --- a/modules/mesh-delivery-checker/internal/check/heard.go +++ b/modules/mesh-delivery-checker/internal/check/heard.go @@ -1,18 +1,14 @@ package check import ( - "context" - "encoding/json" "errors" - "fmt" - "strings" "github.com/nats-io/nats.go" ) // Unheard is the error of a call that reached nobody: the bus had no responder for it, or none answered within the call's -// bound. The step it asked for never began, so the same step may be asked of another node (issue 1820). The program wraps -// the errors of its calls in it where the bus gives a timeout; a no-responders error is known as it is. +// bound. The program wraps the errors of its forge calls in it where the bus gives a timeout; a no-responders error is +// known as it is. A build step is no call since R4g: an ask whose node stops is delivered again by its stream (ADR 0525 §6). type Unheard struct{ Err error } func (u *Unheard) Error() string { return u.Err.Error() } @@ -23,79 +19,3 @@ func isUnheard(err error) bool { var u *Unheard return errors.As(err, &u) || errors.Is(err, nats.ErrNoResponders) } - -// deafNode is a node that did not answer a build module's call during this judging. -type deafNode struct{ node, why string } - -// markDeaf takes a node out of the choice for a build module for the rest of the judging. -func (st *state) markDeaf(bm, node string, err error) { - if st.deaf == nil { - st.deaf = map[string][]deafNode{} - } - st.deaf[bm] = append(st.deaf[bm], deafNode{node, err.Error()}) -} - -func (st *state) isDeaf(bm, node string) bool { - for _, d := range st.deaf[bm] { - if d.node == node { - return true - } - } - return false -} - -// triedWords names every node that did not answer a build module, with what the bus said. -func (st *state) triedWords(bm string) string { - var parts []string - for _, d := range st.deaf[bm] { - parts = append(parts, fmt.Sprintf("%s (%s)", d.node, d.why)) - } - return strings.Join(parts, "; ") -} - -// moveStep is called when the target's node did not answer: the node is left out of the choice, and another build node of the -// same build module takes the target's place. It answers an error naming every node tried when none is left (the words hold -// NodesWords, so the verdict is judged again when the nodes change). -func (st *state) moveStep(t *target, err error) error { - old := t.node - st.markDeaf(t.module, old, err) - node, _, cerr := st.choose(t.module) - if cerr != nil { - return fmt.Errorf("%s; tried: %s", cerr, st.triedWords(t.module)) - } - st.j.logf("%s#%d: %s did not answer (%v), the step goes to %s", t.pull.Repository, t.pull.Number, old, err, node) - t.node = node - other := false - for _, o := range st.targets { - if o != t && o.node == old { - other = true - } - } - if !other { - delete(st.nodes, old) - } - st.nodes[node] = true - return nil -} - -// refetch makes the target's workspace again on its new node: the workspace of a step that was lost is on the node that did -// not answer. -func (st *state) refetch(ctx context.Context, t *target) error { - raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", t.pull.Repository, t.pull.Number, t.node), t.fetchArgs) - if err != nil { - return err - } - var f Fetched - if err := json.Unmarshal(raw, &f); err != nil || f.Workspace == "" { - return fmt.Errorf("the fetch of %s#%d on %s answered no workspace: %s", t.pull.Repository, t.pull.Number, t.node, raw) - } - t.fetched = f - return nil -} - -// stepOnWorkspace sets the workspace a step reads to the target's current one. -func stepOnWorkspace(t *target, args map[string]any) { - if _, ok := args["workspace"]; ok { - args["workspace"] = t.fetched.Workspace - } -} diff --git a/modules/mesh-delivery-checker/internal/check/heard_test.go b/modules/mesh-delivery-checker/internal/check/heard_test.go index 784ffc7..3072d8c 100644 --- a/modules/mesh-delivery-checker/internal/check/heard_test.go +++ b/modules/mesh-delivery-checker/internal/check/heard_test.go @@ -1,9 +1,7 @@ package check import ( - "context" "fmt" - "strings" "testing" "github.com/nats-io/nats.go" @@ -11,127 +9,8 @@ import ( "git.novox.be/novox/mesh-sdk/go/delivery" ) -// Issue 1820: a build step goes to a node whose build module is not heard (a laptop asleep, not yet marked offline by the -// watchman), and the bus answers "no responders". The check moves the step to another node of the same build module. - -// deafNodes makes every tool on the named nodes fail as the bus does when nobody listens there; count says how many -// times the node was called. -func deafNodes(b *bench, deaf ...string) { - for tool, h := range b.steps.handlers { - h := h - b.steps.handlers[tool] = func(c stepCall) (any, error) { - for _, n := range deaf { - if c.node == n { - return nil, fmt.Errorf("%s on %s: %w", c.tool, c.node, nats.ErrNoResponders) - } - } - return h(c) - } - } -} - -func nodesOf(calls []stepCall) []string { - var out []string - for _, c := range calls { - out = append(out, c.node) - } - return out -} - -func TestAFetchThatNobodyAnswersMovesToAnotherBuildNode(t *testing.T) { // 1820/AC1 - b := newBench(t) - b.ex.Toolchains = goEntries("ace", "g14") - deafNodes(b, "ace") - v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict - if v.State != delivery.VerdictPass { - t.Fatalf("verdict = %+v, want a pass on the node that answers", v) - } - if got := nodesOf(b.steps.tool("fetch")); len(got) != 2 || got[0] != "ace" || got[1] != "g14" { - t.Errorf("fetch went to %v, want ace (no responders) then g14", got) - } - for _, c := range b.steps.tool("test") { - if c.node != "g14" { - t.Errorf("a test went to %s, want g14 only", c.node) - } - } - if len(v.Machines) != 1 || v.Machines[0] != "g14" { - t.Errorf("machines = %v, want [g14]: ace ran nothing", v.Machines) - } -} - -func TestATestStepThatNobodyAnswersIsFetchedAgainOnAnotherNode(t *testing.T) { // 1820/AC2 - b := newBench(t) - b.ex.Toolchains = goEntries("ace", "g14") - // ace answers the fetch and the digest, then goes to sleep before the test. - test := b.steps.handlers["build_go_test"] - b.steps.handlers["build_go_test"] = func(c stepCall) (any, error) { - if c.node == "ace" { - return nil, fmt.Errorf("test on ace: %w", nats.ErrNoResponders) - } - return test(c) - } - v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict - if v.State != delivery.VerdictPass { - t.Fatalf("verdict = %+v, want a pass", v) - } - if got := nodesOf(b.steps.tool("fetch")); len(got) != 2 || got[1] != "g14" { - t.Errorf("fetch went to %v, want ace then g14 (the workspace was on ace)", got) - } - for _, c := range b.steps.tool("test") { - if c.node == "g14" && c.args["workspace"] == nil { - t.Errorf("a test on g14 has no workspace") - } - } - if len(v.Machines) != 1 || v.Machines[0] != "g14" { - t.Errorf("machines = %v, want [g14]", v.Machines) - } -} - -func TestWhenNoBuildNodeAnswersTheFailureNamesEveryNodeTried(t *testing.T) { // 1820/AC3 - b := newBench(t) - b.ex.Toolchains = goEntries("ace", "g14", "shanks") - deafNodes(b, "ace", "g14", "shanks") - v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict - if v.State != delivery.VerdictFail { - t.Fatalf("verdict = %+v, want a failure", v) - } - for _, n := range []string{"ace", "g14", "shanks"} { - if !strings.Contains(v.Failed.Words, n) { - t.Errorf("failure %q does not name %s", v.Failed.Words, n) - } - } - if !IsNodesFailure(v) { - t.Errorf("failure %q is not a failure of the nodes, so it would not be judged again when they change", v.Failed.Words) - } - if got := len(b.steps.tool("fetch")); got != 3 { - t.Errorf("fetch was tried %d times, want once on each of 3 nodes", got) - } -} - -func TestAStepThatFailedForTheChangeIsNotMovedToAnotherNode(t *testing.T) { // 1820/AC4 - b := newBench(t) - b.ex.Toolchains = goEntries("ace", "g14") - b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { return nil, fmt.Errorf("fetch on ace: the step failed: git exploded") } - v := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))).Verdict - if v.State != delivery.VerdictFail || len(b.steps.tool("fetch")) != 1 { - t.Errorf("verdict = %+v after %d fetches, want a failure after one", v, len(b.steps.tool("fetch"))) - } -} - -func TestACanceledCheckIsNotMovedToAnotherNode(t *testing.T) { // 1820/AC4 - b := newBench(t) - b.ex.Toolchains = goEntries("ace", "g14") - ctx, cancel := context.WithCancel(context.Background()) - b.steps.handlers["build_go_fetch"] = func(stepCall) (any, error) { - cancel() - return nil, fmt.Errorf("fetch on ace: %w", nats.ErrNoResponders) - } - _, _ = b.ex.Judge(ctx, caseOf(t, pullOf(repoA, 7, head1))) - if n := len(b.steps.tool("fetch")); n != 1 { - t.Errorf("fetch was tried %d times after the check ended, want 1", n) - } -} - +// Issue 1820 AC5: a read of the forge that nobody answers is the forge's failure, not a build node's. (AC1-AC4 moved a +// step to another node; since R4g an ask whose node stops is delivered again by its stream: ADR 0525 §6.) func TestAPinReadThatNobodyAnswersIsTheForgesFailureNotABuildNodes(t *testing.T) { // 1820/AC5 b := groupBench(t, "cb54756c3865") b.ex.Forge.(*fakeForge).err = fmt.Errorf("gitea.gitea_get_file: %w", nats.ErrNoResponders) diff --git a/modules/mesh-delivery-checker/internal/check/hold.go b/modules/mesh-delivery-checker/internal/check/hold.go deleted file mode 100644 index 5725a8d..0000000 --- a/modules/mesh-delivery-checker/internal/check/hold.go +++ /dev/null @@ -1,7 +0,0 @@ -package check - -// PlansBuildingBucket is the controller's bucket of the delivery plans in their build phase, one key for each plan, written -// by the controller alone (issue 1652): a plan key there means a delivery builds on the build nodes, and no new check starts -// meanwhile (ADR 0484 clause 12, ADR 0483 clause 6). The keys age out with no delete marker, so the keys are listed, not watched. The bucket may be absent: the controller's issue 1652 is delivered -// before R4c-5b, and if it is not, the bucket is read as empty. -const PlansBuildingBucket = "mesh-controller_plans-building" diff --git a/modules/mesh-delivery-checker/internal/check/hold_test.go b/modules/mesh-delivery-checker/internal/check/hold_test.go deleted file mode 100644 index 35f6ef1..0000000 --- a/modules/mesh-delivery-checker/internal/check/hold_test.go +++ /dev/null @@ -1,38 +0,0 @@ -package check - -import ( - "context" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -func TestARunningCheckIsNotStoppedWhenADeliveryStartsToBuild(t *testing.T) { // ADR 0484 clause 12, ADR 0483 clause 4 - ctx := context.Background() - held := false - j := &judgeFunc{f: func(_ context.Context, _ *Case, call int) (Outcome, error) { - if call == 1 { - held = true // a delivery starts while this check runs - } - return Outcome{Verdict: passing()}, nil - }} - r := newRig(t, j) - r.ch.Hold = func() bool { return held } - r.ch.Candidate(ctx, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1))) - r.ch.Wait() - if v := r.verdict(t, keyOf(t, "feat/x", candidateOf(t, "feat/x", pullOf(repoA, 7, head1)))); v.State != delivery.VerdictPass { - t.Errorf("verdict = %+v, want the running check to end as it was", v) - } - // A new candidate waits while the plan builds. - r.ch.Candidate(ctx, "feat/y", candidateOf(t, "feat/y", pullOf(repoA, 8, head2))) - r.ch.Wait() - if j.count() != 1 { - t.Errorf("judged %d times while a delivery builds, want only the one already running", j.count()) - } - held = false - r.ch.Reconsider(ctx) - r.ch.Wait() - if j.count() != 2 { - t.Errorf("judged %d times after the delivery ended, want the waiting candidate too", j.count()) - } -} diff --git a/modules/mesh-delivery-checker/internal/check/judge.go b/modules/mesh-delivery-checker/internal/check/judge.go index dcd07f0..a6f134c 100644 --- a/modules/mesh-delivery-checker/internal/check/judge.go +++ b/modules/mesh-delivery-checker/internal/check/judge.go @@ -3,6 +3,7 @@ package check import ( "context" "encoding/json" + "errors" "fmt" "path" "slices" @@ -10,6 +11,7 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/buildstep" "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) @@ -51,9 +53,11 @@ type verdictPtr = *delivery.Verdict func failed(machine, words string) verdictPtr { return fail(machine, words) } -// Steps runs a build module's long step on a node and answers its result (buildrun.Runner). -type Steps interface { - Run(ctx context.Context, module, node, tool string, args map[string]any) (json.RawMessage, error) +// Asks puts a check's ask on its build seat's stream and waits for its step (buildrun.Asker; ADR 0525 §1, §6): the +// checker chooses no node, starts no step and holds none; whichever build node pulls the ask runs the check. +type Asks interface { + Put(ctx context.Context, a delivery.StepAsk) error + Wait(ctx context.Context, a delivery.StepAsk) (delivery.Step, error) } // ToolchainEntries are the nodes' toolchain entries of a language (buildrun.ToolchainSet). @@ -67,24 +71,18 @@ type BuildModules interface { BuildModules(repository string) []string } -// toolPrefix is the start of each build module's tool names. -var toolPrefix = map[string]string{buildrun.GoBuildModule: "build_go_", buildrun.NodeBuildModule: "build_node_"} +// askLanguage is the build seat language of each build module the checker asks. +var askLanguage = map[string]string{buildrun.GoBuildModule: delivery.AskGo, buildrun.NodeBuildModule: delivery.AskNode} -// Examiner judges a case through the build modules (ADR 0484 clauses 3, 4, 10, 13): for each pull request it fetches the head -// merged onto its trunk head on a build node, then runs the unit level and the integration level, taking the kept verdicts -// of unchanged packages, running the change's own tests first and stopping at the first failure. +// Examiner judges a case through the build seats (ADR 0484 clauses 3, 4, 10, 13; ADR 0525 §1, §3): for each pull request +// and build module it puts one check ask whose job fetches the head merged onto its trunk head, then runs the unit level +// and the integration level, taking the kept verdicts of unchanged packages, running the change's own tests first and +// stopping at the first failure; it judges the step's answer as it judged the tools' answers. type Examiner struct { - Steps Steps + Asks Asks + // Toolchains are the nodes' toolchains: a kept verdict names the toolchain the nodes agree on. Toolchains ToolchainEntries - // Presence says which nodes are online; Health whether a node's build module is running and healthy. A node that is not - // is not chosen; nil means every node is. - Presence interface{ Online(node string) bool } - Health interface { - Healthy(node, module string) bool - } - // Places holds a step back while its node is at the checker's capacity (ADR 0484 clause 12). - Places *buildrun.Places - Kept Kept + Kept Kept // Previews is the previews bucket; nil writes none. Previews KV Modules BuildModules @@ -105,8 +103,18 @@ type Examiner struct { } // Trunks are the trunk heads of every repository, for the siblings a repository's tests read. Trunks interface{ Heads() map[string]string } - Now func() time.Time - Logf func(format string, args ...any) + // Compositions answers the assigner's composition check of a check by the check's key (delivery.CompositionKey), + // waiting until the assigner wrote a verdict that does not wait, or ctx ends (ADR 0528 clause 5, issue 1297); nil + // leaves the composition not run. + Compositions CompositionReader + Now func() time.Time + Logf func(format string, args ...any) +} + +// CompositionReader answers the assigner's composition of a check by the check's key, waiting until the assigner wrote +// a verdict that does not wait, or ctx ends. +type CompositionReader interface { + Await(ctx context.Context, key string) (delivery.CompositionVerdict, error) } func (j *Examiner) now() time.Time { @@ -132,22 +140,22 @@ func besideLines(refs []BesideRef) []string { return out } -// target is a pull request on one build module: the node chosen, the workspace its fetch made and what it said. +// target is a pull request on one build module: the node that pulled its ask, the workspace its fetch made and what it +// said. type target struct { pull Pull module string node string toolchain string fetched Fetched - // fetchArgs are the arguments of the target's fetch, so a workspace lost with its node is made again on another. + // fetchArgs are the arguments of the fetch its check ask runs. fetchArgs map[string]any // priority are the Go packages (import paths) or the Node module directories whose test files the merge changes. changed []string // beside are the lines ` ` of the siblings the workspace was given, sorted: its tests may read them, // so a verdict is kept or taken only under a digest that says it folded exactly these in (lead decision L5). beside []string - plans map[string]*goPlan // by level - // runner is RunnerWorkspace while the target's steps are asked to run the candidate (ADR 0535 §1), empty otherwise. + // runner is RunnerWorkspace when the target's ask runs the candidate's own build module (ADR 0535 §1), empty otherwise. runner string // judged is what the line says of the program that judged a change to the build module; empty for any other change. judged string @@ -177,18 +185,11 @@ func (t *target) stepArgs(args map[string]any) map[string]any { return args } -// goPlan is what a Go target has to run at one level: the digests, the packages without a kept verdict and the tests taken. -type goPlan struct { - digests map[string]string - todo []string - // all says the digest step gave nothing to key a kept verdict by, so the level runs whole and keeps nothing. - all bool -} - // acc collects a level's counts across targets. type acc struct { run, kept, skipped int elapsed, setup float64 + lint float64 over bool slowest []TestTime } @@ -200,7 +201,6 @@ type state struct { targets []*target levels map[string]*acc nodes map[string]bool - deaf map[string][]deafNode // by build module: the nodes that did not answer during this judging clone time.Duration slow []string } @@ -209,34 +209,42 @@ type state struct { // be judged at all. func (j *Examiner) Judge(ctx context.Context, c *Case) (Outcome, error) { st := &state{j: j, c: c, levels: map[string]*acc{delivery.LevelUnit: {}, delivery.LevelIntegration: {}}, nodes: map[string]bool{}} - if v := st.fetchAll(ctx); v != nil { + if v := st.judgeAll(ctx); v != nil { return st.finish(ctx, *v), nil } - if v := st.checkPins(ctx); v != nil { - return st.finish(ctx, *v), nil - } - for _, level := range []string{delivery.LevelUnit, delivery.LevelIntegration} { - if c.Try != nil && c.Try.Level != "" && c.Try.Level != level { - continue // a try of one level runs that level alone - } - for _, priority := range []bool{true, false} { - for _, t := range st.targets { - if v := st.runTarget(ctx, t, level, priority); v != nil { - return st.finish(ctx, *v), nil - } - } - } - } return st.finish(ctx, delivery.Verdict{State: delivery.VerdictPass}), nil } -// finish is the outcome of the verdict with the state of the builds the candidate's modules carry. +// finish is the outcome of the verdict with the state of the builds the candidate's modules carry, and, for a check +// whose tests passed, the assigner's composition of it. func (st *state) finish(ctx context.Context, v delivery.Verdict) Outcome { out := st.outcome(v) + if out.Verdict.State == delivery.VerdictPass { + st.composed(ctx, &out.Verdict) + } out.Builds = st.j.buildNotes(ctx, st.c) return out } +// composed reads the assigner's composition of the check into its verdict (ADR 0528 clause 5): a failed composition +// fails the check, naming the machine it failed at. A try is not composed (it has no candidate); a composition that +// cannot be had leaves it not run, said; with no assigner running it is not judged, said at once. +func (st *state) composed(ctx context.Context, v *delivery.Verdict) { + if st.j.Compositions == nil || st.c.Try != nil { + return + } + comp, err := st.j.Compositions.Await(ctx, st.c.Key) + switch { + case errors.Is(err, ErrAssignerNotRunning): + v.Composition = &delivery.Composition{State: delivery.CompositionNotRun, Words: delivery.AssignerNotRunning} + return + case err != nil: + st.j.logf("%s: the composition could not be read, so it is said as not run: %v", st.c.Key, err) + return + } + withComposition(v, comp) +} + // outcome fills the verdict's levels, times, machines and composition from what ran. func (st *state) outcome(v delivery.Verdict) Outcome { if v.State == delivery.VerdictPass || len(v.Levels) == 0 { @@ -247,7 +255,7 @@ func (st *state) outcome(v delivery.Verdict) Outcome { } ms := func(s float64) int64 { return int64(s * 1000) } u, i := st.levels[delivery.LevelUnit], st.levels[delivery.LevelIntegration] - v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)} + v.Took = delivery.Took{Clone: st.clone.Milliseconds(), Unit: ms(u.elapsed), Lint: ms(u.lint), IntegrationSetup: ms(i.setup), IntegrationRun: ms(i.elapsed)} for node := range st.nodes { v.Machines = append(v.Machines, node) } @@ -291,55 +299,18 @@ func fail(machine, words string) *delivery.Verdict { return &v } -// step runs one step of a build module on the node the target is on, holding a place there while it runs. A node that does -// not answer (the bus has no responder, or none answered) is left out and the step goes to another node of the same build -// module; a step after the fetch makes its workspace again there first (issue 1820). -func (st *state) step(ctx context.Context, t *target, tool, label string, args map[string]any) (json.RawMessage, error) { - for { - raw, err := st.stepOnce(ctx, t, tool, label, args) - if err == nil || ctx.Err() != nil || !isUnheard(err) { - return raw, err - } - if merr := st.moveStep(t, err); merr != nil { - return nil, merr - } - if tool != "fetch" { - if rerr := st.refetch(ctx, t); rerr != nil { - return nil, rerr - } - stepOnWorkspace(t, args) - } +// levels are the levels a case's asks run: both, or the one a try names. +func (st *state) levelsOf() []string { + if st.c.Try != nil && st.c.Try.Level != "" { + return []string{st.c.Try.Level} // a try of one level runs that level alone } + return []string{delivery.LevelUnit, delivery.LevelIntegration} } -func (st *state) stepOnce(ctx context.Context, t *target, tool, label string, args map[string]any) (json.RawMessage, error) { - st.c.Progress.Set(label, t.node) - release, err := st.j.Places.Acquire(ctx, t.node) - if err != nil { - return nil, err - } - defer release() - return st.j.Steps.Run(ctx, t.module, t.node, toolPrefix[t.module]+tool, args) -} - -// choose picks the build node for a pull request's repository on a build module. -func (st *state) choose(bm string) (node, toolchain string, err error) { - language := buildrun.LanguageOf[bm] - qualifies := func(node string) bool { - return !st.isDeaf(bm, node) && (st.j.Presence == nil || st.j.Presence.Online(node)) && (st.j.Health == nil || st.j.Health.Healthy(node, bm)) - } - nodes, tc, err := buildrun.ChooseNodes(language, st.j.Toolchains.Entries(language), st.j.Places, qualifies, 1) - if err != nil { - return "", "", err - } - return nodes[0], tc.Version, nil -} - -// fetchAll fetches every pull request's head merged onto its trunk on a build node of each build module the repository -// needs. A head that does not merge, or a step that fails, ends the check at once, running no test (ADR 0484 clause 13). -func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { - began := st.j.now() - defer func() { st.clone = st.j.now().Sub(began) }() +// judgeAll puts one check ask per pull request and build module the repository needs, the mesh-sdk member first, and +// judges each answer. A head that does not merge, a pin that is not allowed or a level that fails ends the check at once +// (ADR 0484 clause 13). +func (st *state) judgeAll(ctx context.Context) *delivery.Verdict { // The mesh-sdk member first: whether its merge changes the TypeScript package's paths decides whether the other members' // Node fetches take the group's package (ADR 0500 §5). pulls := slices.Clone(st.c.Pulls) @@ -356,47 +327,32 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { for _, p := range pulls { var merged []Fetched for _, bm := range st.j.Modules.BuildModules(p.Repository) { - node, tc, err := st.choose(bm) - if err != nil { - return fail(bm, err.Error()) + t := &target{pull: p, module: bm} + if tc, none := delivery.AgreedToolchain(st.j.Toolchains.Entries(buildrun.LanguageOf[bm])); none == "" { + t.toolchain = tc.Version } - t := &target{pull: p, module: bm, node: node, toolchain: tc, plans: map[string]*goPlan{}} - st.nodes[node] = true args := map[string]any{"repository": p.Repository, "commit": p.Head, "ref": refOf(p.Number), "base": p.Trunk} - t.fetchArgs = args if bm == buildrun.NodeBuildModule && groupPackage != nil && p.Repository != SDKRepository { args["package"] = groupPackage } if bm == buildrun.GoBuildModule { beside, err := st.j.besideFor(ctx, st.c, p) if err != nil { - return fail(forgeMachine(err, node), err.Error()+unheardWords(err)) + return fail(forgeMachine(err, bm), err.Error()+unheardWords(err)) } if len(beside) > 0 { args["beside"] = beside t.beside = besideLines(beside) } - } - raw, err := st.step(ctx, t, "fetch", fmt.Sprintf("fetch %s#%d on %s", p.Repository, p.Number, node), args) - node = t.node // the step may have moved to another node - if err != nil { - return fail(node, fmt.Sprintf("fetch of %s#%d: %v%s", p.Repository, p.Number, err, pinHint(st.c, err.Error()))) - } - if err := json.Unmarshal(raw, &t.fetched); err != nil || t.fetched.Workspace == "" { - return fail(node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, raw)) - } - if len(t.fetched.Conflicts) > 0 { - if st.c.Try == nil { - st.j.writePreview(ctx, p, append(merged, t.fetched)) + if p.Repository == catalogueRepository { + t.runner = RunnerWorkspace // used by the build module when the merge changes it (ADR 0535 §1) } - return fail(node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", "))) } - t.changed = t.fetched.Changed - if bm == buildrun.GoBuildModule && changesBuildGo(p.Repository, t.changed) { - t.runner = RunnerWorkspace - } - merged = append(merged, t.fetched) + t.fetchArgs = args st.targets = append(st.targets, t) + if v := st.ask(ctx, t, &merged); v != nil { + return v + } } if st.c.Try == nil { // a try owes the indexer no preview: no pull request holds its head st.j.writePreview(ctx, p, merged) @@ -408,6 +364,139 @@ func (st *state) fetchAll(ctx context.Context) *delivery.Verdict { return nil } +// ask puts the target's check ask, waits for the step of the node that pulled it and judges its answer: the fetch, the +// pins of its go.mod files, then each level's digest, kept verdicts and tests. +func (st *state) ask(ctx context.Context, t *target, merged *[]Fetched) *delivery.Verdict { + p := t.pull + job := &delivery.CheckJob{Fetch: t.fetchArgs, Levels: st.levelsOf(), Kept: true} + if t.runner != "" { + job.Runner, job.RunnerWhen = t.runner, buildGoDir + } + if st.c.Try != nil && st.c.Try.Run != "" { + job.Run, job.Kept = st.c.Try.Run, false // a try of some tests runs them alone: a partial run proves no package + } + kind := delivery.AskCheck + if st.c.Kind == KindTrunk { + kind = delivery.AskRecheck // a re-judge after a trunk move goes after every new head's check (ADR 0536 §4) + } + a := delivery.StepAsk{ID: buildstep.NewID(), Kind: kind, Language: askLanguage[t.module], Asker: delivery.CheckerModule, + At: st.j.now().UTC(), Check: job} + st.c.Progress.Set(fmt.Sprintf("the %s ask %s of %s#%d", kind, a.ID, p.Repository, p.Number), "") + if err := st.j.Asks.Put(ctx, a); err != nil { + return fail(t.module, fmt.Sprintf("the check of %s#%d could not be asked: %v", p.Repository, p.Number, err)) + } + step, err := st.j.Asks.Wait(ctx, a) + if err != nil { + if ctx.Err() != nil { + return fail(t.module, ctx.Err().Error()) + } + return fail(t.module, fmt.Sprintf("the check of %s#%d: %v", p.Repository, p.Number, err)) + } + // The fetch and the digests are what of the ask's time its levels' running did not take (Took.Clone). + took := time.Duration(0) + if !step.Started.IsZero() && step.Ended.After(step.Started) { + took = step.Ended.Sub(step.Started) + } + t.node = step.Node + st.nodes[step.Node] = true + st.c.Progress.Set(fmt.Sprintf("judge the check of %s#%d", p.Repository, p.Number), step.Node) + if step.State != delivery.StepPassed { + return fail(step.Node, fmt.Sprintf("the check of %s#%d %s: %s%s", p.Repository, p.Number, step.State, step.Reason, pinHint(st.c, step.Reason))) + } + var answer delivery.CheckAnswer + if err := json.Unmarshal(step.Result, &answer); err != nil { + return fail(step.Node, fmt.Sprintf("the check of %s#%d answered what is not a check: %s", p.Repository, p.Number, step.Result)) + } + if err := json.Unmarshal(answer.Fetch, &t.fetched); err != nil || t.fetched.Workspace == "" { + return fail(step.Node, fmt.Sprintf("the fetch of %s#%d answered no workspace: %s", p.Repository, p.Number, answer.Fetch)) + } + if len(t.fetched.Conflicts) > 0 { + if st.c.Try == nil { + st.j.writePreview(ctx, p, append(*merged, t.fetched)) + } + return fail(step.Node, fmt.Sprintf("%s#%d does not merge into the trunk: files in conflict: %s", p.Repository, p.Number, strings.Join(t.fetched.Conflicts, ", "))) + } + t.changed = t.fetched.Changed + if !changesBuildGo(p.Repository, t.changed) { + t.runner = "" // the build module used no runner: the merge does not change it + } + *merged = append(*merged, t.fetched) + if v := st.checkPinsOf(ctx, t); v != nil { + return v + } + ran := 0.0 + for _, la := range answer.Levels { + v, elapsed := st.level(ctx, t, la) + ran += elapsed + if v != nil { + st.clone += max(0, took-time.Duration(ran*float64(time.Second))) + return v + } + } + st.clone += max(0, took-time.Duration(ran*float64(time.Second))) + return nil +} + +// level judges one level of a check ask's answer as the tools' answers were judged: the digest keys the kept verdicts +// of the packages that passed, the kept ones taken are counted, and the first test result that did not pass is the +// verdict. It answers the level's running time (its tests' and setup's). +func (st *state) level(ctx context.Context, t *target, la delivery.LevelAnswer) (*delivery.Verdict, float64) { + level := la.Level + a := st.levels[level] + if a == nil { + return fail(t.node, fmt.Sprintf("the check of %s#%d answered the level %q, which is no level", t.pull.Repository, t.pull.Number, level)), 0 + } + var digests map[string]string + if t.module == buildrun.GoBuildModule && len(la.Digest) > 0 { + var d Digested + if err := json.Unmarshal(la.Digest, &d); err == nil { + digests = d.Packages + if t.runner != "" { + t.judgedBy(d.JudgedBy) // the installed module judged it: the runner is not used again + } + } + } + if t.runner != "" && la.Judged != "" { + t.judgedBy(la.Judged) // the candidate judged the level: nothing is kept (ADR 0535 §4) + } + for _, k := range la.Kept { + a.kept += k.Tests + } + elapsed := 0.0 + for _, raw := range la.Tests { + var res Tested + if err := json.Unmarshal(raw, &res); err != nil { + return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)), elapsed + } + if t.runner != "" { + t.judgedBy(res.JudgedBy) + } + elapsed += res.Elapsed + res.Setup + res.Lint + if t.module == buildrun.NodeBuildModule { + if !res.NoTests { + a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10) + } + } else { + a.run += res.Tests + a.skipped += res.Skipped + a.slowest = append(a.slowest, res.Slowest...) + } + a.elapsed += res.Elapsed + a.setup += res.Setup + a.lint += res.Lint + a.over = a.over || res.OverBound + if len(digests) > 0 { + if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, digests, st.j.now()); err != nil { + st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err) + } + } + if !res.Passed { + return levelFailure(t, level, res), elapsed + } + } + return nil, elapsed +} + // changesPackage says whether the merge of a pull request changes a file inside the named paths of one of its repository's // TypeScript packages: a change of the manifest alone, or of files outside those paths, does not (ADR 0500 §5). func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool { @@ -428,51 +517,6 @@ func (j *Examiner) changesPackage(p Pull, merged []Fetched) bool { return false } -// isTestFile says whether a repository-relative path is a test file of the languages the checker judges: Go's _test.go and -// the TypeScript module's .test. and .spec. files and test directories. -func isTestFile(p string) bool { - switch { - case strings.HasSuffix(p, "_test.go"): - return true - case strings.Contains(p, ".test.") || strings.Contains(p, ".spec."): - return true - } - for _, seg := range strings.Split(path.Dir(p), "/") { - if seg == "__tests__" || seg == "test" || seg == "tests" { - return true - } - } - return false -} - -// changesTestOf says whether the merge changes a test file of the Go package with the import path pkg: the directory of the -// file and the import path agree on their last segments, whichever holds the module's directory and path in front. -func changesTestOf(changed []string, pkg string) bool { - for _, f := range changed { - if !strings.HasSuffix(f, "_test.go") { - continue - } - dir := path.Dir(f) - if dir == pkg || strings.HasSuffix(pkg, "/"+dir) || strings.HasSuffix(dir, "/"+pkg) { - return true - } - } - return false -} - -// changesTestIn says whether the merge changes a test file inside the directory of a Node module. -func changesTestIn(changed []string, dir string) bool { - for _, f := range changed { - if !isTestFile(f) || strings.HasSuffix(f, "_test.go") { - continue - } - if dir == "." || dir == "" || strings.HasPrefix(f, strings.TrimSuffix(dir, "/")+"/") { - return true - } - } - return false -} - // failedName names a failure: its package and test. func failedName(f Failure) string { if f.Test == "" { @@ -514,161 +558,3 @@ func levelFailure(t *target, level string, res Tested) *delivery.Verdict { } return &v } - -// runTarget runs one level of a target: first the packages the change's own tests are in (priority), then the others. -func (st *state) runTarget(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict { - if t.module == buildrun.NodeBuildModule { - return st.runNode(ctx, t, level, priority) - } - return st.runGo(ctx, t, level, priority) -} - -// A level's work is planned once for a target: the digest of each package, the kept verdicts taken, the packages left to run. -func (st *state) planGo(ctx context.Context, t *target, level string) *delivery.Verdict { - if t.plans[level] != nil { - return nil - } - plan := &goPlan{digests: map[string]string{}} - t.plans[level] = plan - if st.c.Try != nil && st.c.Try.Run != "" { - // A try of some tests runs them alone: a partial run proves no package, so nothing is taken or kept. - plan.all = true - return nil - } - raw, err := st.step(ctx, t, "digest", fmt.Sprintf("digest the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), - t.stepArgs(map[string]any{"workspace": t.fetched.Workspace, "level": level})) - var d Digested - if err == nil { - err = json.Unmarshal(raw, &d) - } - if ctx.Err() != nil { - return fail(t.node, ctx.Err().Error()) - } - if err == nil && t.runner != "" && t.judgedBy(d.JudgedBy) { - // The candidate's digest decides which verdicts would be taken: every package runs and nothing is kept (ADR 0535 §4). - plan.all = true - return nil - } - if err != nil || len(d.Packages) == 0 { - // The build module named no digest (a level it cannot digest here): the level runs whole and keeps nothing. - if err != nil { - st.j.logf("%s#%d: no digest of the %s level, so it runs whole: %v", t.pull.Repository, t.pull.Number, level, err) - } - plan.all = true - return nil - } - if !slices.Equal(d.Beside, t.beside) { - // The siblings the tests read are not all in the digest (a build module before DigestVersion 3, or another set): a - // package whose tests read one is never kept, nor taken from a verdict kept for other inputs (ADR 0444, L5). - st.j.logf("%s#%d: the %s level's digest covers the siblings %v, not %v, so it runs whole and keeps nothing", t.pull.Repository, t.pull.Number, level, d.Beside, t.beside) - plan.all = true - return nil - } - plan.digests = d.Packages - a := st.levels[level] - pkgs := make([]string, 0, len(d.Packages)) - for p := range d.Packages { - pkgs = append(pkgs, p) - } - sort.Strings(pkgs) - for _, p := range pkgs { - k, found, err := st.j.Kept.Lookup(ctx, level, d.Packages[p]) - if err != nil { - st.j.logf("the kept verdict of %s at %s cannot be read, so it runs: %v", p, level, err) - } - if found { - a.kept += k.Tests - continue - } - plan.todo = append(plan.todo, p) - } - return nil -} - -func (st *state) runGo(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict { - if v := st.planGo(ctx, t, level); v != nil { - return v - } - plan := t.plans[level] - var run []string - switch { - case plan.all: - if priority { - return nil // nothing to put first: the level runs whole after - } - default: - for _, p := range plan.todo { - if changesTestOf(t.changed, p) == priority { - run = append(run, p) - } - } - if len(run) == 0 { - return nil - } - } - args := t.stepArgs(map[string]any{"workspace": t.fetched.Workspace, "level": level}) - if run != nil { - args["packages"] = run - } - if st.c.Try != nil && st.c.Try.Run != "" { - args["run"] = st.c.Try.Run // build_go_test's go test -run pattern - } - raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d on %s", level, t.pull.Repository, t.pull.Number, t.node), args) - if err != nil { - return fail(t.node, fmt.Sprintf("the %s level of %s#%d: %v", level, t.pull.Repository, t.pull.Number, err)) - } - var res Tested - if err := json.Unmarshal(raw, &res); err != nil { - return fail(t.node, fmt.Sprintf("the %s level of %s#%d answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, err)) - } - if t.runner != "" { - t.judgedBy(res.JudgedBy) - } - a := st.levels[level] - a.run += res.Tests - a.skipped += res.Skipped - a.elapsed += res.Elapsed - a.setup += res.Setup - a.over = a.over || res.OverBound - a.slowest = append(a.slowest, res.Slowest...) - if len(plan.digests) > 0 { - if _, err := st.j.Kept.KeepPasses(ctx, t.pull.Repository, level, t.toolchain, res, plan.digests, st.j.now()); err != nil { - st.j.logf("%s#%d: a passing package could not be kept: %v", t.pull.Repository, t.pull.Number, err) - } - } - if !res.Passed { - return levelFailure(t, level, res) - } - return nil -} - -func (st *state) runNode(ctx context.Context, t *target, level string, priority bool) *delivery.Verdict { - for _, dir := range t.fetched.Modules { - if changesTestIn(t.changed, dir) != priority { - continue - } - raw, err := st.step(ctx, t, "test", fmt.Sprintf("test the %s level of %s#%d (%s) on %s", level, t.pull.Repository, t.pull.Number, dir, t.node), - map[string]any{"workspace": t.fetched.Workspace, "module": dir, "level": level}) - if err != nil { - return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s): %v", level, t.pull.Repository, t.pull.Number, dir, err)) - } - var res Tested - if err := json.Unmarshal(raw, &res); err != nil { - return fail(t.node, fmt.Sprintf("the %s level of %s#%d (%s) answered what is not a result: %v", level, t.pull.Repository, t.pull.Number, dir, err)) - } - if res.Module == "" { - res.Module = dir - } - a := st.levels[level] - if !res.NoTests { - a.run++ // a Node package is counted whole: its build module counts no tests (ADR 0484 clause 10) - } - a.elapsed += res.Elapsed - a.setup += res.Setup - a.over = a.over || res.OverBound - if !res.Passed { - return levelFailure(t, level, res) - } - } - return nil -} diff --git a/modules/mesh-delivery-checker/internal/check/judged_test.go b/modules/mesh-delivery-checker/internal/check/judged_test.go index f90febd..e915b39 100644 --- a/modules/mesh-delivery-checker/internal/check/judged_test.go +++ b/modules/mesh-delivery-checker/internal/check/judged_test.go @@ -30,7 +30,7 @@ func candidateBench(t *testing.T, takesRunner bool) *bench { return "installed" } b.steps.handlers["build_go_digest"] = func(c stepCall) (any, error) { - d := b.digest[c.args["level"].(string)] + d := echoBeside(b, b.digest[c.args["level"].(string)]) d.JudgedBy = judged(c) return d, nil } diff --git a/modules/mesh-delivery-checker/internal/check/line.go b/modules/mesh-delivery-checker/internal/check/line.go index 85e6830..3e5afc0 100644 --- a/modules/mesh-delivery-checker/internal/check/line.go +++ b/modules/mesh-delivery-checker/internal/check/line.go @@ -99,6 +99,9 @@ func compositionWords(c *delivery.Composition, took int64) string { return compo // composition is compositionWords; short counts the machines and leaves the SDK commit to the report. func composition(c *delivery.Composition, took int64, short bool) string { + if c != nil && c.State == delivery.CompositionNotRun && c.Words != "" { + return "composition not judged: " + c.Words + } if c == nil || c.State == delivery.CompositionNotRun { return "composition not yet run" } @@ -176,7 +179,7 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s for _, x := range []struct { name string ms int64 - }{{"queue", t.Queue}, {"clone", t.Clone}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}} { + }{{"queue", t.Queue}, {"clone", t.Clone}, {"lint", t.Lint}, {"unit", t.Unit}, {"int setup", t.IntegrationSetup}, {"int run", t.IntegrationRun}} { if x.ms > 0 { times = append(times, x.name+" "+seconds(x.ms)) } @@ -189,6 +192,9 @@ func parts(v delivery.Verdict, cs *Case, p Pull, builds []BuildNote, slowest []s if t.Clone > 0 { short = append(short, "clone "+seconds(t.Clone)) } + if t.Lint > 0 { + short = append(short, "lint "+seconds(t.Lint)) + } if t.Unit > 0 { short = append(short, "unit "+seconds(t.Unit)) } diff --git a/modules/mesh-delivery-checker/internal/check/line_test.go b/modules/mesh-delivery-checker/internal/check/line_test.go index 9f5824f..ce088ff 100644 --- a/modules/mesh-delivery-checker/internal/check/line_test.go +++ b/modules/mesh-delivery-checker/internal/check/line_test.go @@ -154,3 +154,26 @@ func TestTheCompositionsTimeAndMachinesAreSaidOnceTheAssignerReportsThem(t *test t.Errorf("failed composition report = %q", r) } } + +// 1887/D7-go, review round 2 item 5: the line says the unit level's gofmt and go vet time beside its tests'. +func TestTheLineSaysTheLintTime(t *testing.T) { + b := newBench(t) + test := b.test + b.test = func(c stepCall) (any, error) { + r, err := test(c) + if res, ok := r.(Tested); ok && res.Level == delivery.LevelUnit { + res.Lint = 1.5 + return res, err + } + return r, err + } + out := b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + if out.Verdict.Took.Lint != 1500 { + t.Fatalf("took = %+v, want the lint's 1.5 s", out.Verdict.Took) + } + cs := caseOf(t, pullOf(repoA, 7, head1)) + pc := parts(out.Verdict, cs, cs.Pulls[0], out.Builds, out.Slowest, out.Judged) + if !strings.Contains(pc.times, "lint 1.5s") || !strings.Contains(pc.shortTimes, "lint 1.5s") { + t.Errorf("times = %q, short %q, want the lint's time in both forms of the line", pc.times, pc.shortTimes) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/pins.go b/modules/mesh-delivery-checker/internal/check/pins.go index f1e2937..c07a8c4 100644 --- a/modules/mesh-delivery-checker/internal/check/pins.go +++ b/modules/mesh-delivery-checker/internal/check/pins.go @@ -55,19 +55,19 @@ func SDKPin(goMod string) (commit string, ok bool) { return "", false } -// checkPins refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the +// checkPinsOf refuses a Go member whose go.mod names a mesh-sdk commit that is neither on mesh-sdk's trunk nor the head of the // group's mesh-sdk pull request. It reads each go.mod the fetch named from the forge: the one the check builds, which is the // head's when the head holds the file and the trunk's when it does not, since a module added on the trunk after the branch was // cut is in the merge and not at the head (issue 1845). It answers nil when every pin is the group's. -func (st *state) checkPins(ctx context.Context) verdictPtr { +func (st *state) checkPinsOf(ctx context.Context, t *target) verdictPtr { if st.j.Forge == nil { return nil } sdk := st.c.member(SDKRepository) - for _, t := range st.targets { - if t.module != goModule || t.pull.Repository == SDKRepository { - continue - } + if t.module != goModule || t.pull.Repository == SDKRepository { + return nil + } + { for _, dir := range t.fetched.Modules { file := path.Join(dir, "go.mod") st.c.Progress.Set(fmt.Sprintf("read %s of %s#%d", file, t.pull.Repository, t.pull.Number), t.node) diff --git a/modules/mesh-delivery-checker/internal/check/pins_test.go b/modules/mesh-delivery-checker/internal/check/pins_test.go index 703a11b..c9c2bb1 100644 --- a/modules/mesh-delivery-checker/internal/check/pins_test.go +++ b/modules/mesh-delivery-checker/internal/check/pins_test.go @@ -97,7 +97,7 @@ func TestAMemberPinnedToTheSDKMembersHeadPassesAndIsBuiltAgainstIt(t *testing.T) var besideSDK bool for _, f := range b.steps.tool("fetch") { if f.args["repository"] == repoA { - refs, _ := f.args["beside"].([]BesideRef) + refs, _ := refsOf(f.args["beside"]), true for _, r := range refs { besideSDK = besideSDK || (r.Repository == repoSDK && r.Commit == head2) } @@ -116,9 +116,8 @@ func TestAMemberPinnedToAnOlderHeadOfTheSDKMemberFailsNamingTheMemberAndTheCommi !strings.Contains(f.Words, "neither on mesh-sdk's trunk nor the group's head") || !strings.Contains(f.Words, short12(head2)) { t.Fatalf("verdict = %+v", v) } - if n := len(b.steps.tool("test")); n != 0 { - t.Errorf("%d test steps ran after a stale pin", n) - } + // Since R4g the levels run in the member's one check ask before its go.mod pins are judged (ADR 0525 §3); the verdict is + // still the pin's, whatever the levels said. } func TestAMemberPinnedToACommitOnTheTrunkPasses(t *testing.T) { // 1624/AC2, AC4 @@ -213,7 +212,7 @@ func besideOf(t *testing.T, b *bench, repository string) map[string]BesideRef { out := map[string]BesideRef{} for _, f := range b.steps.tool("fetch") { if f.args["repository"] == repository { - refs, _ := f.args["beside"].([]BesideRef) + refs, _ := refsOf(f.args["beside"]), true for _, r := range refs { out[r.Repository] = r } @@ -228,7 +227,8 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR b := newBench(t) b.ex.Trunks = trunksOf(t, map[string]string{controller: trunkA, catalogue: trunkS, host: trunkA2, SDKRepository: "9999999999999999999999999999999999999999"}) b.ex.Runs = runsAt{"mesh-host": hostRuns} - forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865")}, onTrunk: map[string]bool{"cb54756c3865": true}} + forge := &fakeForge{goMods: map[string]string{controller + " go.mod " + head1: capturedGoMod(t, "cb54756c3865"), + catalogue + " go.mod " + head2: "module git.novox.be/novox/mesh-catalog\n"}, onTrunk: map[string]bool{"cb54756c3865": true}} b.ex.Forge = forge // A group of the controller and the catalogue: the controller's tests read the catalogue's trunk, not the group's head. b.judge(t, caseOf(t, pullOf(controller, 7, head1), pullOf(catalogue, 12, head2))) @@ -244,6 +244,39 @@ func TestTheSiblingsAreWhereTheTestsStateThem(t *testing.T) { // review D2, ADR } } +// 1295/R4g: the node-engine's tests read the controller's manifest, so a mesh-host pull request has mesh-controller beside it: +// at the group member's head when the group holds one, else at its trunk. +func hostCase(t *testing.T, pulls ...delivery.CandidatePull) *Case { + t.Helper() + c, err := delivery.DecodeCandidate(candidateOf(t, "feat/x", pulls...)) + if err != nil { + t.Fatal(err) + } + cs := CaseOf(c, map[string]string{"novox/mesh-host": trunkA, controllerRepo: trunkA2}) + if cs == nil { + t.Fatal("no case") + } + return cs +} + +func TestTheNodeEnginesTestsHaveTheControllerBesideThem(t *testing.T) { + const host = "novox/mesh-host" + b := newBench(t) + b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS}) + b.judge(t, hostCase(t, pullOf(host, 5, head1))) + if c := besideOf(t, b, host)[controllerRepo]; c.Commit != trunkA2 || c.Ref != "" { + t.Errorf("the controller beside the node-engine alone = %+v, want its trunk %s", c, trunkA2) + } + b = newBench(t) + b.ex.Trunks = trunksOf(t, map[string]string{host: trunkA, controllerRepo: trunkA2, catalogueRepository: trunkS, SDKRepository: trunkS}) + b.ex.Forge = &fakeForge{goMods: map[string]string{controllerRepo + " go.mod " + head2: capturedGoMod(t, "cb54756c3865")}, + onTrunk: map[string]bool{"cb54756c3865": true}} + b.judge(t, hostCase(t, pullOf(host, 5, head1), pullOf(controllerRepo, 9, head2))) + if c := besideOf(t, b, host)[controllerRepo]; c.Commit != head2 || c.Ref != "refs/pull/9/head" { + t.Errorf("the controller beside the node-engine in a group = %+v, want the member's head %s", c, head2) + } +} + func TestThePinOfTheControllerThatCannotBeReadFailsTheCheck(t *testing.T) { // review D2 b := newBench(t) b.ex.Trunks = trunksOf(t, map[string]string{"novox/mesh-controller": trunkA}) @@ -265,3 +298,51 @@ func TestAForgeThatIsNotHeardFailsAtTheForgeAndIsJudgedAgainWhenTheNodesChange(t t.Errorf("a forge that was not heard is not the change's failure: %+v is not judged again when the nodes change", v.Failed) } } + +// 1887/A12: the change's own repository is placed beside it at the trunk it is merged onto, for every repository, so a +// shrink-only list test reads the trunk's copy at $MESH_CHECK_BESIDE/. +func TestTheChangesOwnRepositoryIsBesideItAtItsTrunk(t *testing.T) { + b := newBench(t) + b.judge(t, caseOf(t, pullOf(repoA, 7, head1))) + got := besideOf(t, b, repoA) + if own := got[repoA]; own.Commit != trunkA || own.Ref != "" { + t.Fatalf("the repository beside its own change = %+v, want its trunk %s", own, trunkA) + } +} + +// 1887/A12: a catalogue change's siblings hold the catalogue itself at the trunk it is merged onto, beside mesh-sdk and +// mesh-controller, so the catalogue's shrink-only list tests read $MESH_CHECK_BESIDE/mesh-catalog. +func TestACatalogueChangeHasTheCatalogueBesideItAtItsTrunk(t *testing.T) { + b := newBench(t) + b.ex.Trunks = trunksOf(t, map[string]string{catalogueRepository: trunkA, SDKRepository: trunkS, controllerRepo: trunkA2}) + b.judge(t, caseOf(t, pullOf(catalogueRepository, 12, head1))) + got := besideOf(t, b, catalogueRepository) + if c := got[catalogueRepository]; c.Commit != trunkA || c.Ref != "" { + t.Fatalf("the catalogue beside its own change = %+v, want its trunk %s", c, trunkA) + } + if _, ok := got[SDKRepository]; !ok { + t.Errorf("mesh-sdk is no longer beside a catalogue change: %v", got) + } +} + +// 1887/A12: a mesh-lab head gets a check: with its trunk known it is judged like any repository, its Go fetch finding +// the nested modules itself. +func TestAMeshLabHeadGetsACheck(t *testing.T) { + const lab = "novox/mesh-lab" + raw := candidateOf(t, "fix/x", pullOf(lab, 3, head1)) + c, err := delivery.DecodeCandidate(raw) + if err != nil { + t.Fatal(err) + } + cs := CaseOf(c, map[string]string{lab: trunkA}) + if cs == nil || len(cs.Pulls) != 1 || cs.Pulls[0].Repository != lab || len(cs.Left) != 0 { + t.Fatalf("the case = %+v, want mesh-lab#3 judged", cs) + } + b := newBench(t) + if v := b.judge(t, cs).Verdict; v.State != delivery.VerdictPass { + t.Fatalf("verdict = %+v", v) + } + if fetched := b.steps.tool("fetch"); len(fetched) == 0 || fetched[0].args["repository"] != lab { + t.Fatalf("mesh-lab was not fetched: %+v", fetched) + } +} diff --git a/modules/mesh-delivery-checker/internal/check/rank_test.go b/modules/mesh-delivery-checker/internal/check/rank_test.go index 0de4329..801fe8e 100644 --- a/modules/mesh-delivery-checker/internal/check/rank_test.go +++ b/modules/mesh-delivery-checker/internal/check/rank_test.go @@ -3,10 +3,8 @@ package check import ( "context" "fmt" - "runtime" "sync" "testing" - "time" "git.novox.be/novox/mesh-sdk/go/delivery" ) @@ -27,146 +25,8 @@ type rankRig struct { errs []error // ctx errors the held check saw when released } -func newRankRig(t *testing.T) *rankRig { - rr := &rankRig{held: make(chan struct{}), release: make(chan struct{})} - j := &judgeFunc{f: func(ctx context.Context, c *Case, _ int) (Outcome, error) { - rr.mu.Lock() - rr.started = append(rr.started, c) - hold := rr.hold - rr.hold = false - rr.mu.Unlock() - if hold { - close(rr.held) - <-rr.release - rr.mu.Lock() - rr.errs = append(rr.errs, ctx.Err()) - rr.mu.Unlock() - } - return Outcome{Verdict: passing()}, nil - }} - rr.rig = newRig(t, j) - rr.ch.Limit = 1 - now := time.Date(2026, 10, 23, 14, 0, 0, 0, time.UTC) - var clock sync.Mutex - rr.ch.Now = func() time.Time { - clock.Lock() - defer clock.Unlock() - now = now.Add(time.Second) - return now - } - return rr -} - func group(n int) string { return fmt.Sprintf("feat/g%d", n) } -// judged puts n candidates in and judges them on the current trunk. -func (rr *rankRig) judged(t *testing.T, n int) { - for i := 1; i <= n; i++ { - rr.ch.Candidate(context.Background(), group(i), candidateOf(t, group(i), pullOf(repoA, i, head1))) - } - rr.ch.Wait() - rr.mu.Lock() - rr.started = nil - rr.mu.Unlock() -} - -// waitFor returns when n checks wait for a place. -func (rr *rankRig) waitFor(t *testing.T, n int) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for { - if w := len(rr.ch.Waiting()); w == n { - return - } else if time.Now().After(deadline) { - t.Fatalf("%d checks wait, want %d", w, n) - } - runtime.Gosched() - } -} - -// moveTrunkHoldingOne moves the trunk, so every judged candidate is asked again; the first re-judge to start is held. -func (rr *rankRig) moveTrunkHoldingOne(t *testing.T, judged int) { - rr.mu.Lock() - rr.hold = true - rr.mu.Unlock() - rr.setTrunks(t, map[string]string{repoA: trunkA2, repoSDK: trunkS}) - rr.ch.TrunkMoved(context.Background()) - <-rr.held - rr.waitFor(t, judged-1) -} - -func TestANewHeadStartsBeforeTheReJudgesOfATrunkMove(t *testing.T) { // 1897/AC1 - rr := newRankRig(t) - rr.judged(t, 4) - rr.moveTrunkHoldingOne(t, 4) // one runs, three re-judges wait - rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2))) - rr.waitFor(t, 4) - close(rr.release) - rr.ch.Wait() - if len(rr.started) != 5 { - t.Fatalf("%d checks started, want 5", len(rr.started)) - } - if g := rr.started[1].Group; g != group(9) { - t.Errorf("after the running check, %s started, want the new head's %s; order %v", g, group(9), groups(rr.started)) - } -} - -func TestTheReJudgeQueuedFirstStartsFirst(t *testing.T) { // 1897/AC2 - rr := newRankRig(t) - rr.judged(t, 3) - rr.moveTrunkHoldingOne(t, 3) // one runs, two re-judges wait - close(rr.release) - rr.ch.Wait() - if len(rr.started) != 3 { - t.Fatalf("%d checks started, want 3", len(rr.started)) - } - if a, b := rr.started[1], rr.started[2]; !a.Queued.Before(b.Queued) { - t.Errorf("%s (queued %s) started before %s (queued %s)", a.Group, a.Queued.Format(time.TimeOnly), b.Group, b.Queued.Format(time.TimeOnly)) - } -} - -func TestANewHeadDoesNotStopARunningReJudge(t *testing.T) { // 1897/AC3 - rr := newRankRig(t) - rr.judged(t, 1) - rr.moveTrunkHoldingOne(t, 1) // the re-judge runs - rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2))) - rr.waitFor(t, 1) - close(rr.release) - rr.ch.Wait() - if len(rr.errs) != 1 || rr.errs[0] != nil { - t.Fatalf("the running re-judge saw %v, want it never stopped", rr.errs) - } - if len(rr.started) != 2 || rr.started[0].Group != group(1) || rr.started[1].Group != group(9) { - t.Errorf("started %v, want the re-judge to end before the new head starts", groups(rr.started)) - } - key := keyOf(t, group(1), candidateOf(t, group(1), pullOf(repoA, 1, head1))) - if v := rr.verdict(t, key); v.State != delivery.VerdictPass || v.Trunk[repoA] != trunkA2 { - t.Errorf("the re-judge's verdict = %+v, want its pass on the moved trunk", v) - } -} - -func TestWaitingChecksAreNamedByKind(t *testing.T) { // 1897/AC4, the checker's own view - rr := newRankRig(t) - rr.judged(t, 2) - rr.moveTrunkHoldingOne(t, 2) - rr.ch.Candidate(context.Background(), group(9), candidateOf(t, group(9), pullOf(repoA, 9, head2))) - rr.waitFor(t, 2) - kinds := map[string]string{} - for _, w := range rr.ch.Waiting() { - kinds[w.Group] = w.Kind - } - close(rr.release) - rr.ch.Wait() - running := rr.started[0].Group - other := group(1) - if running == other { - other = group(2) - } - if kinds[group(9)] != KindNew || kinds[other] != KindTrunk || len(kinds) != 2 { - t.Errorf("waiting = %v, want %s %s and %s %s", kinds, group(9), KindNew, other, KindTrunk) - } -} - func groups(cs []*Case) []string { out := make([]string, len(cs)) for i, c := range cs { @@ -179,7 +39,7 @@ func TestCheckerStatusNamesTheWaitingChecksByKind(t *testing.T) { // 1897/AC4 ctx := context.Background() j := &judgeFunc{f: func(_ context.Context, c *Case, _ int) (Outcome, error) { if c.Group == group(2) { - return Outcome{Verdict: failure("checker", "no "+NodesWords+": every node is offline")}, nil + return Outcome{Verdict: failure("the forge", "go.mod cannot be read: no responders (the forge was "+UnheardWords+")")}, nil } return Outcome{Verdict: passing()}, nil }} diff --git a/modules/mesh-delivery-checker/internal/check/read.go b/modules/mesh-delivery-checker/internal/check/read.go index 7fd9055..02d6686 100644 --- a/modules/mesh-delivery-checker/internal/check/read.go +++ b/modules/mesh-delivery-checker/internal/check/read.go @@ -8,7 +8,6 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/delivery" - "git.novox.be/novox/mesh-sdk/go/delivery/buildrun" ) // The read tools (novox/hq ADR 0484 clauses 8 and 13, issue 1686): what the checker stored, as stored, with the line of the @@ -19,7 +18,7 @@ const ( VerdictsReadTool = "checker_verdicts" PullReadTool = "checker_pull" KeptReadTool = "checker_kept" - // PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker is held by and what it runs (issue 1685). + // PreviewsReadTool reads the previews state (issue 1680); StatusReadTool reads what the checker runs (issue 1685: nothing holds it since R4g). PreviewsReadTool = "checker_previews" StatusReadTool = "checker_status" ) @@ -34,12 +33,10 @@ type StateRead struct { type Reader struct { VerdictState, KeptState, CandidateState StateRead - PreviewState StateRead // the checker's merge previews - PlansState StateRead // the controller's delivery plans in their build phase; zero where the bucket is absent - StepStates map[string]StateRead // the build modules' steps states, by module - Capacity func() (string, error) // the capacity setting as written, `node=steps,...` - HaveState StateRead // the node-engine's have: whether a build module is on a node and running - StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 + PreviewState StateRead // the checker's merge previews + StepStates map[string]StateRead // the build modules' steps states, by module + HaveState StateRead // the node-engine's have: whether a build module is on a node and running + StepBound time.Duration // how long the checker waits for one step; DefaultStepBound when 0 // Trunks are the trunk heads of every repository (the gate's source); nil names no re-judge after a trunk move. Trunks func() (map[string]string, error) Now func() time.Time @@ -125,8 +122,8 @@ func (r Reader) Verdicts(group string) ([]VerdictRead, error) { return out, nil } -// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for an hq pull request, -// that hq is judged by merge-check.sh until issue 1296 (HQNote); for one the indexer holds in no candidate, that none does. +// Pull answers for one pull request: the verdict of the candidate that holds it, with its report; for one the indexer holds +// in no candidate, that none does. func (r Reader) Pull(repository string, number int) (VerdictRead, error) { if r.CandidateState.Keys == nil || r.CandidateState.Get == nil { return VerdictRead{}, fmt.Errorf("the candidates state cannot be read here") @@ -149,9 +146,6 @@ func (r Reader) Pull(repository string, number int) (VerdictRead, error) { if p.Repository != repository || p.Number != number { continue } - if repository == HQRepository { - return VerdictRead{Key: k, Note: HQNote}, nil - } vkey, err := delivery.CheckerVerdictKey(c.Group, c.HeadsHash) if err != nil { return VerdictRead{}, err @@ -266,18 +260,10 @@ type LeftStep struct { Why string `json:"why"` } -// Status is what holds the checker back and what runs now, read from the buckets: the checker's held and running steps live in -// its process, so they are shown as what the buckets say (the steps in flight on the build modules, the candidates without a -// verdict for their heads, and the delivery plans in their build phase), not as the process's own counters. +// Status is what runs now, read from the buckets: the steps in flight on the build modules and the candidates without a +// verdict for their heads. Nothing holds a check since R4g: a check is an ask on the build seats' streams, whose backlog +// the delivery view shows (ADR 0525 §9). type Status struct { - Capacity map[string]int `json:"capacity"` - CapacityNote string `json:"capacity_note,omitempty"` - DefaultPerNode int `json:"default_per_node"` - PlansBuilding struct { - Readable bool `json:"readable"` - Keys []string `json:"keys"` - Holds bool `json:"holds"` - } `json:"plans_building"` Running []RunningStep `json:"running_steps"` LeftRunning []LeftStep `json:"left_running"` RunningByNode map[string]int `json:"running_by_node"` @@ -288,24 +274,9 @@ type Status struct { Notes []string `json:"notes,omitempty"` } -// Status reads the capacity, the plan bucket, the build modules' steps and the candidates against their verdicts. +// Status reads the build modules' steps and the candidates against their verdicts. func (r Reader) Status() (Status, error) { - st := Status{DefaultPerNode: buildrun.DefaultCapacity, Capacity: map[string]int{}, RunningByNode: map[string]int{}} - if r.Capacity == nil { - st.CapacityNote = "the capacity setting cannot be read from here" - } else if setting, err := r.Capacity(); err != nil { - st.CapacityNote = "the capacity setting cannot be read: " + err.Error() - } else if st.Capacity, err = buildrun.ParseCapacity(setting); err != nil { - st.Capacity, st.CapacityNote = map[string]int{}, "the capacity setting is not valid: "+err.Error() - } - if r.PlansState.Keys == nil { - st.Notes = append(st.Notes, PlansBuildingBucket+" is not readable here (absent), so it is read as empty: no delivery holds the checker") - } else if keys, err := r.PlansState.Keys(); err != nil { - st.Notes = append(st.Notes, PlansBuildingBucket+" cannot be listed, so it is read as empty: "+err.Error()) - } else { - sort.Strings(keys) - st.PlansBuilding.Readable, st.PlansBuilding.Keys, st.PlansBuilding.Holds = true, keys, len(keys) > 0 - } + st := Status{RunningByNode: map[string]int{}} mods := make([]string, 0, len(r.StepStates)) for m := range r.StepStates { mods = append(mods, m) @@ -386,9 +357,6 @@ func (r Reader) Status() (Status, error) { if err != nil { continue } - if !checksAny(c) { - continue // hq's pull requests alone: merge-check.sh judges them, the checker never does - } raw, err = r.VerdictState.Get(vkey) if err != nil { continue @@ -410,12 +378,12 @@ func (r Reader) Status() (Status, error) { } } } - st.Notes = append(st.Notes, "the places the checker holds are in its process and in no bucket: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict while a plan key exists, or while its node runs as many steps as its capacity, is held") + st.Notes = append(st.Notes, "nothing holds the checker and it holds no place: running_steps are the build modules' keys that can still be running (started within the step bound, their build module running on the node); a candidate without a verdict has its check ask in a build seat's stream, waiting for a node with a free slot, whose backlog the delivery view shows (ADR 0525 §4, §9)") return st, nil } -// DefaultStepBound is how long the checker waits for one step of a build module (a fetch downloads the Go modules, a test -// level runs a repository's tests); past it the check fails naming the step, and the step's place on its node is free. +// DefaultStepBound is how long one check ask's step may run once a build node took it (its key written running; the fetch, +// the digests and the levels), never counting its wait in the stream; past it the check fails naming the node. const DefaultStepBound = 30 * time.Minute // leftWhy says why a key in the running state runs no step any more, or "" when it can still be running: its build module @@ -445,13 +413,3 @@ func (r Reader) leftWhy(module string, step delivery.Step, now time.Time, bound } return "" } - -// checksAny says whether a candidate has a pull request the checker judges: one outside hq. -func checksAny(c delivery.Candidate) bool { - for _, p := range c.Pulls { - if p.Repository != HQRepository { - return true - } - } - return false -} diff --git a/modules/mesh-delivery-checker/internal/check/read_test.go b/modules/mesh-delivery-checker/internal/check/read_test.go index bbf5079..ecd4109 100644 --- a/modules/mesh-delivery-checker/internal/check/read_test.go +++ b/modules/mesh-delivery-checker/internal/check/read_test.go @@ -3,7 +3,6 @@ package check import ( "context" "encoding/json" - "os" "strings" "testing" "time" @@ -35,7 +34,7 @@ func (f fakeState) read() StateRead { func readerOf(t *testing.T) (Reader, string) { t.Helper() r := newRig(t, passJudge()) - cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(HQRepository, 3, head2)) + cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1), pullOf(hqRepo, 3, head2)) r.ch.Candidate(context.Background(), "feat%2Fx", cand) r.ch.Wait() ckey, _ := delivery.CandidatesKey("feat/x") @@ -67,10 +66,12 @@ func TestTheVerdictToolsAnswerTheStoredVerdictWithTheFullReportOfEachPullRequest } } -func TestAnHqPullRequestIsSaidToBeJudgedByMergeCheckUntilIssue1296(t *testing.T) { // ADR 0484 clause 8 +// 1887/A10: an hq pull request is answered as any other, its report its verdict's (ADR 0527 §2), never a note that a script +// judges it. +func TestAnHqPullRequestIsAnsweredAsAnyOther(t *testing.T) { rd, _ := readerOf(t) - hq, err := rd.Pull(HQRepository, 3) - if err != nil || hq.Note != "hq: judged by merge-check.sh until issue 1296" || hq.Verdict != nil { + hq, err := rd.Pull(hqRepo, 3) + if err != nil || hq.Note != "" || hq.Verdict == nil { t.Errorf("hq pull = %+v, %v", hq, err) } p, err := rd.Pull(repoA, 7) @@ -129,16 +130,14 @@ func TestThePreviewsToolShowsTheStoredPreviewOfAPullRequest(t *testing.T) { } } -// 1685 live acceptance: checker_status reads the capacity, whether a plan key holds the checker, the steps in flight by node and -// the candidates without a verdict. -func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) { +// checker_status reads the steps in flight by node and the candidates without a verdict; since R4g nothing holds the checker +// and it has no capacity of its own (ADR 0525 §4, §7; 1685 closed by the hold's deletion). +func TestTheStatusToolReadsRunningSteps(t *testing.T) { running, _ := delivery.EncodeStep(delivery.Step{ID: "abc1", Tool: "build_go_test", Node: "shanks", State: delivery.StepRunning, Started: time.Date(2026, 10, 22, 15, 0, 0, 0, time.UTC)}) passed, _ := delivery.EncodeStep(delivery.Step{ID: "abc2", Tool: "build_go_test", Node: "ace", State: delivery.StepPassed}) cand := candidateOf(t, "feat/x", pullOf(repoA, 7, head1)) ckey, _ := delivery.CandidatesKey("feat/x") rd := Reader{ - Capacity: func() (string, error) { return "shanks=2,ace=0", nil }, - PlansState: fakeState{"p1": []byte("{}")}.read(), StepStates: map[string]StateRead{"mesh-build-go": fakeState{"shanks.abc1": running, "ace.abc2": passed}.read()}, CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{}.read(), @@ -149,21 +148,15 @@ func TestTheStatusToolReadsCapacityHoldsAndRunningSteps(t *testing.T) { if err != nil { t.Fatal(err) } - if st.Capacity["shanks"] != 2 || st.Capacity["ace"] != 0 || st.DefaultPerNode != 1 { - t.Errorf("capacity = %v default %d", st.Capacity, st.DefaultPerNode) - } - if !st.PlansBuilding.Holds || len(st.PlansBuilding.Keys) != 1 { - t.Errorf("plans = %+v", st.PlansBuilding) - } if len(st.Running) != 1 || st.Running[0].Node != "shanks" || st.RunningByNode["shanks"] != 1 || st.RunningByNode["ace"] != 0 { t.Errorf("running = %+v %v", st.Running, st.RunningByNode) } if len(st.Unjudged) != 1 || st.Unjudged[0] != "feat/x" { t.Errorf("unjudged = %v", st.Unjudged) } - // An absent plan bucket is read as empty and said; an unreadable capacity is said, not guessed. - st, err = Reader{Capacity: func() (string, error) { return "", os.ErrNotExist }}.Status() - if err != nil || st.PlansBuilding.Holds || st.CapacityNote == "" || len(st.Notes) == 0 { + // What cannot be read is said, not guessed. + st, err = Reader{}.Status() + if err != nil || len(st.Notes) == 0 { t.Errorf("empty status = %+v, %v", st, err) } } @@ -228,16 +221,15 @@ func TestTheStatusCountsOnlyStepsThatCanStillRunAndSaysWhyTheOthersAreLeft(t *te } } -// 1890: a group whose pull requests are all hq's is never judged by the checker (merge-check.sh judges hq), so it is not -// a candidate held back. -func TestTheStatusDoesNotCountAnHQOnlyGroupAsUnjudged(t *testing.T) { // 1890/AC4 - cand := candidateOf(t, "pb-x", pullOf(HQRepository, 3, head2)) +// 1887/A10: a group of hq's pull requests alone is judged by the checker since R4g, so without a verdict it is unjudged. +func TestTheStatusCountsAnHQOnlyGroupAsUnjudged(t *testing.T) { + cand := candidateOf(t, "pb-x", pullOf(hqRepo, 3, head2)) ckey, _ := delivery.CandidatesKey("pb-x") st, err := Reader{CandidateState: fakeState{ckey: cand}.read(), VerdictState: fakeState{}.read()}.Status() if err != nil { t.Fatal(err) } - if len(st.Unjudged) != 0 { - t.Errorf("unjudged = %v", st.Unjudged) + if len(st.Unjudged) != 1 || st.Unjudged[0] != "pb-x" { + t.Errorf("unjudged = %v, want pb-x", st.Unjudged) } } diff --git a/modules/mesh-delivery-checker/internal/check/results.go b/modules/mesh-delivery-checker/internal/check/results.go index 7eed537..8336247 100644 --- a/modules/mesh-delivery-checker/internal/check/results.go +++ b/modules/mesh-delivery-checker/internal/check/results.go @@ -71,22 +71,24 @@ type TestTime struct { // Tested is the answer of a test step of either build module (the Go one names packages and counts tests; the Node one // names the package directory as Module and counts none). type Tested struct { - Level string `json:"level"` - Module string `json:"module,omitempty"` - Passed bool `json:"passed"` - Why string `json:"why,omitempty"` - NoTests bool `json:"no_tests,omitempty"` - Tests int `json:"tests"` - Skipped int `json:"skipped"` - Packages []GoPackage `json:"packages,omitempty"` - Elapsed float64 `json:"elapsed"` - Setup float64 `json:"setup,omitempty"` - Bound float64 `json:"bound"` - SetupBound float64 `json:"setup_bound,omitempty"` - Slowest []TestTime `json:"slowest,omitempty"` - Failed []Failure `json:"failed,omitempty"` - OverBound bool `json:"over_bound"` - OverWhich string `json:"over_which,omitempty"` + Level string `json:"level"` + Module string `json:"module,omitempty"` + Passed bool `json:"passed"` + Why string `json:"why,omitempty"` + NoTests bool `json:"no_tests,omitempty"` + Tests int `json:"tests"` + Skipped int `json:"skipped"` + Packages []GoPackage `json:"packages,omitempty"` + Elapsed float64 `json:"elapsed"` + Setup float64 `json:"setup,omitempty"` + // Lint is the unit level's gofmt and go vet time, before its tests (mesh-build-go, ADR 0529 §1). + Lint float64 `json:"lint,omitempty"` + Bound float64 `json:"bound"` + SetupBound float64 `json:"setup_bound,omitempty"` + Slowest []TestTime `json:"slowest,omitempty"` + Failed []Failure `json:"failed,omitempty"` + OverBound bool `json:"over_bound"` + OverWhich string `json:"over_which,omitempty"` // JudgedBy is the program that ran the step (ADR 0535 §5): "installed", or "candidate " under runner // workspace; absent from a build module that does not take the runner. JudgedBy string `json:"judged_by,omitempty"` diff --git a/modules/mesh-delivery-checker/internal/check/try.go b/modules/mesh-delivery-checker/internal/check/try.go index 77e42d3..812773b 100644 --- a/modules/mesh-delivery-checker/internal/check/try.go +++ b/modules/mesh-delivery-checker/internal/check/try.go @@ -108,15 +108,15 @@ type TryStore interface { Create(key string, value []byte) error } -// NotChecked is the error of a try of a repository the checker judges no level of: hq, judged by merge-check.sh (ADR 0484 -// clause 8), and a repository whose trunk the gate's source does not name. +// NotChecked is the error of a try of a repository the checker judges no level of: one whose trunk the gate's source does +// not name. func NotChecked(repository string) error { return fmt.Errorf("no level of %s is checked by the checker", repository) } // AskTry writes a new try, queued, and answers its id: a try of the same head that waits or runs is the answer to a second // ask, and at most MaxTries wait or run at once. trunks are the gate's trunk heads of every repository (source.main): a -// repository with none, and hq, are no repository the checker checks. A trunk asked by name is its trunk head now. +// repository with none is no repository the checker checks. A trunk asked by name is its trunk head now. func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) (string, error) { repo := strings.ToLower(strings.TrimSuffix(strings.TrimSpace(a.Repository), ".git")) head := strings.ToLower(strings.TrimSpace(a.Head)) @@ -136,7 +136,7 @@ func AskTry(store TryStore, trunks map[string]string, a TryAsk, now time.Time) ( trunkHead = h } } - if repo == HQRepository || trunkHead == "" { + if trunkHead == "" { return "", NotChecked(repo) } t := Try{Repository: repo, Head: head, Tests: a.Tests, Level: a.Level, State: TryQueued, Asked: now.UTC()} @@ -206,13 +206,11 @@ type TryOptions struct { Run string } -// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check, and writes its -// end. It holds no timer: a try starts when its key is written, or when a delivery that held it ends (Reconsider). +// Tryer runs the tries: it takes a queued try, judges it through the same Judge as a pull request's check (a check ask, ADR +// 0525 §1), and writes its end. It holds no timer: a try starts when its key is written. type Tryer struct { Tries KV Judge Judge - // Hold says whether a delivery builds, so no try starts (ADR 0484 clause 12); nil never holds. - Hold func() bool Bound time.Duration Now func() time.Time Logf func(format string, args ...any) @@ -250,7 +248,6 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) { if t.Over() { return } - held := tr.Hold != nil && tr.Hold() tr.mu.Lock() if tr.inflight == nil { tr.inflight, tr.seen = map[string]bool{}, map[string]bool{} @@ -259,11 +256,6 @@ func (tr *Tryer) Put(ctx context.Context, key string, value []byte) { tr.mu.Unlock() return } - if held { - tr.mu.Unlock() // not seen yet: a try left running is still taken over when the hold ends - tr.logf("%s: not started, a delivery builds", key) - return - } tr.seen[key] = true tr.inflight[key] = true tr.mu.Unlock() @@ -361,13 +353,14 @@ func (tr *Tryer) run(ctx context.Context, key string) { tr.logf("%s: %s", key, t.State) } -// judgeWithin runs a case within the bound (DefaultBound when 0). A judge that fails to judge, and a check that ran past its -// bound, are verdicts of failure naming the machine and the step, and judged says false for them. +// judgeWithin runs a case within the bound (none when 0: the asks' steps are bounded from their start, ADR 0525 §6). A +// judge that fails to judge, and a check that ran past its bound, are verdicts of failure naming the machine and the step, +// and judged says false for them. func judgeWithin(ctx context.Context, j Judge, cs *Case, bound time.Duration) (Outcome, bool) { - if bound <= 0 { - bound = DefaultBound + run, cancel := context.WithCancel(ctx) + if bound > 0 { + run, cancel = context.WithTimeoutCause(ctx, bound, errBound) } - run, cancel := context.WithTimeoutCause(ctx, bound, errBound) defer cancel() out, err := j.Judge(run, cs) stage, node := cs.Progress.Now() diff --git a/modules/mesh-delivery-checker/internal/check/try_test.go b/modules/mesh-delivery-checker/internal/check/try_test.go index 65b89b5..ce777d6 100644 --- a/modules/mesh-delivery-checker/internal/check/try_test.go +++ b/modules/mesh-delivery-checker/internal/check/try_test.go @@ -23,7 +23,7 @@ func (m memTries) Create(key string, value []byte) error { return err } -var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, HQRepository: trunkA} +var tryTrunks = map[string]string{repoA: trunkA, repoSDK: trunkS, hqRepo: trunkA} func readTry(t testing.TB, kv *memKV, id string) Try { t.Helper() @@ -67,7 +67,7 @@ func TestATryIsAskedQueuedOncePerHeadAndAtMostFourAtOnce(t *testing.T) { // ADR func TestATryOfARepositoryWithNoCheckedLevelIsRefused(t *testing.T) { // the agreed error of the try store := memTries{newMemKV()} - for _, repo := range []string{HQRepository, "novox/mesh-lab"} { + for _, repo := range []string{"novox/no-trunk", "novox/mesh-lab"} { // hq is checked since R4g (ADR 0527): see tryTrunks _, err := AskTry(store, tryTrunks, TryAsk{Repository: repo, Head: head1, Trunk: "main"}, time.Now()) if err == nil || err.Error() != "no level of "+repo+" is checked by the checker" { t.Errorf("%s: %v", repo, err) @@ -153,21 +153,10 @@ func TestAFailingTryNamesWhatFailedAndAStepThatCannotRunIsAnError(t *testing.T) if got := readTry(t, kv, id); got.State != TryFail || len(got.Failed) != 1 || !strings.Contains(got.Failed[0], "TestThreeIsFour") { t.Errorf("try = %+v", got) } - // No build node: the try could not run, which is an error, not a failure of the change. - kv2 := newMemKV() - b2 := newBench(t) - b2.ex.Toolchains = entries{} - id2, _ := AskTry(memTries{kv2}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) - tr2 := &Tryer{Tries: kv2, Judge: b2.ex} - raw, _, _, _ = kv2.Get(ctx, id2) - tr2.Put(ctx, id2, raw) - tr2.Wait() - if got := readTry(t, kv2, id2); got.State != TryError { - t.Errorf("try with no node = %+v, want error", got) - } + // No build node is no error since R4g: the try's ask waits in the check stream until a node pulls it (ADR 0525 §4). } -func TestATryWaitsWhileADeliveryBuildsAndATryLeftRunningIsTakenOver(t *testing.T) { +func TestATryLeftRunningIsTakenOver(t *testing.T) { ctx := context.Background() kv := newMemKV() id, _ := AskTry(memTries{kv}, tryTrunks, TryAsk{Repository: repoA, Head: head1}, time.Now()) @@ -180,14 +169,7 @@ func TestATryWaitsWhileADeliveryBuildsAndATryLeftRunningIsTakenOver(t *testing.T t.Fatal(err) } b := newBench(t) - held := true - tr := &Tryer{Tries: kv, Judge: b.ex, Hold: func() bool { return held }} - tr.Reconsider(ctx) - tr.Wait() - if n := len(b.steps.tool("fetch")); n != 0 { - t.Fatalf("a try ran while a delivery builds") - } - held = false + tr := &Tryer{Tries: kv, Judge: b.ex} tr.Reconsider(ctx) tr.Wait() if got := readTry(t, kv, id); got.State != TryPass { diff --git a/modules/mesh-delivery-checker/manifest_test.go b/modules/mesh-delivery-checker/manifest_test.go index a4b5442..4a9f21b 100644 --- a/modules/mesh-delivery-checker/manifest_test.go +++ b/modules/mesh-delivery-checker/manifest_test.go @@ -46,6 +46,7 @@ type manifest struct { Consumes []string `json:"consumes"` State []state `json:"state"` Reads []string `json:"reads"` + Uses []string `json:"uses"` Invokes []string `json:"invokes"` Config map[string]struct { Type string `json:"type"` @@ -96,12 +97,17 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 t.Errorf("state = %+v, want %+v", m.State, want) } if want := []string{"mesh-build-go.steps", "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-controller.plans-building", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", + "mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", "mesh-delivery-indexer.manifests", "mesh-delivery-indexer.modules", "mesh-delivery-watchman.presence", "node-engine.have"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } - if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file", "mesh-build-go.build_go_digest", "mesh-build-go.build_go_fetch", "mesh-build-go.build_go_test", "mesh-build-node.build_node_fetch", "mesh-build-node.build_node_test"}; !reflect.DeepEqual(sorted(m.Invokes), want) { - t.Errorf("invokes = %v, want only the build modules' fetch, digest and test (no run step before issue 1309, ADR 0484 clause 5) and the forge's two reads for a mesh-sdk pin (clause 9)", m.Invokes) + // 1887/A5b: the checker reads nothing of the controller's plans-building (ADR 0483 superseded by ADR 0525 §7), calls no + // build module's tool and holds no capacity: it puts check asks on the Go and Node build seats (ADR 0525 §1, §4). + if want := []string{"gitea.gitea_commits", "gitea.gitea_compare", "gitea.gitea_get_file"}; !reflect.DeepEqual(sorted(m.Invokes), want) { + t.Errorf("invokes = %v, want only the forge's reads for a mesh-sdk pin (ADR 0484 clause 9)", m.Invokes) + } + if want := []string{"node-build-go", "node-build-node"}; !reflect.DeepEqual(sorted(m.Uses), want) { + t.Errorf("uses = %v, want the Go and Node build seats", m.Uses) } // The checker is woken by a build module's report of a toolchain, as the indexer is: the bucket may not exist when it starts. if want := []string{"mesh-build-go.toolchain.reported", "mesh-build-node.toolchain.reported"}; !reflect.DeepEqual(sorted(m.Consumes), want) { @@ -122,7 +128,7 @@ func TestManifestDeclarations(t *testing.T) { // 1678/AC1 if want := []string{"checker_kept", "checker_previews", "checker_pull", "checker_status", "checker_try", "checker_try_read", "checker_verdicts"}; !reflect.DeepEqual(sorted(m.Tools), want) { t.Errorf("tools = %v, want %v", m.Tools, want) } - for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "uses", "defines-seats", "seat", "seats", "bus"} { + for _, k := range []string{"logs", "emits", "claims", "provides", "requires", "defines-seats", "seat", "seats", "bus", "config"} { if _, ok := top[k]; ok { t.Errorf("the manifest declares %q: the checker has none", k) } @@ -148,35 +154,19 @@ func TestManifestProcess(t *testing.T) { // 1678/AC1 t.Errorf("process = %v", p) } env, _ := p["env"].(map[string]any) - // The capacity of each node is the module's setting `capacity` (ADR 0484 clause 12, as the builder's, issue 1661), laid into the - // settings file the program reads; empty is one step at a time everywhere. The environment holds no capacity. - if len(env) != 2 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" || env["MESH_CHECKER_SETTINGS"] != "${dir:state}/settings.json" { - t.Errorf("env = %v", p["env"]) + if len(env) != 1 || env["MESH_BROKER_FILE"] != "${dir:state}/broker" { + t.Errorf("env = %v, want the broker file alone", p["env"]) } - if !reflect.DeepEqual(p["restart-on"], []any{"settings"}) { - t.Errorf("restart-on = %v, want the settings file, so a changed capacity reaches the running program", p["restart-on"]) - } - c, ok := m.Config["capacity"] - if !ok || len(m.Config) != 1 || c.Type != "string" || c.Default != "" || c.Description == "" { - t.Errorf("config = %+v, want one string setting, capacity, empty (one step at a time on every node) to start", m.Config) - } - var settings map[string]any for _, r := range m.Resources { if r["id"] == "settings" { - settings = r + t.Errorf("a settings resource %v: the checker has no capacity of its own (ADR 0525 §4)", r) } } - if settings == nil || settings["type"] != "file" || settings["merge"] != "json" || !reflect.DeepEqual(settings["takes-config"], []any{"capacity"}) || settings["path"] != "${dir:state}/settings.json" { - t.Errorf("settings resource = %v", settings) - } - if raw, err := os.ReadFile("files/settings.json"); err != nil || !strings.Contains(string(raw), `"capacity": ""`) { - t.Errorf("files/settings.json = %q, %v, want the setting's starting value", raw, err) - } if _, bus := p["bus"]; bus { t.Errorf("the process declares a bus: it runs on its own broker credential") } - if len(m.Build.Artifacts) != 3 || m.Build.Artifacts[2].Name != "files" || m.Build.Artifacts[2].Kind != "archive" || m.Build.Artifacts[2].From != "files" { - t.Fatalf("artifacts = %+v, want the program, the tools and the files", m.Build.Artifacts) + if len(m.Build.Artifacts) != 2 { + t.Fatalf("artifacts = %+v, want the program and the tools", m.Build.Artifacts) } if a := m.Build.Artifacts[1]; a.Name != "tools" || a.Kind != "bundle" || a.Language != "go" || a.Binary != "mesh-delivery-checker-tools" || !reflect.DeepEqual(a.Loads, []string{"mesh-delivery-checker-tools"}) { diff --git a/modules/mesh-delivery-checker/module.json b/modules/mesh-delivery-checker/module.json index 2c22576..1dd6ce1 100644 --- a/modules/mesh-delivery-checker/module.json +++ b/modules/mesh-delivery-checker/module.json @@ -21,7 +21,7 @@ "mesh-build-go.toolchains", "mesh-build-node.steps", "mesh-build-node.toolchains", - "mesh-controller.plans-building", + "mesh-delivery-assigner.compositions", "mesh-delivery-builder.builds", "mesh-delivery-gate.source", "mesh-delivery-indexer.candidates", @@ -30,6 +30,10 @@ "mesh-delivery-watchman.presence", "node-engine.have" ], + "uses": [ + "node-build-go", + "node-build-node" + ], "tools": [ "checker_kept", "checker_previews", @@ -42,21 +46,9 @@ "invokes": [ "gitea.gitea_commits", "gitea.gitea_compare", - "gitea.gitea_get_file", - "mesh-build-go.build_go_digest", - "mesh-build-go.build_go_fetch", - "mesh-build-go.build_go_test", - "mesh-build-node.build_node_fetch", - "mesh-build-node.build_node_test" + "gitea.gitea_get_file" ], - "config": { - "capacity": { - "type": "string", - "description": "How many build steps the checker runs at once on each node, as node=steps pairs separated by commas, such as shanks=2,ace=1 (ADR 0484 clause 12); a node set to 0 never checks, a node not named runs one at a time, and empty is one everywhere. The builder's capacity is the builder's own.", - "pattern": "|[a-z0-9_-]+=[0-9]+(,[a-z0-9_-]+=[0-9]+)*", - "default": "" - } - }, + "own-secrets": { "broker": "${dir:state}/broker" }, @@ -77,17 +69,6 @@ "place": ".", "unknown": "program" }, - { - "id": "settings", - "trusted": false, - "type": "file", - "path": "${dir:state}/settings.json", - "mode": "0600", - "owner": "mesh-delivery-checker", - "takes-config": ["capacity"], - "merge": "json", - "template": "files/settings.json" - }, { "id": "checker", "type": "process", @@ -97,12 +78,8 @@ "./mesh-delivery-checker" ], "env": { - "MESH_BROKER_FILE": "${dir:state}/broker", - "MESH_CHECKER_SETTINGS": "${dir:state}/settings.json" + "MESH_BROKER_FILE": "${dir:state}/broker" }, - "restart-on": [ - "settings" - ], "user": "mesh-delivery-checker", "health": { "kind": "unit" @@ -129,11 +106,6 @@ "loads": [ "mesh-delivery-checker-tools" ] - }, - { - "name": "files", - "kind": "archive", - "from": "files" } ] } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go index 448e24c..b4e7349 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/integration_test.go @@ -185,7 +185,7 @@ func startIndexer(t *testing.T, put func(context.Context, jetstream.KeyValue), b // TestIndexerWritesModulesAtAMerge: a module's record and its newest pointer are in mesh-controller_builds; a merged // message on FORGE makes modules. within 5 s, the fingerprint of the merge's tree and the recorded inputs. func TestIndexerWritesModulesAtAMerge(t *testing.T) { // 1239/AC9; I1, I3: the modules are the merged message's, no build record names them (issue 1851) - // The bucket holds only build-agent's record, the one the indexer reads until R4g (the builder's own input). + // The bucket holds only build-agent's record, which no fingerprint reads since R4g (ADR 0530 §1). js, ctx, _, modules := startIndexer(t, func(ctx context.Context, builds jetstream.KeyValue) { rec := `{"id":"build-1","repository":"novox/mesh-controller","ref":"main","module":"build-agent","commit":"7a74cee83f88e6585a5511c3cc53bdf0f5303d20","path":"modules/build-agent","source-fingerprint":"src1:e846efe73716b78e80860baaac96153de2cb76551325753838ca245d2160a3d7","at":"2026-10-19T09:38:00Z"}` if _, err := builds.Put(ctx, delivery.BuildKey("build-agent", "build-1"), []byte(rec)); err != nil { @@ -276,8 +276,8 @@ func TestIndexerKeepsManifestsAtAMerge(t *testing.T) { // 1444/AC4, 1449/AC4 // TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain: the bucket mesh-build-go_toolchains exists (the state // toolchains of the build module, delivery.ToolchainsBucket) and holds no key; a merged message of mesh-catalog carrying // avahi's real manifest makes modules.avahi with none2 'toolchain not yet reported'; the bucket made after the indexer started, a put of the key n1.go and the build module's event toolchain.reported (the build -// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, none2 "builder's -// fingerprint not yet known". +// module's own layout, delivery.ToolchainKey and EncodeToolchain) makes, within 5 s and with no merge, its fingerprint: +// no builder input is waited for (ADR 0530 §1; issue 1887 I1). func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *testing.T) { // 1444/AC10 manifest, err := os.ReadFile("../../internal/index/testdata/catalogue-a67f0196-avahi.module.json") if err != nil { @@ -342,8 +342,8 @@ func TestIndexerWritesAGoModuleAgainWhenTheBuildModuleReportsItsToolchain(t *tes if _, err := js.Publish(ctx, "mesh.stream.mesh-build-go.event.toolchain.reported", []byte(`{"node":"n1","language":"go"}`)); err != nil { t.Fatal(err) } - if got := none2("builder's fingerprint not yet known"); got != "builder's fingerprint not yet known" { - t.Errorf("none2 = %q within 5 s of the toolchain key, want the builder's fingerprint not yet known", got) + if got := none2(""); got != "" { + t.Errorf("none2 = %q within 5 s of the toolchain key, want a fingerprint and no reason", got) } } diff --git a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go index d921599..10402a9 100644 --- a/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go +++ b/modules/mesh-delivery-indexer/cmd/mesh-delivery-indexer/main.go @@ -18,7 +18,6 @@ import ( "os" "os/signal" "strings" - "sync/atomic" "syscall" "github.com/nats-io/nats.go" @@ -262,20 +261,14 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro return err } known := index.NewBuilds() - // A build of the module that builds the builder moves the builder's Builder input and so every fingerprint of the new - // form, with no merge (issue 1727): the new-form fingerprints are computed again when one is heard, once the indexer exists. - var started atomic.Pointer[index.Indexer] + // The controller's build records name the directory of a module whose key names none (fillPaths); no build of any + // module moves a fingerprint since no fingerprint takes the builder (ADR 0530 §1). each := func(e jetstream.KeyValueEntry) { if opOf(e) { known.Put(e.Key(), e.Value()) } else { known.Delete(e.Key()) } - if ix := started.Load(); ix != nil && strings.HasPrefix(e.Key(), index.BuilderBuiltBy+".build.") { - if err := ix.RecomputeForBuild(ctx); err != nil { - logger.Printf("computing the new-form fingerprints again after %s: %v", e.Key(), err) - } - } } w, err := settle(ctx, builds, each) if err != nil { @@ -327,10 +320,9 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := previews.ensure(); err != nil { return err } - // Only now, with the toolchains read, are the builds heard to compute from (a build before it would write every Go - // module as having no toolchain reported, and back); the builds heard until now are computed once. - started.Store(ix) - if err := ix.RecomputeForBuild(ctx); err != nil { + // Only now, with the toolchains read, are the fingerprints computed once (before it every Go module would be written as + // having no toolchain reported, and back). + if err := ix.RecomputeAtStart(ctx); err != nil { logger.Printf("computing the new-form fingerprints at start: %v", err) } // A build module says each time it wrote its toolchains key: its stream, and the indexer's durable consumer on it, are diff --git a/modules/mesh-delivery-indexer/go.mod b/modules/mesh-delivery-indexer/go.mod index 6390b2e..0aaff15 100644 --- a/modules/mesh-delivery-indexer/go.mod +++ b/modules/mesh-delivery-indexer/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-indexer go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-indexer/go.sum b/modules/mesh-delivery-indexer/go.sum index db1ea18..2a3e9f3 100644 --- a/modules/mesh-delivery-indexer/go.sum +++ b/modules/mesh-delivery-indexer/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-indexer/internal/index/inputs.go b/modules/mesh-delivery-indexer/internal/index/inputs.go index 3af599e..b562587 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs.go @@ -16,8 +16,9 @@ import ( const ( // None2NoManifest is for a module whose directory the merged message carries no manifest of (an older bridge). None2NoManifest = "the message carries no manifest" - // None2Builder is for a module with no artifact of a toolchain: only the builder's fingerprint is missing. - None2Builder = "builder's fingerprint not yet known" + // None2Pending is for a module with no artifact of a toolchain whose fingerprint the recompute of its chain gives (its + // bases' fingerprints; no builder input since ADR 0530 §1). + None2Pending = "the fingerprint is computed with its chain" // None2Context is for a module an artifact of which reads another repository's tree (today's form gives it none // either, NoneContext). None2Context = "context not pinned" @@ -146,7 +147,7 @@ func inputsOfNamed(module, raw, tree string, locks, objects map[string]string, b case len(langs) > 0: out.None2 = delivery.ToolchainNoReport default: - out.None2 = None2Builder + out.None2 = None2Pending } return out } diff --git a/modules/mesh-delivery-indexer/internal/index/inputs_test.go b/modules/mesh-delivery-indexer/internal/index/inputs_test.go index b395d1e..dc54985 100644 --- a/modules/mesh-delivery-indexer/internal/index/inputs_test.go +++ b/modules/mesh-delivery-indexer/internal/index/inputs_test.go @@ -154,7 +154,10 @@ func TestAManifestTheParserRefusesLeavesNoInputsAndSaysWhy(t *testing.T) { // 14 func TestAContextIsNotPinned(t *testing.T) { // 1450 (the rule 1444 keeps: context not pinned, as today's form) // build-agent's manifest from mesh-controller at f2d7bc94 has an image artifact built from a context in another repository. - in := inputsOf("build-agent", fixture(t, "controller-f2d7bc94-build-agent"), "t", nil, noBases) + // Its claim of the retired seat node-build-agent is read as a claim of a build seat: the parser refuses a retired seat + // since R4g (mesh-sdk A9a), and the claim is not what this test is about. + raw := strings.ReplaceAll(fixture(t, "controller-f2d7bc94-build-agent"), `"node-build-agent"`, `"node-build-go"`) + in := inputsOf("build-agent", raw, "t", nil, noBases) if in.Kept == nil || in.None2 != None2Context { t.Errorf("kept %+v none2 %q, want none2 %q", in.Kept, in.None2, None2Context) } diff --git a/modules/mesh-delivery-indexer/internal/index/newform_test.go b/modules/mesh-delivery-indexer/internal/index/newform_test.go index 31247a6..7376ac9 100644 --- a/modules/mesh-delivery-indexer/internal/index/newform_test.go +++ b/modules/mesh-delivery-indexer/internal/index/newform_test.go @@ -31,14 +31,16 @@ func TestAGoModuleAtAMergeHasItsManifestItsInputsAndNoFingerprintOfTheNewForm(t } } -func TestAModuleWithNoToolchainArtifactNamesTheBuildersFingerprint(t *testing.T) { // 1444/AC2 +// 1887/I1, ADR 0530 §1: a module with no artifact of a toolchain waits for no builder input: its fingerprint is written with +// the merge, its inputs naming no builder. +func TestAModuleWithNoToolchainArtifactGetsItsFingerprintAtOnce(t *testing.T) { // 1444/AC2 ix, mods, _ := manifestsIndexer(t, rec(t, "build-1", "ca-trust", catalogueRepo, "modules/ca-trust", "main", "", "2026-10-19T09:00:00Z", aInputs)) f := withManifests(mergedMsg(catalogueRepo, map[string]string{"modules/ca-trust": "newtree"}), map[string]string{"modules/ca-trust": catalogueManifest(t, "ca-trust")}, nil) if err := ix.Apply(context.Background(), f); err != nil { t.Fatal(err) } - if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.None2 != "builder's fingerprint not yet known" || m.Inputs2 == nil { - t.Errorf("module = %+v (%v), want none2 naming the builder's fingerprint", m, ok) + if m, ok := moduleOf(t, mods, "ca-trust"); !ok || m.FP2 == "" || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" { + t.Errorf("module = %+v (%v), want its fingerprint with no builder input", m, ok) } } diff --git a/modules/mesh-delivery-indexer/internal/index/previews.go b/modules/mesh-delivery-indexer/internal/index/previews.go index 5e8b454..c2913ea 100644 --- a/modules/mesh-delivery-indexer/internal/index/previews.go +++ b/modules/mesh-delivery-indexer/internal/index/previews.go @@ -23,8 +23,9 @@ const ( // NonePreviewConflict opens the reason of a module whose pull request's head does not merge into the trunk; the files // in conflict follow. NonePreviewConflict = "does not merge: the head conflicts with the trunk in " - // NonePreviewNoModule is for a module whose directory the preview names no manifest for. - NonePreviewNoModule = "the merge preview names no manifest for the module's directory" + // NonePreviewNoModule is for a module whose directory the preview names no manifest for: mesh-sdk's words, which the + // assigner reads as the change removing the module (issue 1297). + NonePreviewNoModule = delivery.CandidateNoneNoModule // NonePreviewNoManifest is for a module whose manifest at the merge the indexer does not hold. NonePreviewNoManifest = "the manifest of the merge is not kept" // NoneNoDirectory is for a module whose directory in its repository neither the modules bucket nor the preview tells. @@ -234,7 +235,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive out.None2 = in.None2 return out } - if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Builder { + if in.None2 != "" && in.None2 != delivery.ToolchainNoReport && in.None2 != None2Pending { out.None2 = in.None2 // a context, or a base with no fingerprint return out } @@ -248,22 +249,7 @@ func (ix *Indexer) candidateModule(ctx context.Context, m, dir string, pv delive } toolchains[artifact] = t.tc } - inputs := *in.Kept - if m == BuilderModule { - src, none := ix.builderInput() - if src == "" { - out.None2 = none - return out - } - inputs.Builder = src - } else { - fp, none := ix.baseFP(ctx)(BuilderModule) - if fp == "" { - out.None2 = strings.TrimSuffix(None2Builder+": "+none, ": ") - return out - } - inputs.Builder = fp - } + inputs := *in.Kept // no builder input (ADR 0530 §1) fp, none := delivery.SourceFingerprint2(inputs.With(m, toolchains)) if fp == "" { out.None2 = none diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains.go b/modules/mesh-delivery-indexer/internal/index/toolchains.go index 0257f1b..367e179 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains.go @@ -14,20 +14,12 @@ import ( // The new-form fingerprint of a module is written, and written again with no merge, when what it is of changes // (novox/hq ADR 0465 clause 2; issue 1444): a module's tree and lock hashes come with a merge and are kept in -// modules..Inputs2; its builder's, its bases' and its toolchain's come from here. The indexer holds no graph, so each -// recompute reads the manifest of every module from manifests. and takes the modules in the order their bases -// and the builder come first. - -const ( - // BuilderModule is the module that builds every module (novox/hq ADR 0453 clause 5): each module's Builder input is - // its new-form fingerprint. It does not exist before release R4c-4, and a module then has none for want of it. - BuilderModule = "mesh-delivery-builder" - // BuilderBuiltBy is the module that builds BuilderModule until release R4g: the source fingerprint (src1) the controller - // recorded on its newest successful trunk build is the builder's own Builder input, so a change of build-agent moves the - // builder's value (ADR 0453 clause 5). Its fingerprint of today's form is no input: build-agent has none before release - // R4 (ADR 0437 clause 5), so taking it left every module without a new-form fingerprint (issue 1727). - BuilderBuiltBy = "build-agent" -) +// modules..Inputs2; its bases' and its toolchain's come from here. The indexer holds no graph, so each recompute reads +// the manifest of every module from manifests. and takes the modules in the order their bases come first. +// +// **No module's fingerprint takes the builder as an input** (novox/hq ADR 0530 §1, issue 1887 I1): the builder orders and +// records builds and makes no artifact's bytes (ADR 0498 §1), so its Builder input is empty for every module, the builder's +// own included; a change of how an artifact is built is a change of its build module's recipe (ADR 0530 §2). // ToolchainModules are the build module of each language, whose state toolchains (delivery.ToolchainsBucket) the // indexer reads: the language's agreed toolchain is the one every node's entry in it agrees on. @@ -246,22 +238,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli } toolchains[artifact] = t.tc } - var noBuilder string - if module == BuilderModule { - // Built by build-agent until release R4g: the src1 of its newest successful trunk build record (issue 1727). - if src, none := ix.builderInput(); src != "" { - kept.Builder = src - } else { - noBuilder = none - } - } else if r := of(BuilderModule, chain); r.fp != "" { - kept.Builder = r.fp - } else { - noBuilder = None2Builder - if _, has := held[BuilderModule]; has { - noBuilder += ": " + r.none - } - } var r result switch { case k.contexts: @@ -270,8 +246,6 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli r.none = strings.Join(missing, "; ") case noToolchain != "": r.none = noToolchain - case noBuilder != "": - r.none = noBuilder default: r.fp, r.none = delivery.SourceFingerprint2(kept.With(module, toolchains)) if r.fp != "" { @@ -289,7 +263,7 @@ func (ix *Indexer) recomputeModules(ctx context.Context, overlay map[string]deli if _, merged := overlay[key]; merged { final := overlay[key] final.FP2, final.None2, final.Inputs2, final.Toolchains = r.fp, r.none, k.cur.Inputs2, r.tcs - // What the bases and the builder gave is kept as it was while the value it gave does not change, so a merge + // What the bases gave is kept as it was while the value it gave does not change, so a merge // that moves nothing for the module writes nothing (as Recompute does). if value, _, found, err := ix.Modules.Get(ctx, key); err == nil && found { if old, err := delivery.DecodeModule(value); err == nil && old.Inputs2 != nil && old.FP2 == r.fp && old.None2 == r.none { @@ -397,28 +371,15 @@ func toolchainsUsed(by map[string]delivery.Toolchain2) map[string]delivery.Toolc return out } -// builderInput is the Builder input of BuilderModule while BuilderBuiltBy builds it: the source fingerprint the controller -// recorded on that module's newest successful trunk build (issue 1727). With no such build, or one that recorded no -// fingerprint, there is none, and the reason says so. -func (ix *Indexer) builderInput() (src, none string) { - if ix.Builds != nil { - if r := ix.Builds.TrunkOf(BuilderBuiltBy); r != nil && r.SourceFingerprint != "" { - return r.SourceFingerprint, "" - } - } - return "", fmt.Sprintf("%s, which builds it, has no successful trunk build that recorded a source fingerprint", BuilderBuiltBy) -} - -// RecomputeForBuild is Recompute for a build of BuilderBuiltBy heard on the builds bucket, or the one at start (issue 1727). -// While the indexer has read no toolchain at all (it has just started and the toolchains buckets are not settled), every +// RecomputeAtStart is Recompute once the indexer has started. While the indexer has read no toolchain at all (it has just started and the toolchains buckets are not settled), every // Go module would be written as having no toolchain reported and then back: it computes nothing then, and the settling of // the toolchains computes them. -func (ix *Indexer) RecomputeForBuild(ctx context.Context) error { +func (ix *Indexer) RecomputeAtStart(ctx context.Context) error { ix.mu.Lock() none := len(ix.toolchains) == 0 ix.mu.Unlock() if none { - ix.logf("the builds changed before any toolchain was read: the new-form fingerprints are computed when they are") + ix.logf("no toolchain is read yet: the new-form fingerprints are computed when one is") return nil } return ix.Recompute(ctx) @@ -427,7 +388,7 @@ func (ix *Indexer) RecomputeForBuild(ctx context.Context) error { // fillPaths gives every module whose key names no directory (a key written before the indexer learned paths) the one its // newest build record names, once, by compare-and-set, changing nothing else (issue 1614): the builder needs it, and a // rarely merged repository would leave it without one for weeks. A key that names a directory is never written here. -// This read of the controller's build records goes in R4g with build-agent. +// It reads the controller's build records for nothing else: since R4g no fingerprint takes a build record (ADR 0530 §1). func (ix *Indexer) fillPaths(ctx context.Context) error { if ix.Modules == nil || ix.Builds == nil { return nil diff --git a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go index a60fac1..d97fdc4 100644 --- a/modules/mesh-delivery-indexer/internal/index/toolchains_test.go +++ b/modules/mesh-delivery-indexer/internal/index/toolchains_test.go @@ -117,8 +117,8 @@ func TestAgreedToolchainChangesAGoModulesReasonWithNoMerge(t *testing.T) { // 14 reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") reportToolchain(t, ix, "mesh-build-go", "ace", "go", "go1.27.1") m, _ := moduleOf(t, mods, "avahi") - if m.None2 != None2Builder || m.FP2 != "" { - t.Errorf("none2 = %q fp2 = %q, want none2 %q", m.None2, m.FP2, None2Builder) + if m.None2 != "" || m.FP2 == "" { + t.Errorf("none2 = %q fp2 = %q, want a fingerprint: no builder input is waited for (ADR 0530 §1)", m.None2, m.FP2) } } @@ -160,33 +160,31 @@ func fp2(t *testing.T, in delivery.SourceInputs2) string { return fp } -func TestBasesAndTheBuilderAreComputedBeforeTheModulesTakingThem(t *testing.T) { // 1444/AC7, AC8 as 1727 corrects it +// 1887/I1, ADR 0530 §1: bases are computed before the modules taking them, and no module's fingerprint takes the builder, +// the builder's own included: a trunk build of build-agent (or of the builder) moves nothing. +func TestBasesAreComputedFirstAndNoFingerprintTakesTheBuilder(t *testing.T) { // 1444/AC7, AC8 ix, mods := newFormIndexer(t, builderModules(t)...) reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agentSrcX)) - toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", builderFP)) - want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", builderFP, toolsFP))} + builderFP := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", "")) + toolsFP := fp2(t, goInputs(t, ix, mods, "mesh-tools", "")) + want := map[string]string{"mesh-delivery-builder": builderFP, "mesh-tools": toolsFP, "avahi": fp2(t, goInputs(t, ix, mods, "avahi", "", toolsFP))} for name, fp := range want { - if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" { - t.Errorf("%s: fp2 = %q none2 = %q, want %s", name, m.FP2, m.None2, fp) + if m, _ := moduleOf(t, mods, name); m.FP2 != fp || m.None2 != "" || m.Inputs2 == nil || m.Inputs2.Builder != "" { + t.Errorf("%s: fp2 = %q none2 = %q inputs %+v, want %s with no builder input", name, m.FP2, m.None2, m.Inputs2, fp) } } if m, _ := moduleOf(t, mods, "other"); m.FP2 != "" || !strings.Contains(m.None2, "ts-base") || !strings.Contains(m.None2, "base") { t.Errorf("other: fp2 = %q none2 = %q, want none2 naming the base ts-base", m.FP2, m.None2) } - // 1727: a new trunk build of build-agent records another source fingerprint; with no merge, the builder's value moves - // with it, and so do the modules built by it. a := agentBuild("build-1792587792086548572", agentSrcY, "2026-10-21T15:03:00Z") ix.Builds.Put(a[0].(string), a[1].([]byte)) if err := ix.Recompute(context.Background()); err != nil { t.Fatal(err) } - newBuilder := fp2(t, goInputs(t, ix, mods, "mesh-delivery-builder", agentSrcY)) - if m, _ := moduleOf(t, mods, "mesh-delivery-builder"); m.FP2 != newBuilder || newBuilder == builderFP { - t.Errorf("builder: fp2 = %q, want %s (was %s)", m.FP2, newBuilder, builderFP) - } - if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != fp2(t, goInputs(t, ix, mods, "mesh-tools", newBuilder)) { - t.Errorf("mesh-tools did not follow the builder: %+v", m) + for name, fp := range want { + if m, _ := moduleOf(t, mods, name); m.FP2 != fp { + t.Errorf("%s moved with a build of build-agent: fp2 = %q, want %s", name, m.FP2, fp) + } } } @@ -201,7 +199,7 @@ func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { / ix.mu.Lock() ix.toolchains = nil ix.mu.Unlock() - if err := ix.RecomputeForBuild(context.Background()); err != nil { + if err := ix.RecomputeAtStart(context.Background()); err != nil { t.Fatal(err) } if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != before.FP2 || m.None2 != "" { @@ -209,27 +207,6 @@ func TestABuildHeardBeforeAnyToolchainIsReadDropsNoFingerprint(t *testing.T) { / } } -func TestTheBuilderHasNoFingerprintWithoutATrunkBuildOfBuildAgentAndSaysSo(t *testing.T) { // 1727 - ix, mods := newFormIndexer(t, builderModules(t)...) - reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") - ix.Builds.Delete(delivery.BuildKey("build-agent", "build-1792580585452696147")) - // a failed build and a build of a branch are no trunk build - failed, fv := record(t, "b-failed", "build-agent", agentCloneURL, "modules/build-agent", "main", "failed", "2026-10-21T15:03:00Z", aInputs) - branch, bv := record(t, "b-branch", "build-agent", agentCloneURL, "modules/build-agent", "fix/x", "", "2026-10-21T15:04:00Z", aInputs) - ix.Builds.Put(branch, bv) - ix.Builds.Put(failed, fv) - if err := ix.Recompute(context.Background()); err != nil { - t.Fatal(err) - } - m, _ := moduleOf(t, mods, "mesh-delivery-builder") - if m.FP2 != "" || !strings.Contains(m.None2, "build-agent") || !strings.Contains(m.None2, "no successful trunk build") { - t.Errorf("builder: fp2 = %q none2 = %q, want none2 naming build-agent's missing trunk build", m.FP2, m.None2) - } - if m, _ := moduleOf(t, mods, "mesh-tools"); m.FP2 != "" || !strings.Contains(m.None2, "builder") { - t.Errorf("mesh-tools: fp2 = %q none2 = %q, want none2 naming the builder", m.FP2, m.None2) - } -} - func TestAToolchainEntryChangedToTheSameValuesWritesNothing(t *testing.T) { // 1444/AC9 ix, mods := newFormIndexer(t, builderModules(t)...) reportToolchain(t, ix, "mesh-build-go", "novox", "go", "go1.27.1") @@ -261,8 +238,8 @@ func TestAMergeWritesEachModuleOnceWithItsFinalValueAndAReplayWritesNothing(t *t t.Fatal(err) } m, _ := moduleOf(t, mods, "avahi") - if mods.writes != 1 || m.None2 != None2Builder { - t.Errorf("writes = %d none2 = %q after one merge, want one write with %q", mods.writes, m.None2, None2Builder) + if mods.writes != 1 || m.FP2 == "" { + t.Errorf("writes = %d fp2 = %q after one merge, want one write with its fingerprint", mods.writes, m.FP2) } if err := ix.Apply(ctx, f); err != nil { t.Fatal(err) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go index bcadf13..b4c5c4b 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/integration_test.go @@ -165,6 +165,8 @@ const ( tellerAnswers = "mesh-delivery-teller_answers" tellerAsked = "mesh-delivery-teller_asked" gateAsksKV = "mesh-delivery-gate_asks" + assignerAsks = "mesh-delivery-assigner_asks" + viewAnswers = "mesh-delivery-view_answers" seatAsk = "mesh.seat.operator-channel.accept.ask.mesh-delivery-teller" seatCancel = "mesh.seat.operator-channel.accept.cancel.mesh-delivery-teller" seatCondition = "mesh.seat.operator-channel.accept.condition.mesh-delivery-teller" @@ -174,7 +176,9 @@ const ( func fullConf(dir string) string { teller := append(append(append(keepsOf(tellerAnswers), keepsOf(tellerAsked)...), readOf(gateAsksKV)...), seatAsk, seatCancel, seatCondition, "$JS.API.DIRECT.GET.KV_asks.$KV.asks.mesh-delivery-teller.>") - view := append(readOf(tellerAnswers), readOf(gateAsksKV)...) + teller = append(append(teller, readOf(assignerAsks)...), readOf(viewAnswers)...) // issue 1297 B10: the manifest's reads + view := append(append(append(readOf(tellerAnswers), readOf(gateAsksKV)...), readOf(assignerAsks)...), keepsOf(viewAnswers)...) + assigner := keepsOf(assignerAsks) tools := append(keepsOf(gateAsksKV), readOf(tellerAnswers)...) return fmt.Sprintf(` listen: 127.0.0.1:-1 @@ -184,12 +188,13 @@ accounts { jetstream: enabled users = [ { user: "ctl", password: "pw" } -%s%s%s ] +%s%s%s%s ] } } `, filepath.Join(dir, "js"), user("n1.mesh-delivery-teller", teller, []string{seatDecided}), user("n1.mesh-delivery-view", view, nil), + user("n1.mesh-delivery-assigner", assigner, nil), user("n1.node-tools", tools, nil)) } @@ -238,7 +243,7 @@ func startFull(t *testing.T) *fullBed { } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) t.Cleanup(cancel) - for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1} { + for name, history := range map[string]uint8{gateAsksKV: delivery.AsksHistory, tellerAnswers: delivery.AnswersHistory, tellerAsked: 1, assignerAsks: delivery.AssignerAsksHistory, viewAnswers: delivery.ViewAnswersHistory} { if _, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: name, History: history}); err != nil { t.Fatal(err) } @@ -468,3 +473,75 @@ func TestTellerAsksAgainAnAskRefusedForWantOfRoom(t *testing.T) { // 1610/AC1 say(asks.Warrant{Ask: ga2.Ask.ID, Asker: delivery.TellerModule, Outcome: asks.OutcomeExpired, At: now, AskDigest: ga2.Ask.Digest()}) lastAsk(held.Ask.ID, second) } + +// TestTellerFillsAnAskFromTheViewsAnswerOnce starts the program against the bus with the permissions above: the assigner's +// fixture hold ask is written, then the view's answer under `.view`; the answer is under the ask's own key, once +// (rewriting the view's answer does not write a second), and a view answer under the merge's key fills nothing +// (1297/B10 AC: a view answer fills the ask once; an ask needing proof is not answered from a view key). +func TestTellerFillsAnAskFromTheViewsAnswerOnce(t *testing.T) { // 1297/B10 AC + now = time.Now().UTC() + b := startFull(t) + assigner, view := b.as(t, "n1.mesh-delivery-assigner"), b.as(t, "n1.mesh-delivery-view") + broker := filepath.Join(t.TempDir(), "broker") + cred, _ := json.Marshal(credential{URL: b.srv.ClientURL(), Node: "n1", Module: "mesh-delivery-teller", User: "n1.mesh-delivery-teller", Password: "pw"}) + if err := os.WriteFile(broker, cred, 0o600); err != nil { + t.Fatal(err) + } + env := map[string]string{"MESH_BROKER_FILE": broker} + logs := &syncBuffer{} + rn := startRunner(b.ctx, func(c context.Context) error { return run(c, func(k string) string { return env[k] }, logs) }) + t.Cleanup(func() { rn.finish(t, 10*time.Second) }) + + key, value, ask := holdAsk(t) + askKV, err := assigner.KeyValue(b.ctx, assignerAsks) + if err != nil { + t.Fatal(err) + } + viewKV, err := view.KeyValue(b.ctx, viewAnswers) + if err != nil { + t.Fatal(err) + } + answers, err := b.js.KeyValue(b.ctx, tellerAnswers) + if err != nil { + t.Fatal(err) + } + time.Sleep(500 * time.Millisecond) // the program is watching + if _, err := askKV.Put(b.ctx, key, value); err != nil { + t.Fatal(err) + } + if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "release", "the check was a flake")); err != nil { + t.Fatal(err) + } + var e jetstream.KeyValueEntry + for deadline := time.Now().Add(10 * time.Second); time.Now().Before(deadline); time.Sleep(50 * time.Millisecond) { + if e, err = answers.Get(b.ctx, delivery.AnswerKey(key)); err == nil { + break + } + } + if e == nil { + t.Fatalf("no answer under %s: %v\n%s", key, err, logs.String()) + } + if a, err := delivery.DecodeAnswer(e.Value()); err != nil || a.Option != "release" || answeredAsking(e.Value()) != ask.ID { + t.Fatalf("answer %+v err %v", a, err) + } + // The view's answer written again changes nothing under the ask. + if _, err := viewKV.Put(b.ctx, key+".view", viewAnswer(t, "keep", "")); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if again, err := answers.Get(b.ctx, delivery.AnswerKey(key)); err != nil || again.Revision() != e.Revision() { + t.Errorf("the answer was written a second time: %v", err) + } + // An answer under the merge's key fills nothing. + mkey, mvalue, _ := gateAsk(t, twoHeads(), twoHeads()) + if _, err := askKV.Put(b.ctx, mkey, mvalue); err != nil { + t.Fatal(err) + } + if _, err := viewKV.Put(b.ctx, mkey+".view", viewAnswer(t, "approve", "")); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + if _, err := answers.Get(b.ctx, delivery.AnswerKey(mkey)); err == nil { + t.Error("the merge's ask was answered from a view key") + } +} diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go index 525e003..f6b5e09 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/main.go @@ -137,10 +137,43 @@ func run(ctx context.Context, getenv func(string) string, logOut io.Writer) erro if err := client.Listen(t.Answer, t.Said); err != nil { return fmt.Errorf("hearing the router's warrants: %w", err) } - if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil { - return fmt.Errorf("watching the gate's asks: %w", err) + // The view's answers to the assigner's asks (issue 1297 B10): both buckets are the controller's to create, and a bucket not + // there is said and leaves the teller without that path rather than without the gate's asks (a wait for it needs a timer). + failed := make(chan error, 2) + assignerKV, viewKV := optionalBucket(ctx, js, delivery.AssignerAsksBucket, logger), optionalBucket(ctx, js, delivery.ViewAnswersBucket, logger) + if assignerKV != nil && viewKV != nil { + t.Assigner, t.View = kvGate{ctx: ctx, kv: assignerKV}, kvGate{ctx: ctx, kv: viewKV} + go func() { + failed <- watchAsks(ctx, viewKV, func(key string, _ uint64, _ []byte) { t.OnViewAnswer(key) }, func(string) {}, nil) + }() + go func() { + failed <- watchAsks(ctx, assignerKV, func(key string, _ uint64, _ []byte) { t.OnAssignerAsk(key) }, func(string) {}, nil) + }() } - return nil + go func() { + if err := watchAsks(ctx, gate, t.OnAsk, t.OnAskGone, t.OnCurrent); err != nil { + failed <- fmt.Errorf("watching the gate's asks: %w", err) + } + }() + select { + case <-ctx.Done(): + return nil + case err := <-failed: + if err != nil { + return fmt.Errorf("a watch ended: %w", err) + } + return nil + } +} + +// optionalBucket is a bucket the teller reads when it exists, and nil, said once, when it does not. +func optionalBucket(ctx context.Context, js jetstream.JetStream, name string, logger *log.Logger) jetstream.KeyValue { + kv, err := js.KeyValue(ctx, name) + if err != nil { + logger.Printf("the bucket %s cannot be read (%v): the view's answers fill no ask until the teller is started again with it", name, err) + return nil + } + return kv } // askedBucket is the teller's state bucket asked, `_`: what it asked of the router and which condition it diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go index 5d64273..3e36b3f 100644 --- a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/teller.go @@ -58,6 +58,10 @@ type Teller struct { Store store Answers answers Gate gateAsks + // Assigner is the assigner's bucket asks and View the view's bucket answers (issue 1297 B10); with either nil the teller + // answers nothing from the view. + Assigner gateAsks + View gateAsks // Say publishes a condition to the operator (mesh.seat.operator-channel.accept.condition.). Say func(c asks.Condition) error Logf func(format string, args ...any) diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers.go new file mode 100644 index 0000000..235bb4d --- /dev/null +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers.go @@ -0,0 +1,107 @@ +package main + +import ( + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// The view's answers (novox/hq ADR 0528 clause 6, ADR 0437 §10, issue 1297 B10). The operator may answer, in the view, the +// asks of the assigner that need no proof of who answered (the hold of a fingerprint and the restore of a data module set back, +// mesh-sdk go/delivery AskNeedsProof); the view writes its answer under `.view` in its bucket answers, and the teller, +// which alone fills an ask's answer, takes it from there as it takes a warrant from the phone: under the ask's own key, once, +// by compare-and-set. The merge's ask needs proof (ADR 0421 §1, issue 953) and is never answered from a view key. + +// OnViewAnswer handles a key put in the view's bucket answers. A key that is no view answer key, or is the answer to an ask that +// needs proof of who answered, is left and said. +func (t *Teller) OnViewAnswer(key string) { + t.mu.Lock() + defer t.mu.Unlock() + askKey, err := delivery.ParseViewAnswerKey(key) + if err != nil { + t.logf("the view's key %s is left: %v", key, err) + return + } + t.fillFromView(askKey) +} + +// OnAssignerAsk handles a key put in the assigner's bucket asks: a view answer that came before its ask fills it now. +func (t *Teller) OnAssignerAsk(key string) { + t.mu.Lock() + defer t.mu.Unlock() + if _, err := delivery.ViewAnswerKey(key); err != nil { + t.logf("the assigner's key %s is left and never answered from the view: %v", key, err) + return + } + t.fillFromView(key) +} + +// fillFromView writes the answer to the assigner's ask at askKey from the view's answer, when both are there and readable, the +// ask is of a kind that needs no proof, the option is one the ask offers and the asking is not answered yet. +func (t *Teller) fillFromView(askKey string) { + if t.Assigner == nil || t.View == nil { + return + } + viewKey, err := delivery.ViewAnswerKey(askKey) // refuses an ask that needs proof, naming the phone + if err != nil { + t.logf("%s is not answered from the view: %v", askKey, err) + return + } + rawAnswer, found, err := t.View.Get(viewKey) + if err != nil { + t.logf("%s: the view's answer could not be read, so the ask is left: %v", askKey, err) + return + } + if !found { + return + } + rawAsk, found, err := t.Assigner.Get(askKey) + if err != nil { + t.logf("%s: the assigner's ask could not be read, so the view's answer is left: %v", askKey, err) + return + } + if !found { + return // the answer waits for its ask: OnAssignerAsk fills it when it comes + } + va, err := delivery.DecodeViewAnswer(rawAnswer) + if err != nil { + t.logf("%s: the view's answer is not readable and fills nothing: %v", viewKey, err) + return + } + ask, err := delivery.DecodeAssignerAsk(askKey, rawAsk) + if err != nil { + t.logf("%s: the assigner's ask is not readable and takes no answer: %v", askKey, err) + return + } + if err := delivery.ViewAnswerFits(va, ask); err != nil { + t.logf("%s: nothing is written: %v", askKey, err) + return + } + akey := delivery.AnswerKey(askKey) + cur, crev, has, err := t.Answers.Get(akey) + if err != nil { + t.logf("%s: the answer could not be read: %v", askKey, err) + return + } + if has && answeredAsking(cur) == ask.Ask.ID { + return // this asking is answered, from the view or elsewhere + } + words := "answered in the view (mesh-delivery-view): " + va.Option + if w := strings.TrimSpace(va.Words); w != "" { + words += ": " + w + } + body, err := delivery.EncodeAnswer(delivery.Answer{Option: va.Option, Warrant: ask.Ask.ID + "@" + t.now().UTC().Format("2006-01-02T15:04:05.999999999Z07:00"), Words: words}) + if err != nil { + t.logf("%s: the answer could not be made: %v", askKey, err) + return + } + written, err := t.Answers.Write(akey, body, crev, has) + switch { + case err != nil: + t.logf("%s: the answer could not be written: %v", askKey, err) + case !written: + t.logf("%s: another writer answered first, and the view's answer is not written", askKey) + default: + t.logf("%s: %s", askKey, words) + } +} diff --git a/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers_test.go b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers_test.go new file mode 100644 index 0000000..cc86e6d --- /dev/null +++ b/modules/mesh-delivery-teller/cmd/mesh-delivery-teller/viewanswers_test.go @@ -0,0 +1,176 @@ +package main + +import ( + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/asks" + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const fpHeld = "src2:2222222222222222222222222222222222222222222222222222222222222222" + +// holdAsk is a key of the assigner's asks: the ask about a held fingerprint, as a fixture (in release 5 the assigner writes +// none yet; the value is `{"v":1,"ask":}`). +func holdAsk(t *testing.T) (key string, value []byte, ask asks.Ask) { + t.Helper() + key, err := delivery.HoldAskKey("plex", fpHeld) + if err != nil { + t.Fatal(err) + } + ask = asks.Ask{ID: delivery.AskID(key), Headline: "plex is held after a failed health check", Explanation: "ace failed the check.", Who: asks.Operator, + Expires: now.Add(asks.ApproveLasts), OnExpiry: "the fingerprint stays held", + Options: []asks.Option{{ID: "release", Label: "Release", Does: "lets the move go on", Level: asks.Approve}, {ID: "keep", Label: "Keep held", Does: "keeps it held", Level: asks.Approve}}} + value, err = delivery.EncodeAssignerAsk(key, delivery.AssignerAsk{Ask: ask}) + if err != nil { + t.Fatal(err) + } + return key, value, ask +} + +func holdKey() string { + k, _ := delivery.HoldAskKey("plex", fpHeld) + return k +} + +func viewAnswer(t *testing.T, option, words string) []byte { + t.Helper() + v, err := delivery.EncodeViewAnswer(delivery.ViewAnswer{Asking: delivery.AskID(holdKey()), Option: option, Words: words}) + if err != nil { + t.Fatal(err) + } + return v +} + +// withView gives the bed's teller the assigner's asks and the view's answers, both empty. +func withView(b *bed) (assigner, view memGate) { + assigner, view = memGate{}, memGate{} + b.t.Assigner, b.t.View = assigner, view + return assigner, view +} + +func TestAViewAnswerFillsTheAskOnce(t *testing.T) { // 1297/B10 AC + b := newBed(t) + assigner, view := withView(b) + key, value, ask := holdAsk(t) + assigner[key] = value + view[key+".view"] = viewAnswer(t, "release", "the check was a flake") + b.t.OnViewAnswer(key + ".view") + raw, ok := b.answers.written[key] + if !ok { + t.Fatalf("no answer under %s; log:\n%s", key, b.logs.String()) + } + a, err := delivery.DecodeAnswer(raw) + if err != nil || a.Option != "release" || answeredAsking(raw) != ask.ID || !strings.Contains(a.Words, "view") || !strings.Contains(a.Words, "the check was a flake") { + t.Errorf("answer = %+v, %v", a, err) + } + // The same answer heard again (the watch replays, the ask is rewritten) writes nothing more. + b.t.OnViewAnswer(key + ".view") + b.t.OnAssignerAsk(key) + if b.answers.writes != 1 { + t.Errorf("%d writes, want one", b.answers.writes) + } +} + +func TestAViewAnswerHeardBeforeItsAskFillsItWhenTheAskComes(t *testing.T) { // 1297/B10 + b := newBed(t) + assigner, view := withView(b) + key, value, _ := holdAsk(t) + view[key+".view"] = viewAnswer(t, "keep", "") + b.t.OnViewAnswer(key + ".view") + if b.answers.writes != 0 { + t.Fatalf("answered an ask the teller does not hold") + } + assigner[key] = value + b.t.OnAssignerAsk(key) + if a, err := delivery.DecodeAnswer(b.answers.written[key]); err != nil || a.Option != "keep" { + t.Errorf("answer = %+v, %v", a, err) + } +} + +func TestAnAskNeedingProofIsNotAnsweredFromAViewKey(t *testing.T) { // 1297/B10 AC + b := newBed(t) + assigner, view := withView(b) + key, value, _ := gateAsk(t, twoHeads(), twoHeads()) + // Even with the merge's ask under the assigner's bucket and a view answer under its key, nothing is written. + assigner[key] = value + view[key+".view"] = viewAnswer(t, "approve", "") + b.t.OnViewAnswer(key + ".view") + b.t.OnAssignerAsk(key) + if b.answers.writes != 0 || len(b.answers.written) != 0 { + t.Fatalf("the merge's ask was answered from the view: %v", b.answers.written) + } + if !strings.Contains(b.logs.String(), "issue 953") { + t.Errorf("the refusal does not say why:\n%s", b.logs.String()) + } +} + +func TestAViewAnswerOfAnOptionTheAskDoesNotOfferFillsNothing(t *testing.T) { // 1297/B10 + b := newBed(t) + assigner, view := withView(b) + key, value, _ := holdAsk(t) + assigner[key] = value + view[key+".view"] = viewAnswer(t, "delete", "") + b.t.OnViewAnswer(key + ".view") + if b.answers.writes != 0 || !strings.Contains(b.logs.String(), `chose "delete"`) { + t.Errorf("writes %d, log:\n%s", b.answers.writes, b.logs.String()) + } +} + +func TestAViewAnswerNeverReplacesTheAnswerOfTheSameAsking(t *testing.T) { // 1297/B10 + b := newBed(t) + assigner, view := withView(b) + key, value, ask := holdAsk(t) + assigner[key] = value + view[key+".view"] = viewAnswer(t, "keep", "") + early, err := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: ask.ID + "@2026-10-20T09:00:00Z", Words: "answered elsewhere"}) + if err != nil { + t.Fatal(err) + } + b.answers.written[key], b.answers.revs[key] = early, 7 + b.t.OnViewAnswer(key + ".view") + if b.answers.writes != 0 || string(b.answers.written[key]) != string(early) { + t.Errorf("the answer of the same asking was replaced") + } + // An earlier asking's answer (another router id) does not stop this one. + older, _ := delivery.EncodeAnswer(delivery.Answer{Option: "release", Warrant: "an-older-id@2026-10-19T09:00:00Z", Words: "old"}) + b.answers.written[key], b.answers.revs[key] = older, 8 + b.t.OnViewAnswer(key + ".view") + if b.answers.writes != 1 || answeredAsking(b.answers.written[key]) != ask.ID { + t.Errorf("writes %d, answer %s", b.answers.writes, b.answers.written[key]) + } +} + +func TestAnUnreadableViewAnswerOrAskFillsNothing(t *testing.T) { // 1297/B10 + b := newBed(t) + assigner, view := withView(b) + key, value, _ := holdAsk(t) + assigner[key] = []byte(`{"v":1}`) + view[key+".view"] = viewAnswer(t, "release", "") + b.t.OnViewAnswer(key + ".view") + assigner[key] = value + view[key+".view"] = []byte(`{"v":99}`) + b.t.OnViewAnswer(key + ".view") + b.t.OnViewAnswer("not-a-view-key") + b.t.OnViewAnswer("hold.x.view") + if b.answers.writes != 0 { + t.Errorf("filled from something unreadable; log:\n%s", b.logs.String()) + } +} + +func TestAStaleViewAnswerOfAnotherAskingFillsNothing(t *testing.T) { // 1297/B10 (stale answer) + b := newBed(t) + assigner, view := withView(b) + key, value, _ := holdAsk(t) + assigner[key] = value + stale, err := delivery.EncodeViewAnswer(delivery.ViewAnswer{Asking: "an-earlier-asking", Option: "release"}) + if err != nil { + t.Fatal(err) + } + view[key+".view"] = stale + b.t.OnViewAnswer(key + ".view") + b.t.OnAssignerAsk(key) + if b.answers.writes != 0 || !strings.Contains(b.logs.String(), "an-earlier-asking") { + t.Errorf("writes %d; log:\n%s", b.answers.writes, b.logs.String()) + } +} diff --git a/modules/mesh-delivery-teller/go.mod b/modules/mesh-delivery-teller/go.mod index 7138f28..f64ffb6 100644 --- a/modules/mesh-delivery-teller/go.mod +++ b/modules/mesh-delivery-teller/go.mod @@ -3,7 +3,7 @@ module mesh-delivery-teller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 ) diff --git a/modules/mesh-delivery-teller/go.sum b/modules/mesh-delivery-teller/go.sum index ddee607..2a3e9f3 100644 --- a/modules/mesh-delivery-teller/go.sum +++ b/modules/mesh-delivery-teller/go.sum @@ -1,7 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69 h1:kvtV/SPsdRVUL/BZ3xxGS6U8YrXy/t7TRBjVrD6VLw8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261019230201-5786ba190c69/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef h1:xy3wuGWJpqMAXYdXolocYG6w9xnWqBltVUffoaziIKo= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261020060201-88581d0acaef/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468 h1:UcMee+IV2ii5E4dRZdQUaSaeh9IedTFYumuQbBiB0k0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261023150321-a2c813a6b468/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/mesh-delivery-teller/manifest_test.go b/modules/mesh-delivery-teller/manifest_test.go index 438964f..c13bd86 100644 --- a/modules/mesh-delivery-teller/manifest_test.go +++ b/modules/mesh-delivery-teller/manifest_test.go @@ -81,7 +81,8 @@ func TestManifestDeclarations(t *testing.T) { // 1240/AC1 if want := []state{{Name: "answers", History: 5}, {Name: "asked", History: 1}}; !reflect.DeepEqual(m.State, want) { t.Errorf("state = %+v, want %+v", m.State, want) } - if want := []string{"mesh-delivery-gate.asks"}; !reflect.DeepEqual(sorted(m.Reads), want) { + // The assigner's asks and the view's answers to them are read so the view's answer fills an ask that needs no proof (1297/B10). + if want := []string{"mesh-delivery-assigner.asks", "mesh-delivery-gate.asks", "mesh-delivery-view.answers"}; !reflect.DeepEqual(sorted(m.Reads), want) { t.Errorf("reads = %v, want %v", m.Reads, want) } if want := []string{"operator-channel"}; !reflect.DeepEqual(sorted(m.Uses), want) { diff --git a/modules/mesh-delivery-teller/module.json b/modules/mesh-delivery-teller/module.json index 3c616f5..2ec994e 100644 --- a/modules/mesh-delivery-teller/module.json +++ b/modules/mesh-delivery-teller/module.json @@ -7,7 +7,9 @@ { "name": "asked", "history": 1 } ], "reads": [ - "mesh-delivery-gate.asks" + "mesh-delivery-assigner.asks", + "mesh-delivery-gate.asks", + "mesh-delivery-view.answers" ], "uses": [ "operator-channel" diff --git a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main.go b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main.go index fc78740..d5c1e29 100644 --- a/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main.go +++ b/modules/mesh-delivery-view/cmd/mesh-delivery-cli/main.go @@ -1,5 +1,6 @@ // Command mesh-delivery-cli is the operator's command for the delivery view (novox/hq ADR 0429 §3). It asks the -// view's local socket for the rows and prints them; it holds no credential and changes nothing. +// view's local socket for the rows and prints them; it holds no credential. `answer