From 1ed7e3415522f50e2738c90f8b0d2f7f23b9ccf1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:28:13 +0200 Subject: [PATCH 01/38] Read the builder's keys, the indexer's values by commit and its manifests, so the controller can take a module from a done key (issue 1851, A1, A2) --- internal/builderkeys/builderkeys.go | 278 ++++++++++++++++++ .../builderkeys_integration_test.go | 138 +++++++++ internal/builderkeys/builderkeys_test.go | 86 ++++++ 3 files changed, 502 insertions(+) create mode 100644 internal/builderkeys/builderkeys.go create mode 100644 internal/builderkeys/builderkeys_integration_test.go create mode 100644 internal/builderkeys/builderkeys_test.go diff --git a/internal/builderkeys/builderkeys.go b/internal/builderkeys/builderkeys.go new file mode 100644 index 00000000..b931defd --- /dev/null +++ b/internal/builderkeys/builderkeys.go @@ -0,0 +1,278 @@ +// Package builderkeys is how the controller reads what the builder and the indexer wrote (novox/hq ADR 0501, ADR 0498 +// §8, ADR 0453 §5): a module's build at a fingerprint, in the builder's bucket; the indexer's trunk value of a module, +// which names that fingerprint; and a manifest by its digest. From R4e the controller asks no build but the builder's +// own module's: it takes every other module from the builder's done key, and it reads them here. +// +// It only reads. The builder writes its keys and the indexer its values; the controller's account reaches every +// bucket through the JetStream API it is granted (internal/broker: KindController). +package builderkeys + +import ( + "context" + "errors" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" +) + +// Key is a module's build at a fingerprint as the builder's bucket holds it. Found is false when there is no key (or +// no bucket yet): nothing built or queued that fingerprint. Revision is the key's revision, a new one for every +// rebuild of the same fingerprint (ADR 0429 §18). +type Key struct { + Module string + Fingerprint string + Found bool + Revision uint64 + Build delivery.BuilderBuild +} + +// Done says the key is done. +func (k Key) Done() bool { return k.Found && k.Build.State == delivery.BuildDone } + +// Failed says the key failed; its Error is the builder's own words (`digests differ:`, `not stored:`, …). +func (k Key) Failed() bool { return k.Found && k.Build.State == delivery.BuildFailed } + +// Unreferenced are the artifacts of a done key that name no reference (ADR 0498 §8): a key done before R4e's +// delivery, or a candidate's image or package not yet built again for the trunk. Such a key is waited for, never +// delivered from. +func (k Key) Unreferenced() []string { + var out []string + for _, a := range k.Build.Artifacts { + if strings.TrimSpace(a.Reference) == "" { + out = append(out, a.Name) + } + } + return out +} + +// Says is the key's state in words: what a plan waiting on it says. +func (k Key) Says() string { + if !k.Found { + return fmt.Sprintf("no build of %s at %s yet", k.Module, short(k.Fingerprint)) + } + switch k.Build.State { + case delivery.BuildFailed: + return fmt.Sprintf("the build of %s at %s failed: %s", k.Module, short(k.Fingerprint), firstLine(k.Build.Error)) + case delivery.BuildDone: + if missing := k.Unreferenced(); len(missing) > 0 { + return fmt.Sprintf("the build of %s at %s is done and waits for the reference of %s", k.Module, + short(k.Fingerprint), strings.Join(missing, ", ")) + } + return fmt.Sprintf("the build of %s at %s is done", k.Module, short(k.Fingerprint)) + } + if k.Build.By != "" { + return fmt.Sprintf("the build of %s at %s is %s on %s", k.Module, short(k.Fingerprint), k.Build.State, k.Build.By) + } + return fmt.Sprintf("the build of %s at %s is %s", k.Module, short(k.Fingerprint), k.Build.State) +} + +// ErrNoValue is a module the indexer holds no value of at the commit asked: not written yet, or older than the +// bucket's history. +var ErrNoValue = errors.New("the indexer holds no value of the module at that commit") + +// Reader reads the builder's and the indexer's buckets. +type Reader struct { + JS jetstream.JetStream +} + +// bucket is the named bucket, or nil when it is not there yet (the part that writes it not delivered). +func (r Reader) bucket(ctx context.Context, name string) (jetstream.KeyValue, error) { + kv, err := r.JS.KeyValue(ctx, name) + if errors.Is(err, jetstream.ErrBucketNotFound) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("the bucket %s: %w", name, err) + } + return kv, nil +} + +// Build is the builder's key of module at fingerprint fp (`src2:` and 64 hex digits). A missing key, or a missing +// bucket, is a Key not Found and no error; a value that does not decode is an error naming the key. +func (r Reader) Build(ctx context.Context, module, fp string) (Key, error) { + out := Key{Module: module, Fingerprint: fp} + key, err := delivery.BuilderBuildKey(module, fp) + if err != nil { + return out, err + } + kv, err := r.bucket(ctx, delivery.BuilderBuildsBucket) + if err != nil || kv == nil { + return out, err + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return out, nil + } + if err != nil { + return out, fmt.Errorf("the builder's key %s: %w", key, err) + } + b, err := delivery.DecodeBuilderBuild(e.Value()) + if err != nil { + return out, fmt.Errorf("the builder's key %s is not read: %w", key, err) + } + out.Found, out.Revision, out.Build = true, e.Revision(), b + return out, nil +} + +// ModuleAt is the indexer's trunk value of module written for commit (ADR 0501 §2): the newest revision of +// modules. whose Commit is commit, read from the key's history; with commit empty, the value it holds now. +// None held is ErrNoValue. +func (r Reader) ModuleAt(ctx context.Context, module, commit string) (delivery.Module, error) { + key, err := delivery.ModulesKey(module) + if err != nil { + return delivery.Module{}, err + } + kv, err := r.bucket(ctx, delivery.ModulesBucket) + if err != nil { + return delivery.Module{}, err + } + if kv == nil { + return delivery.Module{}, fmt.Errorf("%w (%s: the indexer's bucket is not there)", ErrNoValue, module) + } + var values [][]byte + if commit == "" { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return delivery.Module{}, fmt.Errorf("%w (%s)", ErrNoValue, module) + } + if err != nil { + return delivery.Module{}, fmt.Errorf("the indexer's value of %s: %w", module, err) + } + values = [][]byte{e.Value()} + } else { + entries, err := kv.History(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return delivery.Module{}, fmt.Errorf("%w (%s at %s)", ErrNoValue, module, short(commit)) + } + if err != nil { + return delivery.Module{}, fmt.Errorf("the indexer's values of %s: %w", module, err) + } + for _, e := range entries { + if e.Operation() == jetstream.KeyValuePut { + values = append(values, e.Value()) + } + } + } + return pickAt(module, commit, values) +} + +// pickAt is, of a key's values oldest first, the newest whose commit is commit (any, with commit empty); values that +// do not decode are passed over, and one of them is said when nothing else is found. +func pickAt(module, commit string, values [][]byte) (delivery.Module, error) { + var unread error + for i := len(values) - 1; i >= 0; i-- { + m, err := delivery.DecodeModule(values[i]) + if err != nil { + if unread == nil { + unread = err + } + continue + } + if commit == "" || sameCommit(m.Commit, commit) { + return m, nil + } + } + where := module + if commit != "" { + where += " at " + short(commit) + } + if unread != nil { + return delivery.Module{}, fmt.Errorf("%w (%s; a value is not read: %v)", ErrNoValue, where, unread) + } + return delivery.Module{}, fmt.Errorf("%w (%s)", ErrNoValue, where) +} + +// sameCommit is two commit ids naming one commit: equal, or one a prefix of the other of at least 7 hex digits. +func sameCommit(a, b string) bool { + if a == b { + return a != "" + } + if len(a) < 7 || len(b) < 7 { + return false + } + return strings.HasPrefix(a, b) || strings.HasPrefix(b, a) +} + +// Manifest is the indexer's manifest of digest: the module and its module.json byte for byte. +func (r Reader) Manifest(ctx context.Context, digest string) (delivery.Manifest, error) { + key, err := delivery.ManifestsKey(digest) + if err != nil { + return delivery.Manifest{}, err + } + kv, err := r.bucket(ctx, delivery.ManifestsBucket) + if err != nil { + return delivery.Manifest{}, err + } + if kv == nil { + return delivery.Manifest{}, fmt.Errorf("the manifest %s: the indexer's manifests bucket is not there", short(digest)) + } + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + return delivery.Manifest{}, fmt.Errorf("the indexer holds no manifest %s", short(digest)) + } + if err != nil { + return delivery.Manifest{}, fmt.Errorf("the manifest %s: %w", short(digest), err) + } + m, err := delivery.DecodeManifest(e.Value()) + if err != nil { + return delivery.Manifest{}, fmt.Errorf("the manifest %s is not read: %w", short(digest), err) + } + return m, nil +} + +// Watch calls wake with the module of every key the builder or the indexer writes from now on, until ctx ends: what +// wakes a plan waiting on a key, with no timer (ADR 0501; the R4e context's A7). A bucket not there yet is watched once +// it is: the watch looks again each time wake would have been called for the other bucket, and at every retry. +func Watch(ctx context.Context, js jetstream.JetStream, wake func(module string)) error { + for _, name := range []string{delivery.BuilderBuildsBucket, delivery.ModulesBucket} { + if err := watchOne(ctx, js, name, wake); err != nil { + return err + } + } + return nil +} + +func watchOne(ctx context.Context, js jetstream.JetStream, name string, wake func(module string)) error { + kv, err := js.KeyValue(ctx, name) + if err != nil { + return fmt.Errorf("the bucket %s: %w", name, err) + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + return fmt.Errorf("watching %s: %w", name, err) + } + go func() { + defer w.Stop() + for { + select { + case <-ctx.Done(): + return + case e, ok := <-w.Updates(): + if !ok { + return + } + if e == nil || e.Operation() != jetstream.KeyValuePut { + continue + } + module, _, _ := strings.Cut(e.Key(), ".") + wake(module) + } + } + }() + return nil +} + +func short(s string) string { + s = strings.TrimPrefix(s, "src2:") + if len(s) > 12 { + return s[:12] + } + return s +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(strings.TrimSpace(s), "\n") + return line +} diff --git a/internal/builderkeys/builderkeys_integration_test.go b/internal/builderkeys/builderkeys_integration_test.go new file mode 100644 index 00000000..80352ad1 --- /dev/null +++ b/internal/builderkeys/builderkeys_integration_test.go @@ -0,0 +1,138 @@ +//go:build integration + +package builderkeys + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBus(t *testing.T) jetstream.JetStream { + t.Helper() + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + return js +} + +func put(t *testing.T, kv jetstream.KeyValue, key string, value []byte) { + t.Helper() + if _, err := kv.Put(context.Background(), key, value); err != nil { + t.Fatal(err) + } +} + +// The controller reads the builder's keys, the indexer's values by commit and its manifests (A1, A2), and a key the +// builder writes wakes it with the key's module (A1's watch, no timer). +func TestTheControllerReadsTheBuildersKeysAndTheIndexersValues(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + js := aBus(t) + r := Reader{JS: js} + + // Nothing there yet: no bucket is no key and no error; no value is ErrNoValue. + if k, err := r.Build(ctx, "x", fp); err != nil || k.Found { + t.Fatalf("no bucket: %+v %v", k, err) + } + if _, err := r.ModuleAt(ctx, "x", "abcdef1"); !errors.Is(err, ErrNoValue) { + t.Fatalf("no bucket: %v", err) + } + + builds, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.BuilderBuildsBucket, + History: delivery.BuilderBuildsHistory}) + if err != nil { + t.Fatal(err) + } + modules, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ModulesBucket, + History: delivery.ModulesHistory}) + if err != nil { + t.Fatal(err) + } + manifests, err := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: delivery.ManifestsBucket}) + if err != nil { + t.Fatal(err) + } + woke := make(chan string, 8) + if err := Watch(ctx, js, func(m string) { woke <- m }); err != nil { + t.Fatal(err) + } + + raw := `{"module":"x","version":"1"}` + digest := delivery.ManifestDigest([]byte(raw)) + value, err := delivery.EncodeManifest(delivery.Manifest{Module: "x", JSON: raw}) + if err != nil { + t.Fatal(err) + } + put(t, manifests, digest, value) + + key, _ := delivery.BuilderBuildKey("x", fp) + building, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace"}) + put(t, builds, key, building) + select { + case m := <-woke: + if m != "x" { + t.Fatalf("woken for %q", m) + } + case <-ctx.Done(): + t.Fatal("the builder's write woke nothing") + } + done, _ := delivery.EncodeBuilderBuild(delivery.BuilderBuild{State: delivery.BuildDone, Manifest: digest, + Artifacts: []delivery.BuiltArtifact{{Name: "a", Kind: "bundle", Digest: "sha256:1", + Reference: "artifact-store://x/a/blobs/sha256:1"}}}) + put(t, builds, key, done) + k, err := r.Build(ctx, "x", fp) + if err != nil || !k.Done() || len(k.Unreferenced()) != 0 || k.Build.Artifacts[0].Reference == "" { + t.Fatalf("the done key: %+v %v", k, err) + } + m, err := r.Manifest(ctx, k.Build.Manifest) + if err != nil || m.JSON != raw { + t.Fatalf("the manifest: %+v %v", m, err) + } + + other := "src2:" + strings.Repeat("e", 64) + for _, v := range []struct{ commit, fp2 string }{{"1111111111", other}, {"2222222222", fp}, {"3333333333", other}} { + raw, err := delivery.EncodeModule(delivery.Module{FP: "src1:" + strings.Repeat("a", 64), FP2: v.fp2, + Manifest: digest, Repository: "novox/r", Commit: v.commit}) + if err != nil { + t.Fatal(err) + } + put(t, modules, "x", raw) + } + at, err := r.ModuleAt(ctx, "x", "2222222222") + if err != nil || at.FP2 != fp { + t.Fatalf("FP2 at the plan's commit, overwritten by a later merge: %+v %v", at, err) + } + if now, err := r.ModuleAt(ctx, "x", ""); err != nil || now.Commit != "3333333333" { + t.Fatalf("the value now: %+v %v", now, err) + } + if _, err := r.ModuleAt(ctx, "x", "9999999999"); !errors.Is(err, ErrNoValue) { + t.Fatalf("a commit the indexer never wrote: %v", err) + } +} + +// The controller's account reaches the builder's and the indexer's buckets: it is granted the JetStream API whole. +func TestTheControllersAccountReachesTheBuckets(t *testing.T) { + perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(perms.Publish, "$JS.API.>") || !slices.Contains(perms.Subscribe, "$JS.API.>") { + t.Fatalf("the controller is not granted the JetStream API: %v / %v", perms.Publish, perms.Subscribe) + } +} diff --git a/internal/builderkeys/builderkeys_test.go b/internal/builderkeys/builderkeys_test.go new file mode 100644 index 00000000..47a82a4b --- /dev/null +++ b/internal/builderkeys/builderkeys_test.go @@ -0,0 +1,86 @@ +package builderkeys + +import ( + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +const fp = "src2:" + "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func moduleValue(t *testing.T, commit, fp2 string) []byte { + t.Helper() + raw, err := delivery.EncodeModule(delivery.Module{FP: "src1:" + strings.Repeat("a", 64), FP2: fp2, + Manifest: strings.Repeat("b", 64), Repository: "novox/mesh-catalog", Commit: commit}) + if err != nil { + t.Fatal(err) + } + return raw +} + +// A plan's module is the indexer's value written for the merge the plan carries, not a later merge's (ADR 0501 §2). +func TestPickAtIsTheNewestValueOfTheCommit(t *testing.T) { + other := "src2:" + strings.Repeat("c", 64) + values := [][]byte{ + moduleValue(t, "1111111111", other), + moduleValue(t, "2222222222", other), + moduleValue(t, "2222222222", fp), // the same merge, computed again when a toolchain appeared + moduleValue(t, "3333333333", other), + } + m, err := pickAt("x", "2222222222", values) + if err != nil { + t.Fatal(err) + } + if m.FP2 != fp { + t.Fatalf("FP2 at 2222222222 is %s, want the newest of that commit %s", m.FP2, fp) + } + // An abbreviated commit names the same one. + if m, err := pickAt("x", "2222222", values); err != nil || m.FP2 != fp { + t.Fatalf("an abbreviated commit: %v %v", m.FP2, err) + } + // No commit is the newest value. + if m, err := pickAt("x", "", values); err != nil || m.Commit != "3333333333" { + t.Fatalf("the newest: %v %v", m.Commit, err) + } +} + +func TestPickAtSaysNoValueOfACommitNotWritten(t *testing.T) { + _, err := pickAt("x", "4444444444", [][]byte{moduleValue(t, "1111111111", fp), []byte("{not json")}) + if !errors.Is(err, ErrNoValue) { + t.Fatalf("want ErrNoValue, got %v", err) + } + if !strings.Contains(err.Error(), "x at 4444444444") || !strings.Contains(err.Error(), "not read") { + t.Fatalf("the words name neither the module and commit nor the unread value: %v", err) + } +} + +// A done key with references is delivered from; a done key with an artifact without one is waited for, said +// (ADR 0498 §8); a failed key says the builder's own words. +func TestAKeySaysWhatItIs(t *testing.T) { + missing := Key{Module: "x", Fingerprint: fp} + if missing.Done() || missing.Failed() || !strings.Contains(missing.Says(), "no build of x") { + t.Fatalf("a missing key: %q", missing.Says()) + } + done := Key{Module: "x", Fingerprint: fp, Found: true, Build: delivery.BuilderBuild{State: delivery.BuildDone, + Artifacts: []delivery.BuiltArtifact{ + {Name: "a", Digest: "sha256:1", Reference: "artifact-store://x/a/blobs/sha256:1"}, + {Name: "img", Kind: "image", Digest: "sha256:2"}, + }}} + if !done.Done() || len(done.Unreferenced()) != 1 || done.Unreferenced()[0] != "img" { + t.Fatalf("unreferenced: %v", done.Unreferenced()) + } + if !strings.Contains(done.Says(), "waits for the reference of img") { + t.Fatalf("a done key without a reference: %q", done.Says()) + } + failed := Key{Module: "x", Fingerprint: fp, Found: true, Build: delivery.BuilderBuild{State: delivery.BuildFailed, + Error: "digests differ: ace sha256:1, g14 sha256:2\nmore"}} + if !failed.Failed() || !strings.HasSuffix(failed.Says(), "failed: digests differ: ace sha256:1, g14 sha256:2") { + t.Fatalf("a failed key: %q", failed.Says()) + } + building := Key{Module: "x", Fingerprint: fp, Found: true, Build: delivery.BuilderBuild{State: delivery.BuildBuilding, By: "ace"}} + if building.Done() || !strings.Contains(building.Says(), "building on ace") { + t.Fatalf("a building key: %q", building.Says()) + } +} From eff0d5cbf442a6a7ad140391d343e462bbbe92bf Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:30:05 +0200 Subject: [PATCH 02/38] Vendor the R4e mesh-sdk member's head for its shared launcher rule and the manifest word paths (issue 1851; re-pinned to the final head in E) --- go.mod | 2 +- go.sum | 6 ++- testdata/beside/CAPTURED | 2 +- .../novox/mesh-sdk/go/catalogue/build.go | 39 +++++++++++++++++++ .../novox/mesh-sdk/go/catalogue/manifest.go | 11 ++++++ vendor/modules.txt | 2 +- 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index fe3cabf2..fde4f61c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022132648-bd7731f3588c + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 0fc0fa3b..c8522d52 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,9 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022132648-bd7731f3588c h1:UKlwMw6fmesDqe0aL0lfI/ivhorsGjAvh9XefFjhNI4= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022132648-bd7731f3588c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c h1:YdSbkI42rlVmClybH8BS717ut0vouKIMavO691fpdps= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b h1:cfJ2LK/etUiV1eL58pnaEjmSi34+EVmO4ud9J7pXKOs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index 1dfe88c4..d14a5a7b 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk bd7731f3588cf336620526b032bbfe9867f7837e conformance/events/module-event.json +mesh-sdk dfc2c858d94ba877caa96b5e0b84200590c1391f conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go index e36de7f5..f4fcd25b 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go @@ -394,6 +394,45 @@ func (b *Build) problems(module string) []string { "%s: %q mirrors %q, which names no tag or digest", module, a.Name, a.From)) } problems = append(problems, upstreamFileProblems(module, a)...) + problems = append(problems, pathsProblems(module, a)...) + } + return problems +} + +// pathsProblems is what is wrong with an artifact's `paths` (novox/hq ADR 0499 clause 1): named on a kind whose build +// source is derived or named otherwise, or a path that leaves the module's directory, is its whole directory, or is a +// pattern. +func pathsProblems(module string, a Artifact) []string { + if a.Paths == nil { + return nil + } + switch { + case a.Kind == ArtifactBundle && a.Language == "go": + return []string{fmt.Sprintf("%s: %q is a Go program and names paths: its build source is derived from the "+ + "package it compiles (novox/hq ADR 0267 clause 1, ADR 0499 clause 1)", module, a.Name)} + case a.Kind != ArtifactBundle && a.Kind != ArtifactPackage: + return []string{fmt.Sprintf("%s: %q is a %q and names paths; only a package or a bundle in a language other "+ + "than Go names the paths it is built from (novox/hq ADR 0499 clause 1)", module, a.Name, a.Kind)} + case len(a.Paths) == 0: + return []string{fmt.Sprintf("%s: %q names no path in paths; leave the word out to build from the module's "+ + "directory", module, a.Name)} + } + var problems []string + for _, p := range a.Paths { + clean := path.Clean(p) + switch { + case strings.TrimSpace(p) == "": + problems = append(problems, fmt.Sprintf("%s: %q names an empty path", module, a.Name)) + case strings.HasPrefix(p, "/") || clean == ".." || strings.HasPrefix(clean, "../"): + problems = append(problems, fmt.Sprintf("%s: %q names the path %q, which leaves the module's directory", + module, a.Name, p)) + case clean == ".": + problems = append(problems, fmt.Sprintf("%s: %q names the module's whole directory (%q) as a path; leave "+ + "the word out for that", module, a.Name, p)) + case strings.ContainsAny(p, "*?[]{}\\"): + problems = append(problems, fmt.Sprintf("%s: %q names %q, which is a pattern; paths names files and "+ + "directories", module, a.Name, p)) + } } return problems } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go index 870ea4d0..0c721c74 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/manifest.go @@ -1191,6 +1191,17 @@ type Artifact struct { // (novox/hq ADR 0193). Absent for nearly every bundle, which is then one file per entrypoint. External []string `json:"external,omitempty"` + // Paths are the files and directories, relative to the module's directory, a package or a bundle in a language + // other than Go is built from (novox/hq ADR 0499 clause 1): with the module's manifest they are its build source, + // and a module whose every artifact names them takes its fingerprint's tree from them alone (clause 2). Absent means + // the module's directory, as before. + // + // **Named, because the build source of a package sharing a repository cannot be derived.** A Go program's is + // (ADR 0267 clause 1), an archive's `from` names it and an image's is the context its manifest names, so each of those + // refuses the word. A path stays inside the module's directory and is no pattern: a build source that could reach + // out of it, or grow with a directory listing, would be one no fingerprint pins. + Paths []string `json:"paths,omitempty"` + // Env is what a tools bundle is given on a machine (novox/hq ADR 0192): words and their values, // paths and constants composed with ${dir:…} and ${port:…} exactly as a container's environment // is, never a secret's content. The node's runtime hands it to this bundle and to no other. diff --git a/vendor/modules.txt b/vendor/modules.txt index 6ae05ecf..30efdfb1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022132648-bd7731f3588c +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go git.novox.be/novox/mesh-sdk/go/asks From f00b6aff8b9f1d7bbd739665d8e702ecc5dbbfc7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:39:48 +0200 Subject: [PATCH 03/38] Judge a recorded build, not build-agent's outcome, and read a registration's warrant from the merge a key names, so a done key registers under the same rules (issue 1851, A4a) --- cmd/mesh-controller/build.go | 154 ++++++++++++------ cmd/mesh-controller/build_source.go | 49 +++++- cmd/mesh-controller/forge_address_test.go | 4 +- cmd/mesh-controller/moved.go | 21 ++- .../new_module_approved_test.go | 4 +- 5 files changed, 161 insertions(+), 71 deletions(-) diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index b7077f1b..f78f79f1 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -679,13 +679,6 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu recorded.Ref = followedBranch(was.Ref) } } - if err := namesNoInstallation(manifest); err != nil { - return manifest, kept, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w", - result.On, result.Repository, short(result.Commit), err) - } - // **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk - // below is the trunk of whatever repository was built, which may be one an agent made — and a module named - // `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal. trunk := result.Trunk if was, err := inv.SourceOf(ctx, manifest.Module); err == nil && followedBranch(was.Ref) != "" { trunk = followedBranch(was.Ref) @@ -693,55 +686,111 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu if trunk == "" { trunk = "main" } - repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, recorded, result.ID, result.Path, trunk) + j := takenBuild{kept: kept, manifest: manifest, recorded: recorded, ask: byBuild(result.ID), path: result.Path, + trunk: trunk, by: result.On, what: result.Repository, commit: result.Commit} + // The checks only an outcome of build-agent needs: a key is a trunk build by construction (ADR 0501 §2), in no + // scratch namespace, and carries no list of its own. + j.published = func() error { + // **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's + // default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay + // --register` of one — is for checking, and is never a module's version; nothing could then send it. + // The branch the module already follows is its trunk — never the branch a build was asked at, or a + // build of a feature branch by name would make that branch its trunk. + follows := "" + if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { + follows = followedBranch(was.Ref) + } + if err := publishable(result, follows); err != nil { + return err + } + // **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is + // registered only as the walk's copy in the module's own namespace. + if scratch := inScratch(result.Made); scratch != "" { + return fmt.Errorf("%s is in the scratch namespace, which only a walk's copy leaves (ADR 0350)", scratch) + } + return nil + } + j.carried = func() error { + // **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the + // catalogue that may be registered keeps its list when its commit is newer than the one kept. + carried, err := carriedList(result) + if err != nil { + return err + } + if carried != nil { + if err := publishPredefined(ctx, inv, result, carried); err != nil { + fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err) + } + } + return nil + } + manifest, err = registerJudged(ctx, inv, j) + return manifest, kept, err +} + +// takenBuild is a build the controller judges and registers, whatever told it of the build: build-agent's outcome (the +// builder's own module's, until R4g, ADR 0453 §5) or the builder's done key (ADR 0501 §1). **One judging for both**: +// the rules of who may register, the gate a build failed, the predefined keys and the rule that an unchanged build +// source moves nothing are the same whichever part built it (ADR 0501 §1, §3). +type takenBuild struct { + kept inventory.Build + manifest catalogue.Manifest + recorded inventory.Source + ask registrationAsk + path string + trunk string + // by and what name the build in what is said: who built it, and what was built ("ace", "novox/mesh-catalog"). + by, what, commit string + // published refuses a build that may not become a module's version (off the trunk, in the scratch namespace); + // nil for a key, which is a trunk fingerprint's by construction (ADR 0501 §2). + published func() error + // carried publishes a list of predefined keys the build carried; nil when it carried none. + carried func() error +} + +// registerJudged judges a recorded build and registers its module, or says why not. +func registerJudged(ctx context.Context, inv *inventory.Inventory, j takenBuild) (catalogue.Manifest, error) { + manifest, kept := j.manifest, j.kept + notRegistered := func(err error) (catalogue.Manifest, error) { + return manifest, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", j.by, manifest.Module, + short(j.commit), err) + } + if err := namesNoInstallation(manifest); err != nil { + return manifest, fmt.Errorf("%s built %s (%s), and the mesh does not register it: %w", + j.by, j.what, short(j.commit), err) + } + // **Only from the repository the catalogue builds the module from** (novox/hq ADR 0266): else the trunk + // below is the trunk of whatever repository was built, which may be one an agent made — and a module named + // `sudo` from it would be what the next push sends. Another repository is the operator's, at the terminal. + repoID, err := mayRegisterFrom(ctx, inv, manifest.Module, j.recorded, j.ask, j.path, j.trunk) if err != nil { - return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, - manifest.Module, short(result.Commit), err) + return notRegistered(err) } - // **Only a commit on the trunk is published** (novox/hq ADR 0238): a commit off its repository's - // default branch — a pull request's head, a feature branch built by hand, a `rebuild` or `replay - // --register` of one — is for checking, and is never a module's version; nothing could then send it. - // The branch the module already follows is its trunk — never the branch a build was asked at, or a - // build of a feature branch by name would make that branch its trunk. - follows := "" - if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { - follows = followedBranch(was.Ref) - } - if err := publishable(result, follows); err != nil { - return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", result.On, - manifest.Module, short(result.Commit), err) - } - // **Never a build in the scratch namespace** (novox/hq ADR 0239 §9, ADR 0350 §2): what a check builds is - // registered only as the walk's copy in the module's own namespace. - if scratch := inScratch(result.Made); scratch != "" { - return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %s is in the scratch "+ - "namespace, which only a walk's copy leaves (ADR 0350)", result.On, manifest.Module, short(result.Commit), scratch) + if j.published != nil { + if err := j.published(); err != nil { + return notRegistered(err) + } } // **A build that failed its gate is never registered again** (novox/hq ADR 0236): an outcome heard // twice, or replayed, would otherwise make the build a rollback put back what the module is again, // and the next push would send it. if failed, err := inv.GateFailed(ctx, kept.ID); err != nil { - return manifest, kept, err + return manifest, err } else if failed { - return manifest, kept, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+ - "back: it is recorded and not registered again — a newer build is", result.On, manifest.Module, - short(result.Commit)) + return manifest, fmt.Errorf("%s built %s (%s), which failed its gate on its first machine and was put "+ + "back: it is recorded and not registered again — a newer build is", j.by, manifest.Module, + short(j.commit)) } - // **The newest trunk build's list of predefined keys is published** (novox/hq ADR 0348 §7): a build of the - // catalogue that may be registered keeps its list when its commit is newer than the one kept. Then the module - // is judged against the list the mesh publishes, as every manifest read again is: one naming a key that list - // does not hold, or of another type, is refused here and never enters the catalogue. - if carried, err := carriedList(result); err != nil { - return manifest, kept, err - } else if carried != nil { - if err := publishPredefined(ctx, inv, result, carried); err != nil { - fmt.Printf("%s: the list of predefined keys it carried is not published: %v\n", result.ID, err) + // Then the module is judged against the list the mesh publishes, as every manifest read again is: one naming a + // key that list does not hold, or of another type, is refused here and never enters the catalogue. + if j.carried != nil { + if err := j.carried(); err != nil { + return manifest, err } } if manifest.NamesPredefined() { if _, err := catalogue.ParseManifest(kept.Manifest); err != nil { - return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: judged against the "+ - "list of predefined keys the mesh publishes, %w", result.On, manifest.Module, short(result.Commit), err) + return notRegistered(fmt.Errorf("judged against the list of predefined keys the mesh publishes, %w", err)) } } // **A build whose source is unchanged is never a move** (novox/hq issue 280): a rebuild made from @@ -751,35 +800,34 @@ func takeIn(ctx context.Context, inv *inventory.Inventory, result link.BuildResu if kept.SourceFingerprint != "" { stands, raw, err := inv.StandingBuild(ctx, manifest.Module, kept.ID) if err != nil { - return manifest, kept, err + return manifest, err } if stands != "" && stands != kept.ID && len(raw) > 0 { - if same, err := catalogue.ParseManifest(raw); err == nil && same.Module == manifest.Module { + if was, err := catalogue.ParseManifest(raw); err == nil && was.Module == manifest.Module { fmt.Printf("%s at %s was made from the source %s was: registered with its artifacts, no move\n", - manifest.Module, short(result.Commit), stands) - manifest = same + manifest.Module, short(j.commit), stands) + manifest = was } } } - if err := inv.RegisterModule(ctx, manifest, recorded); err != nil { + if err := inv.RegisterModule(ctx, manifest, j.recorded); err != nil { if errors.Is(err, inventory.ErrSuperseded) { - return manifest, kept, fmt.Errorf("%s built %s (%s), recorded and not registered: %w", - result.On, manifest.Module, short(result.Commit), err) + return notRegistered(err) } - return manifest, kept, err + return manifest, err } // A bus process the new definition gains gets its account now, on each machine assigned the module, so the // next push composes a real account (issue 1226). issueGainedProcessAccounts(ctx, inv, manifest) // Which repository it is registered from, by the forge's own id (novox/hq ADR 0266). if err := inv.SetSourceIdentity(ctx, manifest.Module, repoID); err != nil { - return manifest, kept, err + return manifest, err } // The keep set just moved, and new bytes just landed (novox/hq ADR 0189). Asked here rather // than on a timer of its own: this is the only moment either is true. Never fatal — the build // worked and the module is registered. collect(ctx, inv) - return manifest, kept, nil + return manifest, nil } // inScratch is the first artifact a build made into the scratch namespace; empty when none is. diff --git a/cmd/mesh-controller/build_source.go b/cmd/mesh-controller/build_source.go index a58292cd..0c3d84ea 100644 --- a/cmd/mesh-controller/build_source.go +++ b/cmd/mesh-controller/build_source.go @@ -469,8 +469,39 @@ func (f *sourceForms) onTheForge(repository, seat string) (owner, name string, o // repository and path; or, for the build of a merge, when mesh-delivery announced it on the operator's Approve of // that head (novox/hq ADR 0420 §7 for a moved module, ADR 0432 for a new one), the forge's checks still holding. path is the module's directory as built; branch the trunk it is registered from. func mayRegisterFrom(ctx context.Context, inv *inventory.Inventory, module string, built inventory.Source, - buildID, path, branch string) (int64, error) { - return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, buildID, path, branch) + ask registrationAsk, path, branch string) (int64, error) { + return mayRegisterIn(ctx, inv, newSourceForms(ctx, inv), module, built, ask, path, branch) +} + +// registrationAsk is what a registration's warrant is read from (novox/hq ADR 0501 §3): the build an outcome answers, +// whose kept request says whether it was asked at the terminal or for a merge (build-agent's outcome, the builder's own +// module until R4g); or, for a take-in from the builder's done key, the merge the indexer's trunk value names and +// whether the operator asked the registration at the controller's terminal (ADR 0501 §4). A key asks no build, so it +// keeps no request. +type registrationAsk struct { + // Build is the build whose kept request is read; empty for a take-in from a key. + Build string + // Merge is the commit of the merge a key's module is registered from (the indexer's trunk value); empty for an + // outcome, whose request names it. + Merge string + // Terminal is a key registration the operator asked at the controller's terminal for this repository and path. + Terminal bool +} + +// byBuild is the warrant of a build's outcome: read from the request kept under its id. +func byBuild(id string) registrationAsk { return registrationAsk{Build: id} } + +// String is the ask as the registration's words name it. +func (a registrationAsk) String() string { + switch { + case a.Build != "": + return a.Build + case a.Terminal: + return "the registration asked at the terminal" + case a.Merge != "": + return fmt.Sprintf("the merge %.8s", a.Merge) + } + return "a key" } // catalogueReader is the catalogue's manifests, as the router's records are found in them. @@ -490,15 +521,19 @@ type registrations interface { // mayRegisterIn is mayRegisterFrom on what it reads of the mesh handed in. func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, module string, built inventory.Source, - buildID, path, branch string) (int64, error) { + ask registrationAsk, path, branch string) (int64, error) { + buildID := ask.String() was, err := inv.SourceOf(ctx, module) isNew := errors.Is(err, inventory.ErrNoSuchModule) if err != nil && !isNew { return 0, err } - terminal, err := askedHereFor(ctx, inv, forms, buildID, built, path) - if err != nil { - return 0, err + terminal := ask.Terminal + if ask.Build != "" { + var err error + if terminal, err = askedHereFor(ctx, inv, forms, ask.Build, built, path); err != nil { + return 0, err + } } refuse := func(why string) (int64, error) { return 0, fmt.Errorf("%w: %s. A module is registered from such a source only by a build asked at the "+ @@ -542,7 +577,7 @@ func mayRegisterIn(ctx context.Context, inv registrations, forms *sourceForms, m // The forge's protection and the repository's identity are checked below as for every registration. movedWhy := "" if why != "" && !terminal { - accepted, err := approvedMove(ctx, inv, forms, buildID, built) + accepted, err := approvedMove(ctx, inv, forms, ask, built) switch { case err == nil: movedWhy, why = accepted, "" diff --git a/cmd/mesh-controller/forge_address_test.go b/cmd/mesh-controller/forge_address_test.go index ea50d84f..a793bb06 100644 --- a/cmd/mesh-controller/forge_address_test.go +++ b/cmd/mesh-controller/forge_address_test.go @@ -64,7 +64,7 @@ func registerFrom(t *testing.T, forms *sourceForms) (int64, error) { mesh := &aMeshRegisteredFromTheURL{anEmptyMesh{asked: true, request: inventory.BuildRequest{ID: "build-1", Repository: aRepoOnTheForge, For: "build"}}, aRepoOnTheForge} return mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", inventory.Source{Repository: aRepoOnTheForge}, - "build-1", "", "main") + byBuild("build-1"), "", "main") } // 1277/AC2: an address unread twice, then read, is waited out and the forge is asked. @@ -112,7 +112,7 @@ func TestAKnownAddressAndAnotherHostIsStillNotOnTheForge(t *testing.T) { const elsewhere = "http://elsewhere.example/x/y.git" mesh := &aMeshRegisteredFromTheURL{anEmptyMesh{asked: true, request: inventory.BuildRequest{ID: "build-1", For: "build"}}, elsewhere} _, err := mayRegisterIn(t.Context(), mesh, sourceFormsRead(context.Background(), time.Now, read), "mesh-delivery-view", - inventory.Source{Repository: elsewhere}, "build-1", "", "main") + inventory.Source{Repository: elsewhere}, byBuild("build-1"), "", "main") if !errors.Is(err, errNotItsSource) || !strings.Contains(err.Error(), "is not on the mesh's forge") { t.Fatalf("1277/AC4: the refusal changed: %v", err) } diff --git a/cmd/mesh-controller/moved.go b/cmd/mesh-controller/moved.go index f84c23bb..d1e97efc 100644 --- a/cmd/mesh-controller/moved.go +++ b/cmd/mesh-controller/moved.go @@ -265,25 +265,32 @@ var errNotAMerge = errors.New("the build is not of a merge") // (ADR 0420 §7): the build is of a merge, that merge is a pull request mesh-delivery announced it made, and the // ask it made it on holds an Approve by the operator of that head. Answers why it was accepted; an error says why // it was not, and the refusal of ADR 0266 stands. -func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, buildID string, +func approvedMove(ctx context.Context, inv registrations, forms *sourceForms, ask registrationAsk, built inventory.Source) (string, error) { - r, found, err := inv.BuildRequestByID(ctx, buildID) - if err != nil { - return "", err + commit := ask.Merge + if ask.Build != "" { + r, found, err := inv.BuildRequestByID(ctx, ask.Build) + if err != nil { + return "", err + } + if !found || r.For != "merge" || r.Commit == "" { + return "", errNotAMerge + } + commit = r.Commit } - if !found || r.For != "merge" || r.Commit == "" { + if commit == "" { return "", errNotAMerge } owner, name, onForge := forms.onTheForge(built.Repository, built.Seat) if !onForge { return "", errors.New("the repository is not on the mesh's forge") } - merged, known, err := inv.MergeOf(ctx, owner+"/"+name, r.Commit) + merged, known, err := inv.MergeOf(ctx, owner+"/"+name, commit) if err != nil { return "", err } if !known { - return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, r.Commit) + return "", fmt.Errorf("the controller kept no merge of %s/%s at %.8s", owner, name, commit) } return approvedAnnounced(ctx, inv, owner, name, announcedOf(merged)) } diff --git a/cmd/mesh-controller/new_module_approved_test.go b/cmd/mesh-controller/new_module_approved_test.go index f5293efe..58ba371d 100644 --- a/cmd/mesh-controller/new_module_approved_test.go +++ b/cmd/mesh-controller/new_module_approved_test.go @@ -116,7 +116,7 @@ func TestANewModuleIsRegisteredOnTheOperatorsApproveOfItsMergeAndRefusedOtherwis if c.built != nil { src = *c.built } - id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, "build-1", "", "main") + id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-view", src, byBuild("build-1"), "", "main") switch { case c.accepted && err != nil: t.Fatalf("%s: the Approved merge's new module was refused: %v", c.ac, err) @@ -157,7 +157,7 @@ func TestASecondNewModuleFromARepositoryIsJudgedAlongsideAndAsksNoWarrant(t *tes request: inventory.BuildRequest{ID: "build-2", Repository: "novox/mesh-delivery", Seat: "git", For: "build"}}} forms := &sourceForms{bases: map[string]string{}, base: func(string) string { return "http://forge.internal:20000" }} built := inventory.Source{Repository: "novox/mesh-delivery", Seat: "git"} - if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, "build-2", "", "main"); err != nil || id == 0 { + if id, err := mayRegisterIn(t.Context(), mesh, forms, "mesh-delivery-watchman", built, byBuild("build-2"), "", "main"); err != nil || id == 0 { t.Fatalf("1146/AC5: a second module from a repository the catalogue builds from was refused (id %d): %v", id, err) } } From d984f0a55d3f6cb823d3c7f87cff97312002e5bb Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:39:48 +0200 Subject: [PATCH 04/38] Take a module in from the builder's done key of its trunk fingerprint: one record of the key's artifacts, its manifest judged against the catalogue trunk's predefined keys (issue 1851, A3, A4b, A4c) --- cmd/mesh-controller/key_takein.go | 427 ++++++++++++++++++ .../key_takein_integration_test.go | 192 ++++++++ cmd/mesh-controller/key_takein_test.go | 231 ++++++++++ internal/artifacts/blob.go | 57 +++ internal/artifacts/blob_test.go | 44 ++ internal/builder/builder.go | 31 +- internal/buildreads/buildreads.go | 39 ++ .../novox/mesh-sdk/go/buildstep/fetch.go | 114 +++++ .../novox/mesh-sdk/go/buildstep/guard.go | 52 +++ .../novox/mesh-sdk/go/buildstep/launchers.go | 24 + .../novox/mesh-sdk/go/buildstep/merge.go | 112 +++++ .../novox/mesh-sdk/go/buildstep/pack.go | 100 ++++ .../novox/mesh-sdk/go/buildstep/refusal.go | 13 + .../novox/mesh-sdk/go/buildstep/step.go | 99 ++++ .../novox/mesh-sdk/go/buildstep/trees.go | 67 +++ .../novox/mesh-sdk/go/forgekey/keykeeper.go | 127 ++++++ .../novox/mesh-sdk/go/forgekey/keys.go | 110 +++++ .../novox/mesh-sdk/go/forgekey/runner.go | 93 ++++ vendor/modules.txt | 2 + 19 files changed, 1907 insertions(+), 27 deletions(-) create mode 100644 cmd/mesh-controller/key_takein.go create mode 100644 cmd/mesh-controller/key_takein_integration_test.go create mode 100644 cmd/mesh-controller/key_takein_test.go create mode 100644 internal/artifacts/blob.go create mode 100644 internal/artifacts/blob_test.go create mode 100644 internal/buildreads/buildreads.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go diff --git a/cmd/mesh-controller/key_takein.go b/cmd/mesh-controller/key_takein.go new file mode 100644 index 00000000..435cfacb --- /dev/null +++ b/cmd/mesh-controller/key_takein.go @@ -0,0 +1,427 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "path" + "strings" + "time" + + "git.novox.be/novox/mesh-sdk/go/buildstep" + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/artifacts" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/buildreads" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// The take-in from the builder's done key (novox/hq ADR 0501, ADR 0498 §8, ADR 0453 §5; issue 1851). +// +// From R4e the controller asks no build but the builder's own module's: a module of a merge is taken from the builder's +// key for the fingerprint the indexer's trunk value of the module names (ADR 0501 §2). Only a done key whose every +// artifact names a reference is taken (ADR 0498 §8); a key not done yet, one waiting for a reference, a templates +// archive the store does not answer, or a list of predefined keys the forge does not answer is waited for and said, +// never failed and never delivered from. A failed key fails, in the builder's own words. The take-in then records the +// key as a build of today's record layout, so a controller of before reads it on a rollback, and judges it as every +// build is judged (registerJudged). + +// errKeyWaits says a take-in from a key waits: for the builder, a reference, the store or the forge. Never a failure. +var errKeyWaits = errors.New("waits") + +// errKeyFailed says the module's key failed: the builder's own words follow. +var errKeyFailed = errors.New("the builder's key failed") + +// keyReads is what a take-in from a key reads of the builder and the indexer (builderkeys.Reader, or a test's). +type keyReads interface { + Build(ctx context.Context, module, fp string) (builderkeys.Key, error) + Manifest(ctx context.Context, digest string) (sdkdelivery.Manifest, error) +} + +// keyBuildID is the id a key's build is recorded under: the key and its revision, so the same revision recorded twice +// is one record, and a rebuild of the same fingerprint (a new revision, ADR 0429 §18) another. +func keyBuildID(k builderkeys.Key) string { + part, err := sdkdelivery.FingerprintKeyPart(k.Fingerprint) + if err != nil { + part = strings.TrimPrefix(k.Fingerprint, sdkdelivery.SourcePrefix2) + } + return fmt.Sprintf("key-%s-%d", part, k.Revision) +} + +// isKeyBuildID says a recorded build was taken from a builder's key. +func isKeyBuildID(id string) bool { return strings.HasPrefix(id, "key-") } + +// readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A +// variable so a test needs no store. +var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) { + shelf, err := inv.Catalogue(ctx) + if err != nil { + return nil, err + } + address, err := artifactStoreAddress(ctx, inv, shelf, "") + if err != nil { + return nil, err + } + if address == "" { + return nil, errors.New("the artifact store is on no network the controller reaches") + } + return artifacts.Store{Address: address}.GetBlob(ctx, reference) +} + +// predefinedHead is the list of predefined keys at the head of the catalogue repository's trunk, and that head. +type predefinedHead struct { + List catalogue.PredefinedList + Repository string + Commit string + // Generation orders two heads of the trunk: the head's commit time in seconds. A list kept from a build's + // generation (its commit count) is always older than a head read here. + Generation int +} + +// predefinedAtTrunk is the list of predefined keys at the head of the catalogue repository's trunk (ADR 0501 §8), read +// from the forge. A variable so a test needs no forge. +var predefinedAtTrunk = func(ctx context.Context, inv *inventory.Inventory) (predefinedHead, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return predefinedHead{}, err + } + repository := "" + for _, e := range entries { + if e.Source.Seat == gitSeat && path.Base(strings.Trim(e.Source.Repository, "/")) == catalogue.CatalogueRepository { + repository = strings.Trim(e.Source.Repository, "/") + break + } + } + if repository == "" { + return predefinedHead{}, fmt.Errorf("no module of the catalogue names the repository %s on the forge", + catalogue.CatalogueRepository) + } + var heads []struct { + SHA string `json:"sha"` + Commit struct { + Committer struct { + Date time.Time `json:"date"` + } `json:"committer"` + } `json:"commit"` + } + if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + + "/commits?limit=1&stat=false&verification=false&files=false"}, &heads); err != nil { + return predefinedHead{}, err + } + if len(heads) == 0 || heads[0].SHA == "" { + return predefinedHead{}, fmt.Errorf("the forge named no head of %s's trunk", repository) + } + head := predefinedHead{Repository: repository, Commit: heads[0].SHA, + Generation: int(heads[0].Commit.Committer.Date.Unix())} + var file struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + } + if err := askGitea(ctx, "gitea_api", map[string]any{"path": "/repos/" + repository + "/contents/" + + catalogue.PredefinedFile + "?ref=" + head.Commit}, &file); err != nil { + return predefinedHead{}, err + } + if file.Encoding != "base64" { + return predefinedHead{}, fmt.Errorf("the forge answered %s at %.8s in the encoding %q", catalogue.PredefinedFile, + head.Commit, file.Encoding) + } + raw, err := base64.StdEncoding.DecodeString(file.Content) + if err != nil { + return predefinedHead{}, err + } + if head.List, err = catalogue.ParsePredefined(raw); err != nil { + return predefinedHead{}, fmt.Errorf("%s at %.8s is refused: %w", catalogue.PredefinedFile, head.Commit, err) + } + return head, nil +} + +// askGitea asks the forge's module one tool on the bus and reads its result into into. +func askGitea(ctx context.Context, tool string, args map[string]any, into any) error { + js, err := aBus() + if err != nil { + return err + } + defer js.Close() + raw, _ := json.Marshal(args) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", tool, raw, 30*time.Second) + if err != nil { + return err + } + if answer.Error != "" { + return fmt.Errorf("gitea.%s: %s", tool, answer.Error) + } + var wrapped struct { + Result json.RawMessage `json:"result"` + } + if err := json.Unmarshal(answer.Result, &wrapped); err != nil { + return err + } + return json.Unmarshal(wrapped.Result, into) +} + +// keyTake is one take-in from a key: the module, the indexer's trunk value naming the key's fingerprint, and the +// registration's warrant. +type keyTake struct { + Module string + Trunk sdkdelivery.Module + Ask registrationAsk +} + +// takeInKey takes a module in from the builder's key for the fingerprint its trunk value names: records the key as a +// build and registers the module, or says why not. errKeyWaits when it waits, errKeyFailed when the key failed. +func takeInKey(ctx context.Context, inv *inventory.Inventory, keys keyReads, t keyTake) ( + catalogue.Manifest, inventory.Build, error) { + head, err := predefinedAtTrunk(ctx, inv) + if err != nil { + return catalogue.Manifest{}, inventory.Build{}, fmt.Errorf("%w: the list of predefined keys at the catalogue's "+ + "trunk head cannot be read, so %s is not judged: %v", errKeyWaits, t.Module, err) + } + keepPredefinedHead(ctx, inv, head) + previous, err := inv.Builds(ctx, t.Module, 5) + if err != nil { + return catalogue.Manifest{}, inventory.Build{}, err + } + kept, manifest, err := buildOfKey(ctx, keys, t, keyContext{list: head.List, previous: previous, + archive: func(ctx context.Context, reference string) ([]byte, error) { return readArchive(ctx, inv, reference) }}) + if err != nil { + return catalogue.Manifest{}, kept, err + } + if err := inv.RecordBuild(ctx, kept); err != nil { + return catalogue.Manifest{}, kept, err + } + recorded := inventory.Source{Repository: t.Trunk.Repository, Seat: gitSeat, Path: keyPath(t.Trunk), + BuiltFrom: t.Trunk.Commit, Head: t.Trunk.Commit, Against: kept.Against} + trunk := "main" + if was, err := inv.SourceOf(ctx, manifest.Module); err == nil { + recorded.Ref = followedBranch(was.Ref) + if recorded.Ref != "" { + trunk = recorded.Ref + } + } + registered, err := registerJudged(ctx, inv, takenBuild{kept: kept, manifest: manifest, recorded: recorded, + ask: t.Ask, path: recorded.Path, trunk: trunk, by: "the builder", what: t.Trunk.Repository, + commit: t.Trunk.Commit}) + return registered, kept, err +} + +// keyContext is what a key's build is made with besides the key: the list of predefined keys its manifest is judged +// against, the module's recent builds (the bases its last one stood on are carried) and how an archive is read. +type keyContext struct { + list catalogue.PredefinedList + previous []inventory.Build + archive func(ctx context.Context, reference string) ([]byte, error) +} + +// keyPath is a module's directory as the catalogue records it: empty for a module at its repository's root. +func keyPath(m sdkdelivery.Module) string { + if m.Path == sdkdelivery.RootPath { + return "" + } + return strings.Trim(m.Path, "/") +} + +// buildOfKey reads the key of the trunk value's fingerprint and makes its build record and the manifest it resolves to +// (ADR 0498 §8, ADR 0501 §1, §8): the module's manifest by the key's digest, judged against the list of predefined keys +// at the catalogue's trunk head, resolved with the key's artifacts. +func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) ( + inventory.Build, catalogue.Manifest, error) { + waits := func(format string, a ...any) (inventory.Build, catalogue.Manifest, error) { + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: "+format, append([]any{errKeyWaits}, a...)...) + } + if t.Trunk.FP2 == "" { + why := t.Trunk.None2 + if why == "" { + why = "the indexer wrote no fingerprint of the new form" + } + return waits("%s at %.8s has no fingerprint to take a build by: %s", t.Module, t.Trunk.Commit, why) + } + k, err := keys.Build(ctx, t.Module, t.Trunk.FP2) + if err != nil { + return waits("%s's key cannot be read: %v", t.Module, err) + } + switch { + case k.Failed(): + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("%w: %s", errKeyFailed, k.Says()) + case !k.Done(): + return waits("%s", k.Says()) + case len(k.Unreferenced()) > 0: + return waits("%s", k.Says()) + } + src, err := keys.Manifest(ctx, k.Build.Manifest) + if err != nil { + return waits("%s's manifest: %v", t.Module, err) + } + manifest, err := catalogue.ParseManifestWith([]byte(src.JSON), kc.list) + if err != nil { + // **A word the running controller does not read waits for the controller that reads it** (ADR 0501 §7): the + // key stays done, and the controller that reads the word registers it with no second build. + if word := refusedWord(err.Error()); word != "" { + return waits("%s's manifest uses the word %q, which this controller does not read: it is registered "+ + "when a controller that reads it runs", t.Module, word) + } + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and its manifest is "+ + "refused: %w", t.Module, err) + } + if manifest.Module != t.Module { + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the key of %s names the manifest of %s", + t.Module, manifest.Module) + } + built, err := builtOfKey(ctx, kc.archive, manifest, k) + if err != nil { + return waits("%v", err) + } + resolved, err := manifest.Resolve(built) + if err != nil { + return inventory.Build{}, catalogue.Manifest{}, fmt.Errorf("the builder built %s and it does not resolve: %w", + t.Module, err) + } + raw, err := json.Marshal(resolved) + if err != nil { + return inventory.Build{}, catalogue.Manifest{}, err + } + kept := inventory.Build{ID: keyBuildID(k), Repository: t.Trunk.Repository, Module: t.Module, + Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw, + SourceFingerprint: k.Fingerprint, At: time.Now().UTC()} + // **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are + // left unsaid: the planner then reads the module's whole directory, over-planning and never under-planning, and a + // fingerprint that did not change moves nothing (R4e A9). + for _, c := range buildreads.Of(manifest) { + kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref}) + } + // **The bases its last build stood on, carried** (R4e, lead decision): what the collector keeps a mirrored base by. + // A key names none; the module's newest successful build's are carried so the keep set does not change. + for _, b := range kc.previous { + if b.Worked() && b.Module == t.Module && b.ID != kept.ID { + kept.Against = append([]string(nil), b.Against...) + break + } + } + for _, a := range k.Build.Artifacts { + kept.Made = append(kept.Made, inventory.Artifact{Name: a.Name, Kind: kindOf(manifest, a), + Reference: catalogue.Recorded(a.Reference)}) + } + return kept, resolved, nil +} + +// kindOf is an artifact's kind: the key's, else the manifest's. +func kindOf(m catalogue.Manifest, a sdkdelivery.BuiltArtifact) string { + if a.Kind != "" || m.Build == nil { + return a.Kind + } + for _, d := range m.Build.Artifacts { + if d.Name == a.Name { + return d.Kind + } + } + return "" +} + +// builtOfKey is each of a done key's artifacts as Resolve reads it: its reference and digest, a TypeScript bundle's +// launchers by the build modules' one rule (mesh-sdk buildstep.Launchers), and, for the archive a module packs its +// templates into, each template's text read back from the archive the store holds under the key's reference. +func builtOfKey(ctx context.Context, archive func(context.Context, string) ([]byte, error), m catalogue.Manifest, + k builderkeys.Key) ( + []catalogue.Built, error) { + declared := map[string]catalogue.Artifact{} + if m.Build != nil { + for _, a := range m.Build.Artifacts { + declared[a.Name] = a + } + } + templates := "" + if m.Templated() { + templates = catalogue.TemplatesArtifact(m) + } + var out []catalogue.Built + for _, a := range k.Build.Artifacts { + d := declared[a.Name] + b := catalogue.Built{Name: a.Name, Kind: kindOf(m, a), Reference: catalogue.Recorded(a.Reference), + Digest: a.Digest} + if launchers := buildstep.Launchers(d.Language, d.Entrypoints); len(launchers) > 0 { + b.Launchers = launchers + } + if a.Name == templates && templates != "" { + body, err := archive(ctx, b.Reference) + if err != nil { + return nil, fmt.Errorf("the templates archive %q of %s cannot be read from the artifact store: %v", + a.Name, m.Module, err) + } + if b.Texts, err = unpackTexts(body); err != nil { + return nil, fmt.Errorf("the templates archive %q of %s cannot be read back: %v", a.Name, m.Module, err) + } + } + out = append(out, b) + } + return out, nil +} + +// unpackTexts is every file of a packed templates archive (gzip over tar), by its path inside it. +func unpackTexts(body []byte) (map[string]string, error) { + zipped, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer zipped.Close() + reader := tar.NewReader(zipped) + texts := map[string]string{} + for { + header, err := reader.Next() + if errors.Is(err, io.EOF) { + return texts, nil + } + if err != nil { + return nil, err + } + if header.Typeflag != tar.TypeReg { + continue + } + text, err := io.ReadAll(io.LimitReader(reader, 16<<20)) + if err != nil { + return nil, err + } + texts[strings.TrimPrefix(header.Name, "./")] = string(text) + } +} + +// keepPredefinedHead keeps the list read at the catalogue's trunk head when it is newer than the one kept, as a build of +// the catalogue's did (ADR 0348 §7), and judges every manifest read again against it. Said, never fatal: the list read +// is the one this take-in judges against either way. +func keepPredefinedHead(ctx context.Context, inv *inventory.Inventory, head predefinedHead) { + kept, err := inv.KeepPredefined(ctx, inventory.PredefinedRecord{Keys: head.List, Repository: head.Repository, + Commit: head.Commit, Generation: head.Generation, Build: "the trunk head " + short(head.Commit)}) + if err != nil { + fmt.Printf("the list of predefined keys at %s's trunk head %s is not kept: %v\n", head.Repository, + short(head.Commit), err) + return + } + if kept { + fmt.Printf("the list of predefined keys at %s's trunk head %s is published: %s\n", head.Repository, + short(head.Commit), strings.Join(head.List.Keys(), ", ")) + } + if err := loadPredefined(ctx, inv); err != nil { + fmt.Printf("the list of predefined keys kept cannot be read: %v\n", err) + } +} + +// openBuilderKeys is the builder's and the indexer's buckets, read on a connection for the moment. +func openBuilderKeys() (builderkeys.Reader, func(), error) { + js, err := aBus() + if err != nil { + return builderkeys.Reader{}, nil, err + } + api, err := jetstream.New(js.Conn()) + if err != nil { + js.Close() + return builderkeys.Reader{}, nil, err + } + return builderkeys.Reader{JS: api}, js.Close, nil +} diff --git a/cmd/mesh-controller/key_takein_integration_test.go b/cmd/mesh-controller/key_takein_integration_test.go new file mode 100644 index 00000000..39df46e7 --- /dev/null +++ b/cmd/mesh-controller/key_takein_integration_test.go @@ -0,0 +1,192 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// withTrunkList stands in for the catalogue's trunk head: an empty list of predefined keys, readable, or not. +func withTrunkList(t *testing.T, readable bool) { + t.Helper() + was := predefinedAtTrunk + predefinedAtTrunk = func(context.Context, *inventory.Inventory) (predefinedHead, error) { + if !readable { + return predefinedHead{}, errors.New("the forge did not answer") + } + list, err := catalogue.ParsePredefined([]byte(`{}`)) + return predefinedHead{List: list, Repository: "novox/mesh-catalog", Commit: "abcdef0123456789", + Generation: 1 << 40}, err + } + t.Cleanup(func() { predefinedAtTrunk = was }) +} + +// aKeyFor is a done key of module at fp, of one archive with a reference, and the indexer's trunk value naming it at +// commit in repository/path. +func aKeyFor(module, version, fp, repository, path, commit string) (keyBuckets, sdkdelivery.Module) { + raw := `{"module":"` + module + `","version":"` + version + `","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` + + `"resources":[{"id":"a","type":"archive","path":"/opt/` + module + `","artifact":"a"}]}` + digest := sdkdelivery.ManifestDigest([]byte(raw)) + blob := shaOf(module + version) + k := builderkeys.Key{Module: module, Fingerprint: fp, Found: true, Revision: 1, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, + Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", Kind: "archive", Digest: blob, + Reference: "artifact-store://" + module + "/a/blobs/" + blob}}}} + return keyBuckets{keys: map[string]builderkeys.Key{module: k}, manifests: map[string]string{digest: raw}}, + sdkdelivery.Module{FP2: fp, Manifest: digest, Repository: repository, Commit: commit, Path: path} +} + +func fpOf(c byte) string { return "src2:" + strings.Repeat(string(c), 64) } + +// A3, A4 (ADR 0498 §8, ADR 0501 §1-§3, §8): a done key of a trunk fingerprint records one build of today's layout, +// whose artifacts are the key's, and registers the module under the same rules of who may register as before. +func TestADoneKeyOfTheTrunkRegistersUnderTheSameRules(t *testing.T) { + open := theCatalogue(t) // sudo from novox/mesh-catalog modules/sudo, mesh-host from novox/mesh-host + ctx := t.Context() + inv := open.inventory + withTrunkList(t, true) + + // A known module from the repository and path the catalogue builds it from: registered from its key. + keys, trunk := aKeyFor("sudo", "2", fpOf('a'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + _, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if err != nil { + t.Fatalf("0501 §1: a done key of sudo's trunk fingerprint was not taken in: %v", err) + } + if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "2" { + t.Fatalf("0501 §1: sudo is %q", shelf["sudo"].Version) + } + got, found, err := inv.BuildByID(ctx, kept.ID) + if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference || + got.SourceFingerprint != fpOf('a') { + t.Fatalf("0498 §8: the recorded build is %+v (%v)", got, err) + } + // Recording the same key again is one record. + if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil { + t.Fatal(err) + } + builds, _ := inv.Builds(ctx, "sudo", 20) + n := 0 + for _, b := range builds { + if b.ID == kept.ID { + n++ + } + } + if n != 1 { + t.Fatalf("0498 §8: one key taken in twice is %d records", n) + } + + // A known module from another repository: refused, as a build of it was (ADR 0266). + other, otherTrunk := aKeyFor("sudo", "evil", fpOf('b'), "agent/sudo", "modules/sudo", "d00d0123456789ab") + if _, _, err := takeInKey(ctx, inv, other, keyTake{Module: "sudo", Trunk: otherTrunk, + Ask: registrationAsk{Merge: otherTrunk.Commit}}); !errors.Is(err, errNotItsSource) { + t.Fatalf("0501 §3: a known module from another repository was taken in: %v", err) + } + // A new module from a repository the catalogue builds from: registered. + zram, zramTrunk := aKeyFor("zram", "1", fpOf('c'), "novox/mesh-catalog", "modules/zram", "c0ffee0123456789") + if _, _, err := takeInKey(ctx, inv, zram, keyTake{Module: "zram", Trunk: zramTrunk, Ask: registrationAsk{Merge: zramTrunk.Commit}}); err != nil { + t.Fatalf("0501 §3: a new module from a built repository was refused: %v", err) + } + // A new module from a repository nothing is built from, with no Approve: refused. + tools, toolsTrunk := aKeyFor("agent-tools", "1", fpOf('d'), "agent/tools", "", "e0e0e0e0e0e0e0e0") + if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk, + Ask: registrationAsk{Merge: toolsTrunk.Commit}}); !errors.Is(err, errNotItsSource) { + t.Fatalf("0501 §3: a new repository's module was taken in with no warrant: %v", err) + } + // At the terminal the operator registers it (ADR 0501 §4). + if _, _, err := takeInKey(ctx, inv, tools, keyTake{Module: "agent-tools", Trunk: toolsTrunk, + Ask: registrationAsk{Terminal: true}}); err != nil { + t.Fatalf("0501 §4: the terminal's registration was refused: %v", err) + } +} + +// ADR 0501 §2: a key only a candidate names registers nothing: the trunk value names another fingerprint, whose key is +// not there, and the take-in waits. +func TestAKeyOnlyACandidateNamesRegistersNothing(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + withTrunkList(t, true) + keys, trunk := aKeyFor("sudo", "3", fpOf('e'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + trunk.FP2 = fpOf('f') // the trunk names another fingerprint than the candidate's done key + _, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if !errors.Is(err, errKeyWaits) { + t.Fatalf("0501 §2: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" { + t.Fatalf("0501 §2: a candidate's key registered sudo %q", shelf["sudo"].Version) + } +} + +// ADR 0501 §8: a take-in that cannot read the list of predefined keys at the catalogue's trunk head waits, says so and +// registers nothing. +func TestATakeInWithoutTheTrunksListWaits(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + withTrunkList(t, false) + keys, trunk := aKeyFor("sudo", "4", fpOf('1'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + _, _, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "predefined keys") { + t.Fatalf("0501 §8: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["sudo"].Version != "1" { + t.Fatalf("0501 §8: registered %q", shelf["sudo"].Version) + } +} + +// A3 (the lead's decision on Against and Read): a key's record keeps the collector's keep set as it was and its read +// repositories are its manifest's, fresh at every key. +func TestAKeysRecordKeepsTheKeepSetAndReadsItsManifest(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + withTrunkList(t, true) + before, err := inv.KeepSet(ctx) + if err != nil { + t.Fatal(err) + } + raw := `{"module":"sudo","version":"5","build":{"artifacts":[{"name":"img","kind":"image","from":"Dockerfile",` + + `"context":{"repository":"novox/elsewhere","seat":"git"}}]},"resources":[{"id":"run","type":"container","artifact":"img"}]}` + digest := sdkdelivery.ManifestDigest([]byte(raw)) + k := builderkeys.Key{Module: "sudo", Fingerprint: fpOf('2'), Found: true, Revision: 1, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "g14", Manifest: digest, + Artifacts: []sdkdelivery.BuiltArtifact{{Name: "img", Kind: "image", Digest: shaOf("img5"), + Reference: "artifact-store://sudo/img@" + shaOf("img5")}}}} + keys := keyBuckets{keys: map[string]builderkeys.Key{"sudo": k}, manifests: map[string]string{digest: raw}} + trunk := sdkdelivery.Module{FP2: fpOf('2'), Manifest: digest, Repository: "novox/mesh-catalog", + Commit: "c0ffee0123456789", Path: "modules/sudo"} + if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil { + t.Fatal(err) + } + after, err := inv.KeepSet(ctx) + if err != nil { + t.Fatal(err) + } + for reference, why := range before { + if !reflect.DeepEqual(after[reference], why) && !strings.Contains(strings.Join(after[reference], ","), strings.Join(why, ",")) { + t.Errorf("the keep set changed for %s: %v, was %v", reference, after[reference], why) + } + } + read, err := inv.ReadRepositories(ctx) + if err != nil { + t.Fatal(err) + } + found := false + for _, r := range read["sudo"] { + if !r.Own && r.Repository == "novox/elsewhere" { + found = true + } + } + if !found { + text, _ := json.Marshal(read["sudo"]) + t.Fatalf("the key's record does not read its manifest's context: %s", text) + } +} diff --git a/cmd/mesh-controller/key_takein_test.go b/cmd/mesh-controller/key_takein_test.go new file mode 100644 index 00000000..37928d49 --- /dev/null +++ b/cmd/mesh-controller/key_takein_test.go @@ -0,0 +1,231 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "strings" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// keyBuckets is the builder's and the indexer's buckets as a take-in reads them, for a test: keys by module, manifests by +// digest. +type keyBuckets struct { + keys map[string]builderkeys.Key + manifests map[string]string +} + +func (k keyBuckets) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { + if key, ok := k.keys[module]; ok && key.Fingerprint == fp { + return key, nil + } + return builderkeys.Key{Module: module, Fingerprint: fp}, nil +} + +func (k keyBuckets) Manifest(_ context.Context, digest string) (sdkdelivery.Manifest, error) { + raw, ok := k.manifests[digest] + if !ok { + return sdkdelivery.Manifest{}, errors.New("the indexer holds no manifest " + digest) + } + return sdkdelivery.Manifest{Module: "x", JSON: raw}, nil +} + +const keyFP = "src2:" + "1111111111111111111111111111111111111111111111111111111111111111" + +// aKeyOf is a done key of module x at keyFP for the manifest raw, with these artifacts. +func aKeyOf(raw string, revision uint64, made ...sdkdelivery.BuiltArtifact) (keyBuckets, sdkdelivery.Module) { + digest := sdkdelivery.ManifestDigest([]byte(raw)) + k := builderkeys.Key{Module: "x", Fingerprint: keyFP, Found: true, Revision: revision, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, By: "ace", Manifest: digest, Artifacts: made}} + trunk := sdkdelivery.Module{FP2: keyFP, Manifest: digest, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789", + Path: "modules/x"} + return keyBuckets{keys: map[string]builderkeys.Key{"x": k}, manifests: map[string]string{digest: raw}}, trunk +} + +const tsBundle = `{"module":"x","version":"1","tools":["x_read"], + "build":{"artifacts":[{"name":"tools","kind":"bundle","language":"typescript","entrypoints":["dist/index.js"]}, + {"name":"img","kind":"image","from":"Dockerfile","context":{"repository":"novox/other","seat":"git"}}]}, + "resources":[{"id":"run","type":"container","artifact":"img"}]}` + +func shaOf(b string) string { + sum := sha256.Sum256([]byte(b)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// A3 (ADR 0498 §8, ADR 0501 §1): a done key whose every artifact names a reference is one recorded build, its artifacts +// the key's, its launchers by the build modules' rule, its reads from its manifest, the bases its last build stood on +// carried; recorded twice it is one record. +func TestADoneKeyIsOneRecordedBuildOfItsArtifacts(t *testing.T) { + keys, trunk := aKeyOf(tsBundle, 3, + sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), + Reference: "artifact-store://x/tools/blobs/" + shaOf("t")}, + sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i"), + Reference: "artifact-store://x/img@" + shaOf("i")}) + previous := []inventory.Build{{ID: "build-1", Module: "x", Against: []string{"artifact-store://base@sha256:aa"}}} + take := keyTake{Module: "x", Trunk: trunk} + kept, resolved, err := buildOfKey(t.Context(), keys, take, keyContext{previous: previous}) + if err != nil { + t.Fatal(err) + } + if kept.ID != "key-1111111111111111111111111111111111111111111111111111111111111111-3" { + t.Errorf("the record's id is %s", kept.ID) + } + again, _, _ := buildOfKey(t.Context(), keys, take, keyContext{previous: previous}) + if again.ID != kept.ID { + t.Errorf("one key recorded twice is two records: %s and %s", kept.ID, again.ID) + } + if kept.SourceFingerprint != keyFP || kept.Commit != trunk.Commit || kept.Path != "modules/x" || kept.On != "ace" || + kept.Module != "x" || kept.Repository != "novox/mesh-catalog" { + t.Errorf("the record: %+v", kept) + } + if len(kept.Made) != 2 || kept.Made[0].Reference != "artifact-store://x/tools/blobs/"+shaOf("t") || + kept.Made[1].Reference != "artifact-store://x/img@"+shaOf("i") || kept.Made[1].Kind != "image" { + t.Errorf("Made is not the key's artifacts: %+v", kept.Made) + } + if len(kept.Read) != 1 || kept.Read[0].Repository != "novox/other" { + t.Errorf("Read is not the manifest's contexts: %+v", kept.Read) + } + if kept.Sources != nil || kept.Mirrored != nil { + t.Errorf("a key's record says build sources or mirrored bases it cannot know: %+v %+v", kept.Sources, kept.Mirrored) + } + if len(kept.Against) != 1 || kept.Against[0] != "artifact-store://base@sha256:aa" { + t.Errorf("the bases the last build stood on are not carried: %v", kept.Against) + } + if len(resolved.Bundles) != 1 || resolved.Bundles[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" { + t.Errorf("the bundle's launchers: %+v", resolved.Bundles) + } + var m map[string]any + if err := json.Unmarshal(kept.Manifest, &m); err != nil || !strings.Contains(string(kept.Manifest), shaOf("i")) { + t.Errorf("the recorded manifest names no image digest: %s", kept.Manifest) + } +} + +// A3 (ADR 0498 §8): a key with an artifact naming no reference, a key not done and a missing key are waited for, said; +// a failed key fails in the builder's words; a module with no fingerprint of the new form waits, naming why. +func TestAKeyNotReadyWaitsAndAFailedKeyFails(t *testing.T) { + noRef, trunk := aKeyOf(tsBundle, 1, + sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: shaOf("t"), + Reference: "artifact-store://x/tools/blobs/" + shaOf("t")}, + sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: shaOf("i")}) + take := keyTake{Module: "x", Trunk: trunk} + _, _, err := buildOfKey(t.Context(), noRef, take, keyContext{}) + if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), "waits for the reference of img") { + t.Errorf("a key without a reference: %v", err) + } + building := noRef + k := building.keys["x"] + k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"} + building.keys = map[string]builderkeys.Key{"x": k} + if _, _, err := buildOfKey(t.Context(), building, take, keyContext{}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "building on g14") { + t.Errorf("a building key: %v", err) + } + if _, _, err := buildOfKey(t.Context(), keyBuckets{}, take, keyContext{}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "no build of x") { + t.Errorf("no key: %v", err) + } + k.Build = sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "digests differ: ace sha256:1, g14 sha256:2"} + failed := keyBuckets{keys: map[string]builderkeys.Key{"x": k}} + if _, _, err := buildOfKey(t.Context(), failed, take, keyContext{}); !errors.Is(err, errKeyFailed) || + !strings.Contains(err.Error(), "digests differ: ace sha256:1, g14 sha256:2") { + t.Errorf("a failed key: %v", err) + } + none := take + none.Trunk.FP2, none.Trunk.None2 = "", "no lock" + if _, _, err := buildOfKey(t.Context(), noRef, none, keyContext{}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "no lock") { + t.Errorf("no fingerprint: %v", err) + } +} + +// ADR 0501 §7: a manifest word this controller does not read waits for the controller that reads it; never fails. +func TestAManifestWordTheControllerDoesNotReadWaits(t *testing.T) { + raw := `{"module":"x","version":"1","not-a-word-yet":true,"build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]}, + "resources":[{"id":"a","type":"archive","path":"/opt/x","artifact":"a"}]}` + keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "a", Kind: "archive", Digest: shaOf("a"), + Reference: "artifact-store://x/a/blobs/" + shaOf("a")}) + _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{}) + if !errors.Is(err, errKeyWaits) || !strings.Contains(err.Error(), `"not-a-word-yet"`) { + t.Fatalf("an unread word: %v", err) + } +} + +func packed(t *testing.T, files map[string]string) []byte { + t.Helper() + var buf bytes.Buffer + z := gzip.NewWriter(&buf) + w := tar.NewWriter(z) + for name, text := range files { + if err := w.WriteHeader(&tar.Header{Name: name, Mode: 0o644, Size: int64(len(text)), Typeflag: tar.TypeReg}); err != nil { + t.Fatal(err) + } + if _, err := w.Write([]byte(text)); err != nil { + t.Fatal(err) + } + } + if err := w.Close(); err != nil { + t.Fatal(err) + } + if err := z.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +// A templated module's texts are read back from the archive the store holds under the key's reference; an archive the +// store does not answer is waited for (the lead's Q1b). +func TestATemplatedModulesTextsAreReadFromItsArchive(t *testing.T) { + raw := `{"module":"x","version":"1","templates":"templates", + "build":{"artifacts":[{"name":"templates","kind":"archive","from":"templates"}]}, + "resources":[{"id":"main","type":"file","path":"/etc/x.conf","mode":"0644","template":"templates/x.conf"}]}` + body := packed(t, map[string]string{"x.conf": "listen 1\n"}) + sum := sha256.Sum256(body) + digest := "sha256:" + hex.EncodeToString(sum[:]) + keys, trunk := aKeyOf(raw, 1, sdkdelivery.BuiltArtifact{Name: "templates", Kind: "archive", Digest: digest, + Reference: "artifact-store://x/templates/blobs/" + digest}) + asked := "" + archive := func(_ context.Context, reference string) ([]byte, error) { asked = reference; return body, nil } + _, resolved, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: archive}) + if err != nil { + t.Fatal(err) + } + if asked != "artifact-store://x/templates/blobs/"+digest { + t.Errorf("the archive asked: %q", asked) + } + if text, _ := json.Marshal(resolved.Resources); !strings.Contains(string(text), "listen 1") { + t.Errorf("the template's text is not in the file: %s", text) + } + gone := func(context.Context, string) ([]byte, error) { return nil, errors.New("the store does not hold it") } + if _, _, err := buildOfKey(t.Context(), keys, keyTake{Module: "x", Trunk: trunk}, keyContext{archive: gone}); !errors.Is(err, errKeyWaits) || + !strings.Contains(err.Error(), "cannot be read from the artifact store") { + t.Errorf("an archive the store does not answer: %v", err) + } +} + +// The launchers a key's bundle is resolved with are mesh-sdk's one rule (buildstep.Launchers), the names +// mesh-build-node writes: a TypeScript bundle's entrypoint x.js runs as x.serve.mjs, a Go bundle has none. +func TestALaunchersRuleIsTheBuildModules(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(tsBundle)) + if err != nil { + t.Fatal(err) + } + k := builderkeys.Key{Build: sdkdelivery.BuilderBuild{Artifacts: []sdkdelivery.BuiltArtifact{{Name: "tools", Digest: "sha256:1", + Reference: "artifact-store://x/tools/blobs/sha256:1"}}}} + built, err := builtOfKey(t.Context(), nil, m, k) + if err != nil { + t.Fatal(err) + } + if built[0].Launchers["dist/index.js"] != "dist/index.serve.mjs" || built[0].Kind != "bundle" { + t.Fatalf("built: %+v", built[0]) + } +} diff --git a/internal/artifacts/blob.go b/internal/artifacts/blob.go new file mode 100644 index 00000000..9f769422 --- /dev/null +++ b/internal/artifacts/blob.go @@ -0,0 +1,57 @@ +package artifacts + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" +) + +// blobBound is the most GetBlob reads: a templates archive is a few kilobytes; anything near this is no archive a +// take-in reads text from. +const blobBound = 64 << 20 + +// GetBlob is the archive a recorded reference names (`artifact-store:///blobs/sha256:`), read whole +// and checked against the digest it names: bytes that hash to another digest are refused, never read (R4e: the +// controller reads a templates archive by the reference of a builder's done key). Gone when the store does not hold +// it; ErrNotOurs for a reference outside the store or one that is not an archive. +func (s Store) GetBlob(ctx context.Context, reference string) ([]byte, error) { + repository, digest, archive, err := s.archive(reference) + if err != nil { + return nil, err + } + if !archive { + return nil, fmt.Errorf("%w: %s is not an archive", ErrNotOurs, reference) + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url(repository, "blobs", digest), nil) + if err != nil { + return nil, err + } + response, err := s.client().Do(request) + if err != nil { + return nil, fmt.Errorf("cannot reach the artifact store at %s: %w", s.Address, err) + } + defer response.Body.Close() + switch response.StatusCode { + case http.StatusOK: + case http.StatusNotFound: + return nil, Gone + default: + return nil, fmt.Errorf("the artifact store answered %s for %s/blobs/%s", response.Status, repository, digest) + } + body, err := io.ReadAll(io.LimitReader(response.Body, blobBound+1)) + if err != nil { + return nil, fmt.Errorf("reading %s/blobs/%s: %w", repository, digest, err) + } + if len(body) > blobBound { + return nil, fmt.Errorf("%s/blobs/%s is larger than %d bytes, which no archive read here is", repository, digest, blobBound) + } + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return nil, fmt.Errorf("the artifact store answered %s/blobs/%s with bytes of the digest %s: refused", repository, + digest, got) + } + return body, nil +} diff --git a/internal/artifacts/blob_test.go b/internal/artifacts/blob_test.go new file mode 100644 index 00000000..94b3b61d --- /dev/null +++ b/internal/artifacts/blob_test.go @@ -0,0 +1,44 @@ +package artifacts + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// A take-in reads a templates archive by its reference, and only bytes of the digest the reference names. +func TestGetBlobReadsOnlyTheBytesOfItsDigest(t *testing.T) { + good := []byte("the archive") + sum := sha256.Sum256(good) + digest := "sha256:" + hex.EncodeToString(sum[:]) + liar := "sha256:" + strings.Repeat("0", 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+digest), strings.HasSuffix(r.URL.Path, "/x/files/blobs/"+liar): + _, _ = w.Write(good) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer server.Close() + s := Store{Address: strings.TrimPrefix(server.URL, "http://")} + ctx := context.Background() + body, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+digest) + if err != nil || string(body) != string(good) { + t.Fatalf("the archive: %q %v", body, err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/files/blobs/"+liar); err == nil || !strings.Contains(err.Error(), "refused") { + t.Fatalf("bytes of another digest are read: %v", err) + } + if _, err := s.GetBlob(ctx, "artifact-store://x/other/blobs/"+digest); !errors.Is(err, Gone) { + t.Fatalf("a blob the store does not hold: %v", err) + } + if _, err := s.GetBlob(ctx, "registry.example/x@"+digest); !errors.Is(err, ErrNotOurs) { + t.Fatalf("a reference outside the store: %v", err) + } +} diff --git a/internal/builder/builder.go b/internal/builder/builder.go index eb5b09e3..ba962644 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -23,6 +23,7 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/buildreads" ) // Turning a repository into artifacts the mesh can pin. @@ -2001,33 +2002,9 @@ func instructions(recipe string) []string { return out } -// readBy is every repository other than the module's own that this build's recipes read source from, -// each once and in a fixed order, so two builds of one commit report the same thing the same way. -func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext { - if manifest.Build == nil { - return nil - } - seen := map[string]bool{} - var out []catalogue.ArtifactContext - for _, a := range manifest.Build.Artifacts { - if a.Context == nil || a.Context.Repository == "" { - continue - } - key := a.Context.Repository + "#" + a.Context.Ref - if seen[key] { - continue - } - seen[key] = true - out = append(out, *a.Context) - } - sort.Slice(out, func(i, j int) bool { - if out[i].Repository != out[j].Repository { - return out[i].Repository < out[j].Repository - } - return out[i].Ref < out[j].Ref - }) - return out -} +// readBy is every repository other than the module's own that this build's recipes read source from: the rule +// build-agent and the controller share (internal/buildreads). +func readBy(manifest catalogue.Manifest) []catalogue.ArtifactContext { return buildreads.Of(manifest) } // binaryName is what a compiled bundle's executable is called: what the artifact says, or the name of // the package it is built from, which is what a compiler would have chosen anyway. diff --git a/internal/buildreads/buildreads.go b/internal/buildreads/buildreads.go new file mode 100644 index 00000000..6f400bda --- /dev/null +++ b/internal/buildreads/buildreads.go @@ -0,0 +1,39 @@ +// Package buildreads is which repositories other than a module's own its build reads source from, as its manifest +// names them (novox/hq 04-ISSUES/131): the artifacts' `context` repositories. One rule for build-agent, which says it +// in a build's outcome, and for the controller, which reads it from a done key's manifest (R4e, issue 1851): it lives +// here, not in internal/builder, so it outlasts build-agent (R4g). +package buildreads + +import ( + "sort" + + "git.novox.be/novox/mesh-sdk/go/catalogue" +) + +// Of is every repository other than the module's own that this build's recipes read source from, each once and in a +// fixed order, so two builds of one commit report the same thing the same way. +func Of(manifest catalogue.Manifest) []catalogue.ArtifactContext { + if manifest.Build == nil { + return nil + } + seen := map[string]bool{} + var out []catalogue.ArtifactContext + for _, a := range manifest.Build.Artifacts { + if a.Context == nil || a.Context.Repository == "" { + continue + } + key := a.Context.Repository + "#" + a.Context.Ref + if seen[key] { + continue + } + seen[key] = true + out = append(out, *a.Context) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Repository != out[j].Repository { + return out[i].Repository < out[j].Repository + } + return out[i].Ref < out[j].Ref + }) + return out +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go new file mode 100644 index 00000000..df6a1b01 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/fetch.go @@ -0,0 +1,114 @@ +package buildstep + +import ( + "context" + "fmt" + "net/url" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Option changes FetchCommit. +type Option func(*fetchOptions) + +type fetchOptions struct{ allowFile bool } + +// AllowFileRemote lets FetchCommit take a file:// remote. It is for a test's local repository; production +// code does not pass it, and FetchCommit then refuses anything but ssh://. +func AllowFileRemote() Option { return func(o *fetchOptions) { o.allowFile = true } } + +var ( + commitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + refOK = regexp.MustCompile(`^refs/[A-Za-z0-9._/-]+$`) +) + +// FetchCommit fetches one commit of a forge repository into dir, an empty directory, and checks it out detached +// (novox/hq ADR 0466 §4, §5; issue 1474). remote is ssh:////.git, the ssh name being the +// one the build module's templates set up (ADR 0409 §1); commit is a full 40-hex sha; ref is optional, and names +// the pull request's head (refs/pull//head) when the commit is only there. Every git command runs through run +// as `git -C dir ...` with GIT_TERMINAL_PROMPT=0 and no credential helper, so nothing is asked for and no +// credential is used. A remote that is not ssh://, or one with user information, and a commit that is not 40 +// hex digits are refused before anything runs. +func FetchCommit(ctx context.Context, run forgekey.Runner, dir, remote, commit, ref string, opts ...Option) error { + var o fetchOptions + for _, opt := range opts { + opt(&o) + } + if err := checkFetch(o, dir, remote, commit, ref); err != nil { + return err + } + env := []string{"GIT_TERMINAL_PROMPT=0"} + git := func(args ...string) (string, error) { + full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) + r := run(ctx, env, "git", full...) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr)) + } + return r.Stdout, nil + } + if _, err := git("init", "-q"); err != nil { + return err + } + if _, err := git("remote", "add", "origin", remote); err != nil { + return err + } + if _, err := git("fetch", "--no-tags", "--depth", "1", "origin", commit); err != nil { + if ref == "" { + return err + } + if _, rerr := git("fetch", "--no-tags", "origin", ref); rerr != nil { + return fmt.Errorf("%v; then the ref %s: %w", err, ref, rerr) + } + if _, verr := git("cat-file", "-e", commit+"^{commit}"); verr != nil { + return fmt.Errorf("the commit %s is not in %s: %w", commit, ref, verr) + } + } + if _, err := git("checkout", "-q", "--detach", commit); err != nil { + return err + } + out, err := git("rev-parse", "HEAD") + if err != nil { + return err + } + if head := strings.TrimSpace(out); head != commit { + return fmt.Errorf("the checkout is at %s, not at the commit %s", head, commit) + } + return nil +} + +func checkFetch(o fetchOptions, dir, remote, commit, ref string) error { + if dir == "" { + return fmt.Errorf("buildstep: no workspace directory") + } + if !commitOK.MatchString(commit) { + return fmt.Errorf("buildstep: %q is not a full commit (40 hex digits)", commit) + } + if ref != "" && !refOK.MatchString(ref) { + return fmt.Errorf("buildstep: %q is not a ref (refs/...)", ref) + } + u, err := url.Parse(remote) + if err != nil { + return fmt.Errorf("buildstep: the remote is not a URL: %w", err) + } + if u.User != nil { + return fmt.Errorf("buildstep: the remote carries user information; the key is the credential") + } + switch { + case u.Scheme == "ssh" && u.Host != "": + case u.Scheme == "file" && o.allowFile: + default: + return fmt.Errorf("buildstep: the remote %q is not ssh://", remote) + } + return nil +} + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go new file mode 100644 index 00000000..a9dc49cd --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/guard.go @@ -0,0 +1,52 @@ +package buildstep + +import ( + "context" + "encoding/json" + "fmt" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Guard is a build module's check that the toolchain installed on its node is the one its recipe builds with +// (novox/hq ADR 0469 §1; issue 1473). The same guard serves ADR 0470 §4's postgres major for the integration +// level, with its own Probe and Want. +type Guard struct { + // Language is the toolchain's key part: go or typescript (delivery.ToolchainKey). + Language string + // Want is the toolchain version the recipe builds with, and Recipe the recipe's own version. The indexer + // compares both texts as they are written here. + Want, Recipe string + // Probe reads the installed version (go version, node --version). + Probe func(ctx context.Context) (installed string, err error) + Node string + // Put is the toolchains state's Put (stdio.State(delivery.ToolchainsState).Put). + Put Put +} + +// Check probes the installed version, writes it to . of the toolchains state every time it runs, +// and answers nil when it is the recipe's. Otherwise it answers a *Refusal naming both versions; the installed +// version is written even then, so the indexer sees that builders disagree. A probe that fails is a *Refusal +// naming its failure, and nothing is written. A write that fails is an error that is not a Refusal. +func (g Guard) Check(ctx context.Context) error { + installed, err := g.Probe(ctx) + if err != nil { + return &Refusal{Text: fmt.Sprintf("the installed %s version could not be read: %v", g.Language, err), Want: g.Want} + } + key, err := delivery.ToolchainKey(g.Node, g.Language) + if err != nil { + return err + } + raw, err := delivery.EncodeToolchain(delivery.Toolchain{Language: g.Language, Version: installed, Recipe: g.Recipe, At: now().UTC()}) + if err != nil { + return err + } + if _, err := g.Put(key, json.RawMessage(raw)); err != nil { + return fmt.Errorf("buildstep: writing the toolchain %s: %w", key, err) + } + if installed != g.Want { + return &Refusal{Text: fmt.Sprintf("%s %s installed, the recipe builds with %s %s", g.Language, installed, g.Language, g.Want), + Want: g.Want, Have: installed} + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go new file mode 100644 index 00000000..4b59f5cd --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/launchers.go @@ -0,0 +1,24 @@ +package buildstep + +import "strings" + +// LauncherSuffix is what a TypeScript entrypoint's launcher is called beside it: index.js is started as +// index.serve.mjs (novox/hq ADR 0193), an ES module by its own extension whatever the bundle's package.json says. +const LauncherSuffix = ".serve.mjs" + +// Launchers are the launchers a bundle's build writes, by entrypoint, relative to the bundle's root: for TypeScript +// every entrypoint `x.js` is started by `x.serve.mjs` beside it; an entrypoint that is not JavaScript and every other +// language, whose build is already executable, get none. The one rule mesh-build-node, build-agent and the controller +// share (R4e), so the name a build writes is the name a reader expects. +func Launchers(language string, entrypoints []string) map[string]string { + out := map[string]string{} + if language != "typescript" { + return out + } + for _, entry := range entrypoints { + if strings.HasSuffix(entry, ".js") { + out[entry] = strings.TrimSuffix(entry, ".js") + LauncherSuffix + } + } + return out +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go new file mode 100644 index 00000000..6bc8df9e --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/merge.go @@ -0,0 +1,112 @@ +package buildstep + +import ( + "context" + "fmt" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// MergeResult is what MergeInto made of a head and a trunk commit: the trunk commit (Base) and the head, whole, and the +// merge commit and the paths the merge changes against the trunk commit; or, when the head does not merge cleanly, the +// files in conflict, with Merge and Changed empty. +type MergeResult struct { + Base, Head, Merge string + Conflicts []string + Changed []string +} + +// MergeInto makes, in the workspace dir that FetchCommit left at commit, the merge of commit into the trunk commit +// base, as the forge's default merge would (a merge commit, never a fast-forward), and leaves the workspace at the merge +// (novox/hq ADR 0484 clause 3, ADR 0323). It is the one merge the build modules' fetch steps make, with the identity +// and flags of the controller's check merge (mesh-controller internal/builder/check_merge.go): a name nobody owns, no +// signature, no hook, nothing of the machine's git configuration that changes a merge. Nothing is pushed. +// +// A merge needs both histories, so it makes the fetch whole (the shallow clone FetchCommit made is deepened) and +// fetches base by its sha. A head that does not merge cleanly is an answer, not an error: the files in conflict, the +// merge undone and the workspace left at base. A head already in base answers a Merge equal to Base and no changed +// paths. An error is that the merge could not be tried: a base or commit that is not a full sha, no base, a workspace +// not at commit, or a fetch or git command that failed. remote and the options are those of FetchCommit; the runner's +// bound must allow a whole fetch. +func MergeInto(ctx context.Context, run forgekey.Runner, dir, remote, base, commit string, opts ...Option) (MergeResult, error) { + var o fetchOptions + for _, opt := range opts { + opt(&o) + } + res := MergeResult{Base: base, Head: commit} + if base == "" { + return res, fmt.Errorf("buildstep: no base: a merge needs the trunk commit it is made onto") + } + if err := checkFetch(o, dir, remote, base, ""); err != nil { + return res, err + } + if err := checkFetch(o, dir, remote, commit, ""); err != nil { + return res, err + } + env := []string{"GIT_TERMINAL_PROMPT=0"} + git := func(args ...string) (string, error) { + full := append([]string{"-c", "credential.helper=", "-C", dir}, args...) + r := run(ctx, env, "git", full...) + if r.Status != 0 || r.Err != "" { + return "", fmt.Errorf("git %s: %s", args[0], firstLine(r.Err+" "+r.Stderr)) + } + return r.Stdout, nil + } + at, err := git("rev-parse", "HEAD") + if err != nil { + return res, err + } + if head := strings.TrimSpace(at); head != commit { + return res, fmt.Errorf("buildstep: the workspace is at %s, not at the commit %s", head, commit) + } + shallow, err := git("rev-parse", "--is-shallow-repository") + if err != nil { + return res, err + } + fetch := []string{"fetch", "--no-tags", "origin", base} + if strings.TrimSpace(shallow) == "true" { + fetch = []string{"fetch", "--no-tags", "--unshallow", "origin", base} + } + if _, err := git(fetch...); err != nil { + return res, fmt.Errorf("the trunk commit %s: %w", base, err) + } + if _, err := git("checkout", "-q", "--detach", base); err != nil { + return res, fmt.Errorf("the trunk commit %s cannot be checked out: %w", base, err) + } + message := fmt.Sprintf("Check the merge of %s into %s (made for a check, never pushed)", short(commit), short(base)) + args := append(append([]string{}, mergeIdentity...), "merge", "--no-ff", "--no-edit", "--no-verify", "--quiet", "-m", message, commit) + if _, err := git(args...); err != nil { + out, _ := git("diff", "--name-only", "--diff-filter=U") + res.Conflicts = strings.Fields(out) + _, _ = git("merge", "--abort") + if len(res.Conflicts) == 0 { + return res, fmt.Errorf("the head %s could not be merged into the trunk commit %s: %w", short(commit), short(base), err) + } + return res, nil + } + merge, err := git("rev-parse", "HEAD") + if err != nil { + return res, fmt.Errorf("the merge cannot be read: %w", err) + } + res.Merge = strings.TrimSpace(merge) + changed, err := git("diff", "--name-only", "--no-renames", base, res.Merge) + if err != nil { + return res, fmt.Errorf("the paths the merge changes cannot be read: %w", err) + } + res.Changed = strings.Fields(changed) + return res, nil +} + +// mergeIdentity is who a check's merge is made as: it is never pushed, so it names nobody's account. No hook of the +// clone's or the build machine's git configuration runs, and no setting there changes the merge (the controller's +// check merge, internal/builder/check_merge.go). +var mergeIdentity = []string{"-c", "user.name=mesh merge check", "-c", "user.email=merge-check@mesh.invalid", + "-c", "commit.gpgsign=false", "-c", "core.hooksPath=/dev/null", "-c", "merge.verifySignatures=false"} + +func short(commit string) string { + if len(commit) > 12 { + return commit[:12] + } + return commit +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go new file mode 100644 index 00000000..bab02b2b --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/pack.go @@ -0,0 +1,100 @@ +package buildstep + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "sort" +) + +// Pack tars and gzips a directory the way the build agent does, byte for byte, and answers the archive and its +// digest, "sha256:" and the hex of the archive's bytes (issue 1479; novox/hq ADR 0405 §7, ADR 0453 §7). It is a +// copy of mesh-controller's internal/builder/builder.go pack (origin/main 073ccc29, :1351-1425), which stays +// where it is until the build agent goes: a build module's digest is compared with the build agent's for one +// fingerprint, so the bytes must be equal. +// +// Deterministically: regular files only (a symlink or a device is refused), sorted relative paths with +// slashes, each header carrying its name, permission bits and size and nothing else, the gzip default writer. +func Pack(root string) (archive []byte, digest string, err error) { + raw, err := pack(root) + if err != nil { + return nil, "", err + } + sum := sha256.Sum256(raw) + return raw, "sha256:" + hex.EncodeToString(sum[:]), nil +} + +func pack(root string) ([]byte, error) { + info, err := os.Stat(root) + if err != nil { + return nil, err + } + if !info.IsDir() { + return nil, fmt.Errorf("%s is not a directory", root) + } + + var paths []string + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || !info.Mode().IsRegular() { + // Only files. A symlink or a device in an archive is refused by the host that unpacks + // it, so putting one in would build something that cannot be applied. + if !info.IsDir() && !info.Mode().IsRegular() { + return fmt.Errorf("%s is neither a file nor a directory, and an archive carries "+ + "only those", path) + } + return nil + } + paths = append(paths, path) + return nil + }) + if err != nil { + return nil, err + } + // filepath.Walk walks in lexical order; the sort is kept as the build agent keeps it, since an archive + // whose digest changes because the traversal did is a silent failure. + sort.Strings(paths) + + var out bytes.Buffer + zipped := gzip.NewWriter(&out) + writer := tar.NewWriter(zipped) + for _, path := range paths { + body, err := os.ReadFile(path) + if err != nil { + return nil, err + } + relative, err := filepath.Rel(root, path) + if err != nil { + return nil, err + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + mode := int64(info.Mode().Perm()) + if err := writer.WriteHeader(&tar.Header{ + Name: filepath.ToSlash(relative), Mode: mode, Size: int64(len(body)), + Typeflag: tar.TypeReg, + // Everything else left at its zero value on purpose: no times, owners or names. + }); err != nil { + return nil, err + } + if _, err := writer.Write(body); err != nil { + return nil, err + } + } + if err := writer.Close(); err != nil { + return nil, err + } + if err := zipped.Close(); err != nil { + return nil, err + } + return out.Bytes(), nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go new file mode 100644 index 00000000..0f885294 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/refusal.go @@ -0,0 +1,13 @@ +package buildstep + +// Refusal is a step that is not run on purpose, as against one that failed: the toolchain installed is not the +// recipe's, the integration level's postgres major is not the one it is built for (novox/hq ADR 0469 §1, ADR 0470 +// §4; issue 1473). A step's run returns one and the runner ends the step refused with its text. +type Refusal struct { + // Text is the reason, in words the operator reads. + Text string + // Want is the version the recipe asks for and Have the one found; either may be empty. + Want, Have string +} + +func (r *Refusal) Error() string { return r.Text } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go new file mode 100644 index 00000000..c5ae0d95 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/step.go @@ -0,0 +1,99 @@ +// Package buildstep is what both build modules share: the runner of a long step that answers a tool call at once +// and ends on its key in the module's `steps` state (novox/hq ADR 0469 §2; issue 1471), the toolchain guard +// (issue 1473) and the fetch of one commit over SSH (issue 1474). +package buildstep + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// Put writes one key of a state: stdio.State(delivery.StepsState).Put's shape (go/state.go). +type Put func(key string, value any) (uint64, error) + +// EndAttempts and EndRetry bound how a step's end is written: a write that fails is tried again, since the end +// is the only way the step's reader learns it is over. +var ( + EndAttempts = 5 + EndRetry = 2 * time.Second +) + +var now = time.Now + +// NewID is a step id: the time in milliseconds in base 36 and six random hex digits, lower-case letters and +// digits only, so it is one key token and differs between two steps of one node even in one millisecond. +func NewID() string { + var b [3]byte + _, _ = rand.Read(b[:]) + return strconv.FormatInt(now().UnixMilli(), 36) + hex.EncodeToString(b[:]) +} + +// Start begins a step of tool on node and answers its id at once: the key . holds state running when +// Start returns. run is then called in a goroutine of its own, under ctx (the process's, not the tool call's: the +// call is cut at 30 s, the step is not). When it returns, the key is written passed with run's result, failed with +// the error's text, or refused when the error is a *Refusal; a panic in run ends the step failed, naming it, and +// the process goes on. If the running write fails, nothing was started and Start answers the error. +func Start(ctx context.Context, put Put, node, tool string, run func(ctx context.Context) (result any, err error)) (string, error) { + step := delivery.Step{ID: NewID(), Tool: tool, Node: node, State: delivery.StepRunning, Started: now().UTC()} + key, err := delivery.StepKey(node, step.ID) + if err != nil { + return "", err + } + if err := write(put, key, step); err != nil { + return "", fmt.Errorf("buildstep: writing the running step %s: %w", key, err) + } + go func() { + result, err := protect(ctx, run) + step.Ended = now().UTC() + switch ref := (*Refusal)(nil); { + case err == nil: + step.State = delivery.StepPassed + if result != nil { + raw, merr := json.Marshal(result) + if merr != nil { + step.State, step.Reason = delivery.StepFailed, "the result is not JSON: "+merr.Error() + } else { + step.Result = raw + } + } + case errors.As(err, &ref): + step.State, step.Reason = delivery.StepRefused, err.Error() + default: + step.State, step.Reason = delivery.StepFailed, err.Error() + } + for attempt := 1; ; attempt++ { + if werr := write(put, key, step); werr == nil || attempt >= EndAttempts { + return + } + time.Sleep(EndRetry) + } + }() + return step.ID, nil +} + +// protect is run with a panic turned into the error that ends the step. +func protect(ctx context.Context, run func(ctx context.Context) (any, error)) (result any, err error) { + defer func() { + if p := recover(); p != nil { + result, err = nil, fmt.Errorf("the step panicked: %v", p) + } + }() + return run(ctx) +} + +func write(put Put, key string, step delivery.Step) error { + raw, err := delivery.EncodeStep(step) + if err != nil { + return err + } + _, err = put(key, json.RawMessage(raw)) + return err +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go new file mode 100644 index 00000000..58b2ccd8 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go @@ -0,0 +1,67 @@ +package buildstep + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "os" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// ModuleTree is what a merge holds of one module directory: its tree id, the sha256 of its module.json and the sha256 +// of each lock file it has (go.sum, package-lock.json), from which the checker and the indexer take the module's inputs +// (novox/hq ADR 0484 clause 3). +type ModuleTree struct { + Tree string `json:"tree"` + Manifest string `json:"manifest"` + Locks map[string]string `json:"locks,omitempty"` +} + +var lockFiles = []string{"go.sum", "package-lock.json"} + +// ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory +// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is +// the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge. +func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) { + r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json") + if r.Status != 0 || r.Err != "" { + return nil, fmt.Errorf("git ls-files: %s", firstLine(r.Err+" "+r.Stderr)) + } + trees := map[string]ModuleTree{} + for _, l := range strings.Split(r.Stdout, "\n") { + if l = strings.TrimSpace(l); l == "" { + continue + } + mdir := filepath.ToSlash(filepath.Dir(l)) + spec := commit + ":" + mdir + if mdir == "." { + spec = commit + "^{tree}" + } + t := run(ctx, env, "git", "-C", dir, "rev-parse", spec) + if t.Status != 0 || t.Err != "" { + return nil, fmt.Errorf("git rev-parse %s: %s", spec, firstLine(t.Err+" "+t.Stderr)) + } + mt := ModuleTree{Tree: strings.TrimSpace(t.Stdout)} + raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(l))) + if err != nil { + return nil, err + } + mt.Manifest = sha256Hex(raw) + for _, lock := range lockFiles { + if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil { + if mt.Locks == nil { + mt.Locks = map[string]string{} + } + mt.Locks[lock] = sha256Hex(raw) + } + } + trees[mdir] = mt + } + return trees, nil +} + +func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go new file mode 100644 index 00000000..cb756685 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keykeeper.go @@ -0,0 +1,127 @@ +// Package forgekey is the key keeper every module that reaches the forge over SSH shares: ssh-client for a +// machine's accounts, a build module for its own (design 47 R4c-3, ADR 0466 §2; issue 1469). +package forgekey + +// The key this machine's account uses to reach the forge over SSH (novox/hq ADR 0399, ADR 0400; issue 955, +// area S1). It is made here, on the machine, once: ssh-keygen writes ~/.ssh/mesh_ed25519 only when it is +// missing, so the private half never leaves the machine and an existing key is never replaced. The public +// half and its fingerprint go into this module's `keys` state, one entry per account under ., +// for the forge's provider to register. The tools bundle is the one process of this module, so the keeper +// runs inside it beside the tools (main.go), the way claude-code reports what it holds. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +// KeyFile is the key's name under ~/.ssh: never one of the operator's own id_* files. +const KeyFile = "mesh_ed25519" + +// The kinds of key (ADR 0400 §1, ADR 0466 §2): a machine's account's own, and a build module's. +const ( + KindMachine = "machine" + KindBuilder = "builder" +) + +// KeyRetry is how soon it looks again after a failure. +const KeyRetry = 30 * time.Second + +// KeyRound is how often the keeper looks again. +const KeyRound = 10 * time.Minute + +// KeyEntry is one value of the keys state. It carries the public half only. +type KeyEntry struct { + Node string `json:"node"` + Account string `json:"account"` + Kind string `json:"kind"` + Public string `json:"public"` + Fingerprint string `json:"fingerprint"` + At string `json:"at"` +} + +// KeyPutter is the keys state, as far as the keeper needs it. +type KeyPutter interface { + Put(key string, value any) (uint64, error) +} + +// KeyKeeper keeps one account's key and its entry. +type KeyKeeper struct { + Home, Node, Account string + Kind string // KindMachine or KindBuilder + Run Runner + State KeyPutter + Now func() time.Time + // After waits between rounds; time.After when nil. A test sets it to see which pause was chosen. + After func(time.Duration) <-chan time.Time + + reported string // the public half and fingerprint last put +} + +func (k *KeyKeeper) path() string { return filepath.Join(k.Home, ".ssh", KeyFile) } + +// Once makes the key when it is missing and puts the entry when it differs from the one last put. +func (k *KeyKeeper) Once(ctx context.Context) error { + if err := k.ensure(ctx); err != nil { + return err + } + raw, err := os.ReadFile(k.path() + ".pub") + if err != nil { + return fmt.Errorf("the public half: %w", err) + } + pk, err := ParseKeyLine(string(raw)) + if err != nil { + return fmt.Errorf("%s.pub: %w", k.path(), err) + } + public := strings.TrimSpace(string(raw)) + if sig := public + "|" + pk.Fingerprint; sig == k.reported { + return nil + } + entry := KeyEntry{Node: k.Node, Account: k.Account, Kind: k.Kind, Public: public, + Fingerprint: pk.Fingerprint, At: k.Now().UTC().Format(time.RFC3339)} + if _, err := k.State.Put(k.Node+"."+k.Account, entry); err != nil { + return err + } + k.reported = public + "|" + pk.Fingerprint + return nil +} + +// ensure makes the pair when the private half is missing. Never over an existing key. +func (k *KeyKeeper) ensure(ctx context.Context) error { + if _, err := os.Lstat(k.path()); err == nil { + return nil + } else if !os.IsNotExist(err) { + return err + } + if err := os.MkdirAll(filepath.Dir(k.path()), 0o700); err != nil { + return err + } + r := k.Run(ctx, nil, "ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", k.Account+"@"+k.Node+" (mesh)", "-f", k.path()) + if r.Status != 0 || r.Err != "" { + return fmt.Errorf("ssh-keygen: %s", firstLine(r.Err+" "+r.Stderr)) + } + return nil +} + +// Keep runs Once now and every KeyRound, reporting a failure through say and trying again at the next round. +func (k *KeyKeeper) Keep(ctx context.Context, say func(string, ...any)) { + for { + pause := KeyRound + if err := k.Once(ctx); err != nil { + say("the forge key: %v", err) + pause = KeyRetry // the state or the bus's grant may arrive after the process + } + after := k.After + if after == nil { + after = time.After + } + select { + case <-ctx.Done(): + return + case <-after(pause): + } + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go new file mode 100644 index 00000000..4af0499b --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/keys.go @@ -0,0 +1,110 @@ +package forgekey + +// Keys, by their public half only. A fingerprint is computed here from the public key's bytes, as +// ssh-keygen -l does (SHA256 of the key blob, unpadded base64); a private key is never read by this +// process — its first line, the PEM header, says it is one, and ssh-keygen is asked about it. + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "fmt" + "math/big" + "strings" +) + +// KeyTypes are the public key algorithms an authorized_keys or known_hosts line can carry. +var KeyTypes = map[string]bool{ + "ssh-ed25519": true, "ssh-rsa": true, "ssh-dss": true, + "ecdsa-sha2-nistp256": true, "ecdsa-sha2-nistp384": true, "ecdsa-sha2-nistp521": true, + "sk-ssh-ed25519@openssh.com": true, "sk-ecdsa-sha2-nistp256@openssh.com": true, +} + +// PublicKey is one public key as a line carries it. +type PublicKey struct { + Type string `json:"type"` + Bits int `json:"bits"` + Fingerprint string `json:"fingerprint"` + Comment string `json:"comment"` + Options string `json:"options,omitempty"` + Weak string `json:"weak,omitempty"` +} + +// ParseKeyLine reads one authorized_keys line (options, type, key, comment) or a public key file's. +func ParseKeyLine(line string) (PublicKey, error) { + fields := fieldsQuoted(strings.TrimSpace(line)) + for i, f := range fields { + if !KeyTypes[f] || i+1 >= len(fields) { + continue + } + k := PublicKey{Type: f, Options: strings.Join(fields[:i], " "), Comment: strings.Join(fields[i+2:], " ")} + blob, err := base64.StdEncoding.DecodeString(fields[i+1]) + if err != nil { + return k, fmt.Errorf("the key after %s is not base64", f) + } + sum := sha256.Sum256(blob) + k.Fingerprint = "SHA256:" + base64.RawStdEncoding.EncodeToString(sum[:]) + k.Bits = bitsOf(f, blob) + switch { + case f == "ssh-dss": + k.Weak = "DSA: refused by current OpenSSH" + case f == "ssh-rsa" && k.Bits > 0 && k.Bits < 3072: + k.Weak = fmt.Sprintf("RSA of %d bits: below 3072", k.Bits) + } + return k, nil + } + return PublicKey{}, fmt.Errorf("no public key on this line") +} + +// fieldsQuoted splits on spaces outside double quotes, as sshd reads an options field. +func fieldsQuoted(s string) []string { + var out []string + var cur strings.Builder + quoted := false + for _, ch := range s { + switch { + case ch == '"': + quoted = !quoted + cur.WriteRune(ch) + case (ch == ' ' || ch == '\t') && !quoted: + if cur.Len() > 0 { + out = append(out, cur.String()) + cur.Reset() + } + default: + cur.WriteRune(ch) + } + } + if cur.Len() > 0 { + out = append(out, cur.String()) + } + return out +} + +// bitsOf reads a key's size from its blob: the RSA modulus, the curve, or ed25519's fixed 256. +func bitsOf(kind string, blob []byte) int { + strs := [][]byte{} + for len(blob) >= 4 { + n := binary.BigEndian.Uint32(blob) + if int(n) > len(blob)-4 { + break + } + strs = append(strs, blob[4:4+n]) + blob = blob[4+n:] + } + switch { + case strings.Contains(kind, "ed25519"): + return 256 + case kind == "ssh-rsa" && len(strs) >= 3: + return new(big.Int).SetBytes(strs[2]).BitLen() + case kind == "ssh-dss" && len(strs) >= 2: + return new(big.Int).SetBytes(strs[1]).BitLen() + case strings.Contains(kind, "nistp256"): + return 256 + case strings.Contains(kind, "nistp384"): + return 384 + case strings.Contains(kind, "nistp521"): + return 521 + } + return 0 +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go new file mode 100644 index 00000000..14bba8af --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/forgekey/runner.go @@ -0,0 +1,93 @@ +package forgekey + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "time" +) + +// Ran is what a command did: its output, its exit status, and why it never ran to an answer. +type Ran struct { + Stdout string + Stderr string + Status int + // Err is "ENOENT" when the program is not installed, or says it was ended for taking too long. + Err string +} + +// Runner runs one command with extra environment words, so every tool can be tested without ssh. +type Runner func(ctx context.Context, env []string, name string, args ...string) Ran + +// CallTimeout bounds one command of ExecRunner: below the runtime's thirty-second call limit. A variable so a +// test can lower it. +var CallTimeout = 20 * time.Second + +// ExecRunner runs a command on this machine, with no terminal and nothing on its stdin, bounded by +// CallTimeout. +func ExecRunner(ctx context.Context, env []string, name string, args ...string) Ran { + return ExecRunnerFor(CallTimeout)(ctx, env, name, args...) +} + +// ExecRunnerFor is ExecRunner with its own bound (issue 1497): a build module's long step passes 0, so only the +// step's own context ends a command, and a fetch may outlast a tool call's thirty seconds. +func ExecRunnerFor(bound time.Duration) Runner { + return func(ctx context.Context, env []string, name string, args ...string) Ran { + return execRun(ctx, bound, false, env, name, args...) + } +} + +// ExecRunnerClean is ExecRunnerFor whose commands get exactly the env argument as their environment, and nothing of +// this process's (issue 1497; novox/hq ADR 0466 §4: no registry password in a fetch's environment). The caller +// gives PATH and HOME itself. A directory is the command's own business (`git -C`, `go -C`). +func ExecRunnerClean(bound time.Duration) Runner { + return func(ctx context.Context, env []string, name string, args ...string) Ran { + return execRun(ctx, bound, true, env, name, args...) + } +} + +func execRun(ctx context.Context, bound time.Duration, clean bool, env []string, name string, args ...string) Ran { + if bound > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, bound) + defer cancel() + } + cmd := exec.CommandContext(ctx, name, args...) + if clean { + cmd.Env = append([]string{}, env...) // non-nil, so an empty list is an empty environment + } else { + cmd.Env = append(os.Environ(), env...) + } + var out, errb bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &errb + err := cmd.Run() + r := Ran{Stdout: out.String(), Stderr: errb.String()} + var exitErr *exec.ExitError + switch { + case errors.Is(ctx.Err(), context.DeadlineExceeded): + r.Status, r.Err = 124, "ended by its step's deadline" + if bound > 0 { + r.Err = fmt.Sprintf("no answer within %d s", int(bound/time.Second)) + } + case errors.Is(err, exec.ErrNotFound): + r.Status, r.Err = 127, "ENOENT" + case errors.As(err, &exitErr): + r.Status = exitErr.ExitCode() + case err != nil: + r.Status, r.Err = 1, err.Error() + } + return r +} + +func firstLine(s string) string { + for _, l := range strings.Split(s, "\n") { + if l = strings.TrimSpace(l); l != "" { + return l + } + } + return "" +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 30efdfb1..26ad86b3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2,11 +2,13 @@ ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go git.novox.be/novox/mesh-sdk/go/asks +git.novox.be/novox/mesh-sdk/go/buildstep git.novox.be/novox/mesh-sdk/go/catalogue git.novox.be/novox/mesh-sdk/go/compose git.novox.be/novox/mesh-sdk/go/declaration git.novox.be/novox/mesh-sdk/go/delivery git.novox.be/novox/mesh-sdk/go/facts +git.novox.be/novox/mesh-sdk/go/forgekey git.novox.be/novox/mesh-sdk/go/membership git.novox.be/novox/mesh-sdk/go/overlay git.novox.be/novox/mesh-sdk/go/process From ea3c8ad7f686a489f4dff1eb00a9b0ffa6522001 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:45:55 +0200 Subject: [PATCH 05/38] Register a new module from the builder's first done key and ask no first build; its condition reads the key and names a manifest word this controller does not read (issue 1851, A5a, A5b) --- cmd/mesh-controller/batches.go | 4 +- cmd/mesh-controller/build_source.go | 36 --- .../builder_on_the_bus_test.go | 131 +++++++++ .../first_build_ask_integration_test.go | 84 ------ .../first_build_warrant_test.go | 29 -- cmd/mesh-controller/first_builds.go | 240 ++++------------- .../first_builds_held_integration_test.go | 221 --------------- cmd/mesh-controller/first_builds_held_test.go | 141 ---------- .../first_builds_integration_test.go | 95 +++---- cmd/mesh-controller/main.go | 7 - .../new_module_asked_once_test.go | 37 --- .../new_module_on_merge_test.go | 155 +++++------ cmd/mesh-controller/new_modules.go | 251 ++++++++++++++++++ cmd/mesh-controller/new_modules_test.go | 33 +++ cmd/mesh-controller/plan_looks.go | 4 - cmd/mesh-controller/push.go | 3 + cmd/mesh-controller/upgrades.go | 110 +------- internal/builderkeys/builderkeys.go | 33 +++ .../builderkeys_integration_test.go | 5 + 19 files changed, 608 insertions(+), 1011 deletions(-) create mode 100644 cmd/mesh-controller/builder_on_the_bus_test.go delete mode 100644 cmd/mesh-controller/first_build_ask_integration_test.go delete mode 100644 cmd/mesh-controller/first_build_warrant_test.go delete mode 100644 cmd/mesh-controller/first_builds_held_integration_test.go delete mode 100644 cmd/mesh-controller/first_builds_held_test.go delete mode 100644 cmd/mesh-controller/new_module_asked_once_test.go create mode 100644 cmd/mesh-controller/new_modules.go create mode 100644 cmd/mesh-controller/new_modules_test.go diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go index ae7627a1..44f6426a 100644 --- a/cmd/mesh-controller/batches.go +++ b/cmd/mesh-controller/batches.go @@ -232,8 +232,8 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T fmt.Print(nothingReadsIt(repository, m)) return nil } - // **A module the merge adds is asked for now** (novox/hq issue 1617), not when the batch is cut. - askNewModulesAtHearing(ctx, inv, m, entries, read, facts.from) + // A module the merge adds is registered from the builder's first done key of it, which the builder's write wakes + // (novox/hq ADR 0501 §6): nothing is asked here. // **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk. own := ownPath(touches) waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries)) diff --git a/cmd/mesh-controller/build_source.go b/cmd/mesh-controller/build_source.go index 0c3d84ea..3520c8fd 100644 --- a/cmd/mesh-controller/build_source.go +++ b/cmd/mesh-controller/build_source.go @@ -697,37 +697,6 @@ func sourceWords(repository, seat string) string { return buildSource{Repository: repository, Seat: seat}.String() } -// firstBuildWarrant is what the controller checked before it asks the first build of the modules a merge adds to a -// repository nothing is built from: the merge was made on the operator's Approve of its head (approvedAnnounced). It -// travels in the context of that ask alone, in this process, so no verb carries one. -type firstBuildWarrant struct { - repository string // owner/name on the git seat - paths map[string]bool - why string -} - -type firstBuildWarrantKey struct{} - -// withFirstBuildWarrant is ctx carrying the warrant for asking the first builds of paths in repository. -func withFirstBuildWarrant(ctx context.Context, repository string, paths []string, why string) context.Context { - w := &firstBuildWarrant{repository: repository, paths: map[string]bool{}, why: why} - for _, p := range paths { - w.paths[strings.Trim(p, "/")] = true - } - return context.WithValue(ctx, firstBuildWarrantKey{}, w) -} - -func firstBuildWarrantIn(ctx context.Context) *firstBuildWarrant { - w, _ := ctx.Value(firstBuildWarrantKey{}).(*firstBuildWarrant) - return w -} - -// covers says the warrant is for exactly this ask: the repository on the git seat, and a module path the merge holds. -func (w *firstBuildWarrant) covers(source buildSource, path string) bool { - return w != nil && source.Seat == gitSeat && strings.EqualFold(strings.Trim(source.Repository, "/"), w.repository) && - w.paths[strings.Trim(path, "/")] -} - // verbMayAsk refuses, for a call through a verb, a build of a repository the catalogue builds no module from // (novox/hq ADR 0266): the build node would run what an agent wrote, and its outcome could never be // registered anyway. url is the repository as it is cloned. At the terminal anything may be asked. @@ -735,11 +704,6 @@ func verbMayAsk(ctx context.Context, source buildSource, url, path string) error if startedAtTheTerminal() { return nil } - // **Or the first build of a module a merge adds on the operator's Approve** (novox/hq ADR 0432, issue 1146): the - // controller's own ask, carrying the warrant it checked, for that repository and the module paths the merge holds. - if firstBuildWarrantIn(ctx).covers(source, path) { - return nil - } open, err := openStores(ctx) if err != nil { return err diff --git a/cmd/mesh-controller/builder_on_the_bus_test.go b/cmd/mesh-controller/builder_on_the_bus_test.go new file mode 100644 index 00000000..c20a6d6a --- /dev/null +++ b/cmd/mesh-controller/builder_on_the_bus_test.go @@ -0,0 +1,131 @@ +//go:build integration + +package main + +import ( + "context" + "sync" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +// builderOnTheBus is the builder's and the indexer's buckets on a test's own bus, which the controller's connection for +// the moment (servingBus) reaches: what a take-in from a key reads (R4e, issue 1851). +type builderOnTheBus struct { + js *broker.JetStream + builds, modules, manifests jetstream.KeyValue +} + +func aBuilderOnTheBus(t *testing.T) builderOnTheBus { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + ctx := t.Context() + b := builderOnTheBus{js: js} + for _, c := range []struct { + into *jetstream.KeyValue + bucket string + history uint8 + }{ + {&b.builds, sdkdelivery.BuilderBuildsBucket, sdkdelivery.BuilderBuildsHistory}, + {&b.modules, sdkdelivery.ModulesBucket, sdkdelivery.ModulesHistory}, + {&b.manifests, sdkdelivery.ManifestsBucket, 1}, + } { + kv, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: c.bucket, History: c.history}) + if err != nil { + t.Fatal(err) + } + *c.into = kv + } + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + withTrunkList(t, true) + return b +} + +// indexed writes the indexer's trunk value of module at commit, naming fp, and its manifest: a module of one archive. +func (b builderOnTheBus) indexed(t *testing.T, module, version, repository, path, commit, fp string) string { + t.Helper() + keys, trunk := aKeyFor(module, version, fp, repository, path, commit) + raw := keys.manifests[trunk.Manifest] + value, err := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: module, JSON: raw}) + if err != nil { + t.Fatal(err) + } + if _, err := b.manifests.Put(context.Background(), trunk.Manifest, value); err != nil { + t.Fatal(err) + } + trunk.FP = "src1:" + fp[len("src2:"):] + encoded, err := sdkdelivery.EncodeModule(trunk) + if err != nil { + t.Fatal(err) + } + if _, err := b.modules.Put(context.Background(), module, encoded); err != nil { + t.Fatal(err) + } + return trunk.Manifest +} + +// built writes the builder's key of module at fp: done, its one archive named with a reference, or in another state. +func (b builderOnTheBus) built(t *testing.T, module, version, fp, manifest, state string) { + t.Helper() + keys, _ := aKeyFor(module, version, fp, "", "", "") + v := keys.keys[module].Build + v.Manifest = manifest + if state != sdkdelivery.BuildDone { + v = sdkdelivery.BuilderBuild{State: state, By: "ace"} + if state == sdkdelivery.BuildFailed { + v.Error = "digests differ: ace sha256:1, g14 sha256:2" + } + } + value, err := sdkdelivery.EncodeBuilderBuild(v) + if err != nil { + t.Fatal(err) + } + key, err := sdkdelivery.BuilderBuildKey(module, fp) + if err != nil { + t.Fatal(err) + } + if _, err := b.builds.Put(context.Background(), key, value); err != nil { + t.Fatal(err) + } +} + +// noBuildAsked fails the test if anything asks build-agent for a build (ADR 0501 §6, ADR 0503 §6). +func noBuildAsked(t *testing.T) { + t.Helper() + was := askABuild + askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) { + t.Errorf("a build was asked of build-agent, at %q", path) + return "", context.Canceled + } + t.Cleanup(func() { askABuild = was }) +} + +// asksWithPaths records every build a merge asks for, as repository, path and ref. The merge window's goroutine and +// the test's own may ask at once, so the record is written under a lock (issue 1543). +func asksWithPaths(t *testing.T) *[][3]string { + t.Helper() + var asked [][3]string + var mu sync.Mutex + was := askABuild + askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { + mu.Lock() + defer mu.Unlock() + asked = append(asked, [3]string{source.Repository, path, ref}) + return "b-" + path, nil + } + t.Cleanup(func() { askABuild = was }) + return &asked +} diff --git a/cmd/mesh-controller/first_build_ask_integration_test.go b/cmd/mesh-controller/first_build_ask_integration_test.go deleted file mode 100644 index 68d84fdd..00000000 --- a/cmd/mesh-controller/first_build_ask_integration_test.go +++ /dev/null @@ -1,84 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "errors" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/identity" - "github.com/novox/mesh-controller/internal/link" -) - -// aForgeAndABuildSeat gives a test what a real ask of a build needs: the forge holding the git seat, assigned to anchor, so -// the clone URL composes; the identity store; the build seat's queue on a real bus; and the environment of the serving controller, which is -// never the controller's terminal (a verb's, in effect: startedAtTheTerminal is false). -func aForgeAndABuildSeat(t *testing.T, open *stores) *broker.JetStream { - t.Helper() - js, _ := aBuildQueue(t) - // The ask signs what it sends with the controller's identity, as the serving controller does. - identity.ForTest(t) - forge, err := catalogue.ParseManifest([]byte(`{"module":"gitea","version":"1", - "claims":[{"name":"git","scope":"mesh"}],"provides":[{"name":"git","scope":"mesh"}], - "serves":{"git":{"scheme":"http","port":3000}}}`)) - if err != nil { - t.Fatal(err) - } - register(t, open, forge) - if _, err := assign(t.Context(), open, "anchor", "gitea"); err != nil { - t.Fatal(err) - } - t.Setenv(servedVar, "1") - return js -} - -// 1146/AC9, through the real ask (askABuild, buildOneAsked, verbMayAsk) and a real bus: the first build of a module a -// merge adds on the operator's Approve is queued on the build seat; the same ask without that warrant, and the warrant's -// ask for a path the merge does not hold, are refused by the guard of a build through a verb. -func TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued(t *testing.T) { - const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000" - open := aMesh(t) - ctx := t.Context() - js := aForgeAndABuildSeat(t, open) - - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) - wasKept, wasAsk := keptDeliveryMerge, routerAskOf - keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) { - return a.merge, true, nil - } - routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) { - return &a.ask, &a.warrant, nil - } - t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) - - // Without the warrant the guard refuses, whatever the repository holds. - _, err := askABuild(ctx, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", "main") - var policy *heldAtTheTerminal - if !errors.As(err, &policy) { - t.Fatalf("1146/AC9: a first build asked without the warrant was not refused: %v", err) - } - // And a path the merge does not hold is refused on the warrant. - warranted := withFirstBuildWarrant(ctx, "novox/mesh-delivery", []string{""}, "the Approve") - if _, err := askABuild(warranted, buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", "main"); !errors.As(err, &policy) { - t.Fatalf("1146/AC9: the warrant covered a path the merge does not hold: %v", err) - } - - m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7, - MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"}, - ModuleDirs: []string{"."}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - queue, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(queue.Asks) != 1 { - t.Fatalf("1146/AC9: the build queue holds %+v, want the one first build", queue) - } -} diff --git a/cmd/mesh-controller/first_build_warrant_test.go b/cmd/mesh-controller/first_build_warrant_test.go deleted file mode 100644 index e72b3965..00000000 --- a/cmd/mesh-controller/first_build_warrant_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package main - -import "testing" - -// 1146/AC9: the warrant the controller checked covers exactly the repository on the git seat and the module paths the -// merge holds; nothing else passes verbMayAsk on it. -func TestTheFirstBuildWarrantCoversOnlyItsRepositoryAndItsModulePaths(t *testing.T) { - ctx := withFirstBuildWarrant(t.Context(), "novox/mesh-delivery", []string{"", "view"}, "the Approve") - w := firstBuildWarrantIn(ctx) - for _, c := range []struct { - source buildSource - path string - covers bool - }{ - {buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "", true}, - {buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "/view/", true}, - {buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "other", false}, - {buildSource{Repository: "agent/mesh-delivery", Seat: "git"}, "", false}, - {buildSource{Repository: "novox/mesh-delivery"}, "", false}, - {buildSource{Repository: "novox/mesh-delivery", Seat: "elsewhere"}, "", false}, - } { - if got := w.covers(c.source, c.path); got != c.covers { - t.Errorf("1146/AC9: %v at %q covered = %v, want %v", c.source, c.path, got, c.covers) - } - } - if firstBuildWarrantIn(t.Context()).covers(buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, "") { - t.Error("1146/AC9: a context with no warrant covers an ask") - } -} diff --git a/cmd/mesh-controller/first_builds.go b/cmd/mesh-controller/first_builds.go index fb05302b..c6f6322c 100644 --- a/cmd/mesh-controller/first_builds.go +++ b/cmd/mesh-controller/first_builds.go @@ -47,7 +47,7 @@ func (f following) firstBuilds(ctx context.Context, m link.SourceMoved, entries case len(added) == 0: return false, nil } - if err := askFirstBuilds(ctx, inv, m, now, added, why); err != nil { + if err := askFirstBuilds(ctx, f.open, m, now, added, why); err != nil { return true, notNow(err) } return true, nil @@ -68,27 +68,35 @@ func keepFirstMerge(ctx context.Context, inv *inventory.Inventory, m link.Source return err } -// askFirstBuilds keeps the merge, asks the builds, and marks the merge as no longer waiting for them. -func askFirstBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, now time.Time, added []string, +// askFirstBuilds keeps the merge as answered and registers its modules from the builder's done keys (novox/hq ADR 0501 +// §3, §6): nothing asks their first builds. The merge kept is the registration's warrant (approvedMove reads it by the +// commit the indexer's trunk value names); a module whose key is not done yet is registered when the key's write wakes +// the take-in (new_modules.go). +func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now time.Time, added []string, why string) error { + inv := open.inventory repository := m.Owner + "/" + m.Repo if err := keepFirstMerge(ctx, inv, m, now, false); err != nil { return err } - fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so the "+ - "first build of %s is asked\n", repository, m.Base, m.Commit, why, strings.Join(added, ", ")) - // The ask carries the warrant just checked, for this repository and these module paths alone: the guard of a build - // through a verb (verbMayAsk) lets exactly that through. - paths := make([]string, 0, len(added)) - for _, dir := range added { - if dir == "." { - dir = "" - } - paths = append(paths, dir) + fmt.Printf("%s merged into %s (%.8s) on the operator's Approve (%s): the mesh builds nothing from it yet, so %s "+ + "are registered from the builder's done keys, asking no build\n", repository, m.Base, m.Commit, why, + strings.Join(added, ", ")) + // Answered: it no longer waits, so the word's arrival again does nothing twice. + if err := inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false); err != nil { + return err } - askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, m, buildSource{Repository: repository, Seat: gitSeat}, added) - // Answered: it no longer waits, so the word's arrival again asks nothing twice. - return inv.AnswerMerge(ctx, repository, m.Commit, inventory.WaitingRecord, false) + modules, err := newModulesOf(ctx, repository) + switch { + case err != nil: + // Said, not failed: the merge is answered, and the look at every module when a controller starts, or the next + // write of the builder or the indexer for one of them, registers them. + fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+ + "write of one wakes the take-in\n", repository, err) + case len(modules) > 0: + go takeNewModules(context.WithoutCancel(ctx), open, modules...) + } + return nil } // resumeFirstBuilds runs firstBuilds for the first merges waiting for the word mesh-delivery just announced for a pull @@ -126,7 +134,7 @@ func (f following) resumeFirstBuilds(ctx context.Context, d deliveryMerge) error case err != nil: return notNow(err) case len(added) > 0: - if err := askFirstBuilds(ctx, inv, k.SourceMoved, time.Now().UTC(), added, why); err != nil { + if err := askFirstBuilds(ctx, f.open, k.SourceMoved, time.Now().UTC(), added, why); err != nil { return notNow(err) } } @@ -174,7 +182,9 @@ func firstBuildsOf(ctx context.Context, inv catalogueReader, m link.SourceMoved, } // heldFirstBuild says a first build was refused because its manifest uses a word the running build-agent does not read, -// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). +// and its merge is kept waiting for a build-agent that reads it (novox/hq ADR 0482 §5, issue 1173). Read from a merge kept +// before R4e, never written since: nothing asks a first build (ADR 0501 §6), and a controller that starts answers such a +// merge and registers its modules from their keys (answerHeldMerges). type heldFirstBuild struct { Module string `json:"module"` Path string `json:"path"` @@ -182,52 +192,6 @@ type heldFirstBuild struct { At time.Time `json:"at"` } -// judgeRefusedFirstBuild is the hold a first build's outcome puts its merge under: every module the merge holds already, -// and this one, unless it is among them; nil unless the build failed on a word of its manifest the running build-agent -// does not know, and was asked for a first merge, answered by the record of merges no plan answers or put back to waiting -// by an earlier hold (a second refusal in one merge is held too, review of issue 1173). -func judgeRefusedFirstBuild(result link.BuildResult, req inventory.BuildRequest, merge inventory.BatchedMerge) []heldFirstBuild { - word := refusedWord(result.Failed) - if word == "" || req.For != "merge" { - return nil - } - var held []heldFirstBuild - switch k := announcedOf(merge); { - case merge.Plan == inventory.WaitingRecord: - // Answered when its builds were asked: a hold from before (a build asked again) is over, and starts anew. - case merge.Plan == "" && len(k.Held) > 0: - held = k.Held - default: - return nil - } - this := heldFirstBuild{Module: req.Name(), Path: strings.Trim(req.Path, "/"), Word: word, At: time.Now().UTC()} - for i, h := range held { - if strings.Trim(h.Path, "/") == this.Path { - out := append([]heldFirstBuild(nil), held...) - out[i] = this - return out - } - } - return append(append([]heldFirstBuild(nil), held...), this) -} - -// heldDirsOf is the directories a held merge asks again: its held modules' own, each once, "." for the repository's root. -func heldDirsOf(k keptMerge) []string { - var dirs []string - seen := map[string]bool{} - for _, h := range k.Held { - dir := strings.Trim(h.Path, "/") - if dir == "" { - dir = "." - } - if !seen[dir] { - seen[dir] = true - dirs = append(dirs, dir) - } - } - return dirs -} - // unknownWord is how the strict reading of a manifest says a word is not one it knows: encoding/json's own words, which // the build-agent's answer carries whole. The build's answer has no typed field for it (Failed is the builder's words). var unknownWord = regexp.MustCompile(`json: unknown field "([^"]+)"`) @@ -240,55 +204,12 @@ func refusedWord(failed string) string { return "" } -// holdRefusedFirstBuild keeps a first build's merge waiting when the build was refused for a word the running build-agent -// does not read (novox/hq ADR 0482 §5, issue 1173): no failed build is recorded, nothing expires the merge, and it is -// named by a condition until a delivery of build-agent asks the build again. Answers whether the outcome was dealt with. -func holdRefusedFirstBuild(ctx context.Context, inv *inventory.Inventory, result link.BuildResult) (bool, error) { - if result.Failed == "" || refusedWord(result.Failed) == "" { - return false, nil - } - req, found, err := inv.BuildRequestByID(ctx, result.ID) - if err != nil || !found { - return false, err - } - merge, found, err := inv.MergeOf(ctx, req.Repository, req.Commit) - if err != nil || !found { - return false, err - } - held := judgeRefusedFirstBuild(result, req, merge) - if len(held) == 0 { - return false, nil - } - k := announcedOf(merge) - k.Held = held - event, err := json.Marshal(k) - if err != nil { - return false, err - } - if err := inv.HoldMerge(ctx, merge.Repository, merge.Commit, event); err != nil { - return false, err - } - this := held[len(held)-1] - for _, h := range held { - if h.Path == strings.Trim(req.Path, "/") { - this = h - } - } - fmt.Printf("%s: the first build of %s was refused for the word %q, which the running build-agent does not read; the "+ - "merge %.8s of %s is kept waiting for a build-agent that reads it (holding %s), and the build is asked again when "+ - "one is delivered\n", result.ID, this.Module, this.Word, merge.Commit, merge.Repository, heldNames(held)) - return true, nil -} - // probeFirstBuildsID is the probe that reads the first merges waiting for mesh-delivery's word. const probeFirstBuildsID = "D18" // kindFirstBuildUnasked is what a first merge waiting too long for mesh-delivery's word raises. const kindFirstBuildUnasked = "first-build-unasked" -// kindFirstBuildHeld is what a first merge held for a build-agent that reads its manifest raises. -const kindFirstBuildHeld = "first-build-held" - // firstBuildWaitBound is how long a first merge waits for mesh-delivery's announcement of the operator's Approve before // it is said: a bound in the sense of ADR 0426 §3, not a delay. Nothing waits on it: the word's arrival asks the builds // at once, and the merge is said, never dropped, once it is past this (issue 1146). @@ -300,10 +221,6 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c var out []conditions.Observation for _, w := range waiting { k := announcedOf(w) - if len(k.Held) > 0 { - out = append(out, heldFirstBuildObservation(k)) - continue - } if !k.FirstBuild || now.Sub(w.Heard) <= firstBuildWaitBound { continue } @@ -318,108 +235,43 @@ func firstBuildObservations(waiting []inventory.BatchedMerge, now time.Time) []c Headline: fmt.Sprintf("The first build of %s is not asked: no Approve was announced", repository), Explanation: fmt.Sprintf("The merge %.8s into %s adds modules the mesh builds nothing from yet. It registers them "+ "only when mesh-delivery announces that it merged the pull request on your Approve of that head, and no such "+ - "announcement has come in %s. The merge is kept, and the builds are asked the moment the announcement is kept.", + "announcement has come in %s. The merge is kept, and the modules are registered from the builder's keys the moment "+ + "the announcement is kept.", k.Commit, repository, for_), - Resolved: "mesh-delivery announces the merge, or a hand `build` registers the modules", + Resolved: "mesh-delivery announces the merge, or the operator registers the modules at the controller's terminal", }) } return out } -// heldNames is the modules a merge holds, each with the word it waits for: "watchman (key-ttl), view (per-machine)". -func heldNames(held []heldFirstBuild) string { - names := make([]string, 0, len(held)) - for _, h := range held { - names = append(names, fmt.Sprintf("%s (%q)", h.Module, h.Word)) - } - return strings.Join(names, ", ") -} - -// heldFirstBuildObservation names a first merge held for a build-agent that reads its manifests: at once, since nothing -// is lost and nothing times out, but the operator can see what the merge waits for. One condition names every module held. -func heldFirstBuildObservation(k keptMerge) conditions.Observation { - repository := k.Owner + "/" + k.Repo - modules := make([]string, 0, len(k.Held)) - for _, h := range k.Held { - modules = append(modules, h.Module) - } - held := heldNames(k.Held) - return conditions.Observation{Scope: conditions.ScopeMerge, ID: fmt.Sprintf("%s_%.8s", repository, k.Commit), - Kind: kindFirstBuildHeld, Severity: conditions.Warning, - Summary: fmt.Sprintf("the merge %.8s of %s adds modules whose manifests use words the running build-agent does not "+ - "read: %s; the merge waits for a build-agent that reads them", k.Commit, repository, held), - Said: fmt.Sprintf("%s at %s waits for a build-agent that reads the words of %s", repository, k.Commit, held), - Headline: fmt.Sprintf("The first build of %s waits for a newer build-agent", strings.Join(modules, ", ")), - Explanation: fmt.Sprintf("The merge %.8s into %s adds modules whose manifests use words the build-agent now "+ - "running does not know: %s. The merge is kept, not failed: their first builds are asked again when a delivery "+ - "of build-agent ends.", k.Commit, repository, held), - Resolved: "a delivery of build-agent ends and the first builds register the modules, or a hand `build` does", - } -} - // probeFirstBuilds is D18: no first merge waits past its bound for mesh-delivery's word unsaid. func probeFirstBuilds(ctx context.Context, d *doctor) ([]conditions.Observation, error) { waiting, err := d.open.inventory.WaitingMerges(ctx) if err != nil { return nil, err } - return firstBuildObservations(waiting, time.Now().UTC()), nil + now := time.Now().UTC() + return append(firstBuildObservations(waiting, now), newModuleObservations(now)...), nil } -// askHeldFirstBuildsAgain asks again the first build of each first merge held for a build-agent that reads its manifest, -// when the plan that ended delivered build-agent (novox/hq issue 1173): once for each end, since the ask answers the merge -// and a refusal that comes again holds it anew. Answers how many merges it asked for. -func askHeldFirstBuildsAgain(ctx context.Context, open *stores, p inventory.Plan) int { - if !deliversBuildAgent(p) { - return 0 - } - actingOnMerges.Lock() - defer actingOnMerges.Unlock() - inv := open.inventory +// answerHeldMerges answers every merge a controller of before held for a build-agent that reads its new modules' +// manifests (issue 1173): nothing asks their first builds any more, and the look at every module when this controller +// starts registers them from their keys (ADR 0501 §6, §7). +func answerHeldMerges(ctx context.Context, inv *inventory.Inventory) { waiting, err := inv.WaitingMerges(ctx) if err != nil { - fmt.Printf("%s delivered build-agent, and the first merges held for it could not be read: %v\n", p.ID, err) - return 0 + fmt.Printf("the merges held for a newer build-agent could not be read: %v\n", err) + return } - asked := 0 for _, w := range waiting { - k := announcedOf(w) - if len(k.Held) == 0 { - continue - } - // Its own held modules, not the rule of a first merge: a module of the merge that passed is registered by now, - // and the repository no longer counts as one the mesh builds nothing from (review of issue 1173). - why, err := approvedAnnounced(ctx, inv, k.Owner, k.Repo, keptMerge{SourceMoved: k.SourceMoved}) - if err != nil { - fmt.Printf("%s delivered build-agent, and the merge %.8s of %s held for it asks no build now (%v): it stays "+ - "waiting\n", p.ID, w.Commit, w.Repository, err) - continue - } - dirs := heldDirsOf(k) - fmt.Printf("%s delivered build-agent: the first builds of %s are asked again\n", p.ID, heldNames(k.Held)) - paths := make([]string, 0, len(dirs)) - for _, dir := range dirs { - if dir == "." { - dir = "" + if k := announcedOf(w); len(k.Held) > 0 { + if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil { + fmt.Printf("the merge %.8s of %s held for a newer build-agent could not be answered: %v\n", w.Commit, + w.Repository, err) + continue } - paths = append(paths, dir) + fmt.Printf("the merge %.8s of %s was held for a newer build-agent; its modules are registered from the "+ + "builder's keys instead\n", w.Commit, w.Repository) } - repository := k.Owner + "/" + k.Repo - askBuilds(withFirstBuildWarrant(ctx, repository, paths, why), inv, k.SourceMoved, - buildSource{Repository: repository, Seat: gitSeat}, dirs) - // Answered: it no longer waits, so this end asks nothing twice; a refusal that comes again holds it anew. - if err := inv.AnswerMerge(ctx, w.Repository, w.Commit, inventory.WaitingRecord, false); err != nil { - fmt.Printf(" and the merge could not be marked answered: %v\n", err) - continue - } - asked++ } - return asked -} - -// deliversBuildAgent says a delivery plan ended delivered with build-agent built in it: the reader a held first build -// waits for is the one now running. -func deliversBuildAgent(p inventory.Plan) bool { - m := p.Modules["build-agent"] - return p.WalkSaid == inventory.WalkEnded && p.Outcome() == inventory.WalkDelivered && m != nil && m.State == "built" } diff --git a/cmd/mesh-controller/first_builds_held_integration_test.go b/cmd/mesh-controller/first_builds_held_integration_test.go deleted file mode 100644 index fb69d5e2..00000000 --- a/cmd/mesh-controller/first_builds_held_integration_test.go +++ /dev/null @@ -1,221 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "fmt" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// 1173/AC1, 1173/AC3 whole against the store: a first merge's build refused for a word the running build-agent does not -// read keeps the merge waiting with no failed build recorded; a delivery plan that does not deliver build-agent asks -// nothing; the one that ends delivering it asks the first build again once, and its pass registers the module. -func TestAHeldFirstMergeIsAskedAgainWhenABuildAgentIsDelivered(t *testing.T) { - const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000" - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) - wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild - keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) { - return a.merge, true, nil - } - routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) { - return &a.ask, &a.warrant, nil - } - var asked []string - askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { - id := "build-" + string(rune('1'+len(asked))) - asked = append(asked, id) - return id, nil - } - t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk }) - - m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7, - MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), Paths: []string{"module.json", "main.go"}, - ModuleDirs: []string{"."}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - if len(asked) != 1 { - t.Fatalf("1173/AC3: the first merge asked %v, want the one first build", asked) - } - - // The running build-agent refuses the manifest's word. - refused := onTrunk("build-1", "novox/mesh-delivery", "git", "", nil) - refused.Failed, refused.Manifest, refused.Commit = refusedKeyTTL, nil, commit - if err := (builds{inv: inv, open: open}).Built(ctx, refused); err != nil { - t.Fatal(err) - } - if _, recorded, _ := inv.BuildByID(ctx, "build-1"); recorded { - t.Fatal("1173/AC1: a refused first build was recorded as a failed build") - } - waiting, err := inv.WaitingMerges(ctx) - if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) == 0 { - t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, held", waiting, err) - } - if obs := firstBuildObservations(waiting, time.Now()); len(obs) != 1 || obs[0].Kind != kindFirstBuildHeld { - t.Fatalf("1173/AC2: the held merge is named by %+v", obs) - } - - // A plan that ended without delivering build-agent asks nothing; one that delivered it asks once. - other := inventory.Plan{ID: "p1", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded, - Modules: map[string]*inventory.PlanModule{"mesh-delivery": {State: "built"}}} - if n := askHeldFirstBuildsAgain(ctx, open, other); n != 0 || len(asked) != 1 { - t.Fatalf("1173/AC3: a plan that delivered no build-agent asked %d builds again (%v)", n, asked) - } - delivered := inventory.Plan{ID: "p2", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded, - Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}} - if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 1 || len(asked) != 2 { - t.Fatalf("1173/AC3: the plan that delivered build-agent asked %d builds again (%v), want one", n, asked) - } - if n := askHeldFirstBuildsAgain(ctx, open, delivered); n != 0 || len(asked) != 2 { - t.Fatalf("1173/AC3: the same end asked again: %d (%v)", n, asked) - } - - // Its pass registers the module. - pass := onTrunk("build-2", "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-watchman", "version": "1"}) - pass.Commit = commit - if err := (builds{inv: inv, open: open}).Built(ctx, pass); err != nil { - t.Fatal(err) - } - if _, known := inventoryHolds(t, inv, "mesh-delivery-watchman"); !known { - t.Fatal("1173/AC3: the module is not registered after the first build passed") - } -} - -// aTwoModuleFirstMerge is a first merge into novox/mesh-delivery adding the modules watchman and view, made on the -// operator's Approve of its head, whose first builds are asked; asked says the build id asked for each path, in order. -func aTwoModuleFirstMerge(t *testing.T, open *stores, commit string) *[][2]string { - t.Helper() - const head = "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2" - a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@b2b2b2b2b2b2": head}) - wasKept, wasAsk, realAsk := keptDeliveryMerge, routerAskOf, askABuild - keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) { - return a.merge, true, nil - } - routerAskOf = func(context.Context, catalogueReader, string, string) (*asks.Ask, *asks.Warrant, error) { - return &a.ask, &a.warrant, nil - } - asked := &[][2]string{} - askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) { - id := fmt.Sprintf("build-%d", len(*asked)+1) - *asked = append(*asked, [2]string{path, id}) - return id, nil - } - t.Cleanup(func() { keptDeliveryMerge, routerAskOf, askABuild = wasKept, wasAsk, realAsk }) - m := link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", HeadSHA: head, Commit: commit, Number: 7, - MergedAt: time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano), - Paths: []string{"view/module.json", "view/main.go", "watchman/module.json", "watchman/main.go"}, - ModuleDirs: []string{"view", "watchman"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { - t.Fatal(err) - } - if len(*asked) != 2 { - t.Fatalf("the first merge asked %v, want the two first builds", *asked) - } - return asked -} - -// idOf is the build asked last for path. -func idOf(t *testing.T, asked [][2]string, path string) string { - t.Helper() - for i := len(asked) - 1; i >= 0; i-- { - if asked[i][0] == path { - return asked[i][1] - } - } - t.Fatalf("no build asked for %s: %v", path, asked) - return "" -} - -// built hands the controller a build's outcome: refused for the word, or passing as module. -func built(t *testing.T, open *stores, id, path, commit, refusedFor, module string) { - t.Helper() - r := onTrunk(id, "novox/mesh-delivery", "git", path, map[string]any{"module": module, "version": "1"}) - r.Commit = commit - if refusedFor != "" { - r.Failed, r.Manifest = `this is not a module manifest: json: unknown field "`+refusedFor+`"`, nil - } - if err := (builds{inv: open.inventory, open: open}).Built(t.Context(), r); err != nil { - t.Fatal(err) - } -} - -var buildAgentDelivered = inventory.Plan{ID: "p-ba", State: inventory.PlanDone, WalkSaid: inventory.WalkEnded, - Modules: map[string]*inventory.PlanModule{"build-agent": {State: "built"}}} - -// 1173/AC3 (review round 1, item 1): in a first merge of two modules where view passes and watchman is refused, the -// held watchman is asked again when build-agent is delivered, though the repository builds view by then. -func TestAHeldModuleIsAskedAgainAfterItsSiblingRegistered(t *testing.T) { - const commit = "0177ea7b00000000" - open := aMesh(t) - ctx := t.Context() - asked := aTwoModuleFirstMerge(t, open, commit) - - built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view") - if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-view"); !known { - t.Fatal("view is not registered after its first build passed") - } - refused := idOf(t, *asked, "watchman") - built(t, open, refused, "watchman", commit, "key-ttl", "") - if _, recorded, _ := open.inventory.BuildByID(ctx, refused); recorded { - t.Fatal("1173/AC1: the refused first build of watchman was recorded as a failed build") - } - waiting, err := open.inventory.WaitingMerges(ctx) - if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 1 { - t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding watchman", waiting, err) - } - - if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 3 || (*asked)[2][0] != "watchman" { - t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want watchman's first build alone", n, *asked) - } - built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman") - if _, known := inventoryHolds(t, open.inventory, "mesh-delivery-watchman"); !known { - t.Fatal("1173/AC3: watchman is not registered after the first build asked again passed") - } -} - -// 1173/AC1, AC2, AC3 (review round 1, item 2): in a first merge of two modules both refused, neither is recorded as a -// failed build, the condition names both, and the delivery of build-agent asks both again. -func TestTwoRefusedModulesOfOneFirstMergeAreBothHeld(t *testing.T) { - const commit = "0177ea7c00000000" - open := aMesh(t) - ctx := t.Context() - asked := aTwoModuleFirstMerge(t, open, commit) - - for _, path := range []string{"watchman", "view"} { - id := idOf(t, *asked, path) - built(t, open, id, path, commit, "key-ttl", "") - if _, recorded, _ := open.inventory.BuildByID(ctx, id); recorded { - t.Fatalf("1173/AC1: the refused first build of %s was recorded as a failed build", path) - } - } - waiting, err := open.inventory.WaitingMerges(ctx) - if err != nil || len(waiting) != 1 || len(announcedOf(waiting[0]).Held) != 2 { - t.Fatalf("1173/AC1: the merges waiting are %+v (%v), want the first merge, holding both", waiting, err) - } - obs := firstBuildObservations(waiting, time.Now()) - if len(obs) != 1 || !strings.Contains(obs[0].Summary, "watchman") || !strings.Contains(obs[0].Summary, "view") { - t.Fatalf("1173/AC2: the held merge is named by %+v, want one condition naming both", obs) - } - - if n := askHeldFirstBuildsAgain(ctx, open, buildAgentDelivered); n != 1 || len(*asked) != 4 { - t.Fatalf("1173/AC3: the delivery of build-agent asked %d merges again (%v), want both first builds", n, *asked) - } - built(t, open, idOf(t, *asked, "watchman"), "watchman", commit, "", "mesh-delivery-watchman") - built(t, open, idOf(t, *asked, "view"), "view", commit, "", "mesh-delivery-view") - for _, module := range []string{"mesh-delivery-watchman", "mesh-delivery-view"} { - if _, known := inventoryHolds(t, open.inventory, module); !known { - t.Fatalf("1173/AC3: %s is not registered after the first build asked again passed", module) - } - } -} diff --git a/cmd/mesh-controller/first_builds_held_test.go b/cmd/mesh-controller/first_builds_held_test.go deleted file mode 100644 index 96110263..00000000 --- a/cmd/mesh-controller/first_builds_held_test.go +++ /dev/null @@ -1,141 +0,0 @@ -package main - -import ( - "encoding/json" - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -const refusedKeyTTL = `this is not a module manifest: a state is either a name or {name, history, ttl-seconds, per-machine}: ` + - `json: unknown field "key-ttl"` - -// aFirstMergeAnswered is a first merge as askFirstBuilds leaves it: answered by the record of merges no plan answers. -func aFirstMergeAnswered() (inventory.BuildRequest, inventory.BatchedMerge) { - req := inventory.BuildRequest{ID: "build-1", Repository: "novox/mesh-delivery", Seat: "git", Path: "watchman", Ref: "main", - Commit: "0177ea7a00", For: "merge"} - event, _ := json.Marshal(keptMerge{SourceMoved: link.SourceMoved{Owner: "novox", Repo: "mesh-delivery", Base: "main", - Commit: "0177ea7a00", Number: 3, HeadSHA: "a1a1a1a1a1a1"}, FirstBuild: true}) - return req, inventory.BatchedMerge{Repository: "novox/mesh-delivery", Branch: "main", Commit: "0177ea7a00", - Plan: inventory.WaitingRecord, Event: event} -} - -// 1173/AC1: a first build refused for a word its manifest uses and the running build-agent does not know holds its merge. -func TestAFirstBuildRefusedForAnUnknownWordHoldsItsMerge(t *testing.T) { - req, merge := aFirstMergeAnswered() - held := judgeRefusedFirstBuild(link.BuildResult{ID: "build-1", Failed: refusedKeyTTL}, req, merge) - if len(held) != 1 || held[0].Word != "key-ttl" || held[0].Module != "watchman" || held[0].Path != "watchman" { - t.Fatalf("1173/AC1: a first build refused for \"key-ttl\" is judged %+v, want a hold on that word for watchman", held) - } -} - -// 1173/AC2: a held first merge is named at once by one condition: the merge, the module, the word and what it waits for. -func TestAHeldFirstMergeIsNamedByACondition(t *testing.T) { - now := time.Date(2026, 10, 22, 14, 0, 0, 0, time.UTC) - _, merge := aFirstMergeAnswered() - var k keptMerge - _ = json.Unmarshal(merge.Event, &k) - k.Held = []heldFirstBuild{{Module: "watchman", Path: "watchman", Word: "key-ttl", At: now}, - {Module: "view", Path: "view", Word: "per-machine", At: now}} - merge.Event, _ = json.Marshal(k) - merge.Heard = now.Add(-time.Second) - got := firstBuildObservations([]inventory.BatchedMerge{merge}, now) - if len(got) != 1 { - t.Fatalf("1173/AC2: %d conditions for a held first merge, want one: %+v", len(got), got) - } - o := got[0] - if o.Kind != kindFirstBuildHeld || o.Scope != conditions.ScopeMerge { - t.Fatalf("1173/AC2: the condition is %+v", o) - } - for _, want := range []string{"0177ea7a", "watchman", "key-ttl", "view", "per-machine", "build-agent"} { - if !strings.Contains(o.Summary, want) { - t.Fatalf("1173/AC2: the summary %q does not name %q", o.Summary, want) - } - } -} - -// 1173/AC4: a first build that fails for any other reason is not held. -func TestAFirstBuildFailingForAnyOtherReasonIsNotHeld(t *testing.T) { - req, merge := aFirstMergeAnswered() - if judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge) == nil { - t.Fatal("1173/AC4: the control, a refusal for an unknown word, is not held") - } - for _, c := range []struct { - what string - failed string - change func(*inventory.BuildRequest, *inventory.BatchedMerge) - }{ - {what: "a compile error", failed: "main.go:3: undefined: foo"}, - {what: "no failure", failed: ""}, - {what: "a missing field", failed: `json: cannot unmarshal string into Go struct field`}, - {what: "a build a person asked", failed: refusedKeyTTL, change: func(r *inventory.BuildRequest, _ *inventory.BatchedMerge) { r.For = "build" }}, - {what: "a merge a plan answers", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "plan-7" }}, - {what: "a waiting merge held for nothing", failed: refusedKeyTTL, change: func(_ *inventory.BuildRequest, m *inventory.BatchedMerge) { m.Plan = "" }}, - } { - r, m := req, merge - if c.change != nil { - c.change(&r, &m) - } - if held := judgeRefusedFirstBuild(link.BuildResult{Failed: c.failed}, r, m); held != nil { - t.Fatalf("1173/AC4: %s is held: %+v", c.what, held) - } - } -} - -// 1173/AC1, AC2 (review round 1, item 2): a second refusal in a merge already held is held too, its module added to the -// hold, so no failed build is recorded and the condition names both. -func TestASecondRefusalInAHeldMergeIsAddedToTheHold(t *testing.T) { - req, merge := aFirstMergeAnswered() - first := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge) - k := announcedOf(merge) - k.Held = first - merge.Event, _ = json.Marshal(k) - merge.Plan = "" // HoldMerge put it back to waiting - req.ID, req.Path = "build-2", "view" - held := judgeRefusedFirstBuild(link.BuildResult{Failed: `json: unknown field "per-machine"`}, req, merge) - if len(held) != 2 || held[0].Module != "watchman" || held[1].Module != "view" || held[1].Word != "per-machine" { - t.Fatalf("1173/AC1: the second refusal in a held merge is judged %+v, want watchman and view held", held) - } - // The same module refused again is held once. - k.Held = held - merge.Event, _ = json.Marshal(k) - if again := judgeRefusedFirstBuild(link.BuildResult{Failed: refusedKeyTTL}, req, merge); len(again) != 2 { - t.Fatalf("1173/AC1: view refused again is held as %+v, want it once", again) - } -} - -// 1173/AC3 (review round 1, item 1): a held merge is asked again for its own held modules' directories, not by the rule of -// a first merge, which finds nothing once another module of the repository is registered. -func TestAHeldMergeIsAskedAgainForItsHeldModulesAlone(t *testing.T) { - k := keptMerge{Held: []heldFirstBuild{{Module: "watchman", Path: "watchman"}, {Module: "mesh-delivery", Path: ""}, - {Module: "watchman", Path: "/watchman/"}}} - if got := heldDirsOf(k); strings.Join(got, ",") != "watchman,." { - t.Fatalf("1173/AC3: a held merge asks again %v, want watchman and the root once each", got) - } -} - -// 1173/AC3 (the part that needs no store): only a plan that ended delivered with build-agent built in it is the end a held -// first build waits for. -func TestOnlyADeliveredPlanOfBuildAgentIsWhatAHeldFirstBuildWaitsFor(t *testing.T) { - plan := func(state, walk string, modules map[string]*inventory.PlanModule) inventory.Plan { - return inventory.Plan{ID: "p", State: state, WalkSaid: walk, Modules: modules} - } - built := map[string]*inventory.PlanModule{"build-agent": {State: "built"}} - if !deliversBuildAgent(plan(inventory.PlanDone, inventory.WalkEnded, built)) { - t.Fatal("1173/AC3: a plan that delivered build-agent is not the end a held first build waits for") - } - for what, p := range map[string]inventory.Plan{ - "a plan still running": plan(inventory.PlanBuilding, "", built), - "a plan that failed": plan(inventory.PlanFailed, inventory.WalkEnded, built), - "a failed build-agent": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"build-agent": {State: "failed"}}), - "another module's plan": plan(inventory.PlanDone, inventory.WalkEnded, map[string]*inventory.PlanModule{"mailu": {State: "built"}}), - } { - if deliversBuildAgent(p) { - t.Fatalf("1173/AC3: %s is the end a held first build waits for", what) - } - } -} diff --git a/cmd/mesh-controller/first_builds_integration_test.go b/cmd/mesh-controller/first_builds_integration_test.go index e82d7284..bab2559b 100644 --- a/cmd/mesh-controller/first_builds_integration_test.go +++ b/cmd/mesh-controller/first_builds_integration_test.go @@ -5,71 +5,48 @@ package main import ( "context" "encoding/json" - "slices" - "sync" "testing" "time" "github.com/nats-io/nats.go/jetstream" "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) -// 1146/AC7, whole against a real bus and the store, in both orders of the two events (novox/hq ADR 0432, issue 1146): -// mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's merge, -// heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the controller -// consumes with the router's record of the ask in its bucket, come in either order; each way the first build of the -// module the merge commit holds is asked exactly once, its outcome registers the module, and it is in the catalogue. -// Without the Approve in the record nothing is asked. Store-backed: skips without one. -func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule(t *testing.T) { +// 1146/AC7 on keys (ADR 0501 §3, §6; issue 1851, A5), whole against a real bus and the store, in both orders of the two +// events: mesh-delivery #7 merged on the operator's Approve into a repository the mesh builds nothing from. The forge's +// merge, heard as the controller hears it (SourceMoved), and mesh-delivery's word, published on the subject the +// controller consumes with the router's record of the ask in its bucket, come in either order; each way the module the +// merge commit holds is registered from the builder's done key of it, and nothing asks a build. Without the Approve in +// the record it is not registered. Store-backed: skips without one. +func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *testing.T) { const head, commit = "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "0177ea7a00000000" for _, c := range []struct { - what string - wordFirst bool - approved bool // the router's record holds the operator's Approve of the merged head - wantRegiter bool + what string + wordFirst bool + approved bool // the router's record holds the operator's Approve of the merged head + wantRegister bool }{ - {what: "the forge's merge before mesh-delivery's word", approved: true, wantRegiter: true}, - {what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegiter: true}, + {what: "the forge's merge before mesh-delivery's word", approved: true, wantRegister: true}, + {what: "mesh-delivery's word before the forge's merge", wordFirst: true, approved: true, wantRegister: true}, {what: "a word whose record holds no Approve", wordFirst: true}, } { t.Run(c.what, func(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory - // The real ask, guard and queue included: only its answers are kept, so the build's outcome can be matched. - aForgeAndABuildSeat(t, open) - // Written by the controller's merge path on its own goroutine, read by the test: under one lock. - var mu sync.Mutex - var asks [][3]string - var ids []string - asked := func() [][3]string { mu.Lock(); defer mu.Unlock(); return slices.Clone(asks) } - realAsk := askABuild - askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { - id, err := realAsk(c, source, path, ref) - if err == nil { - mu.Lock() - asks = append(asks, [3]string{source.Repository, path, ref}) - ids = append(ids, id) - mu.Unlock() - } - return id, err - } - t.Cleanup(func() { askABuild = realAsk }) + noBuildAsked(t) + b := aBuilderOnTheBus(t) + js := b.js if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "operator-channel", Version: "1", DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.AsksSeat, Records: []string{"asks"}}}}, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/operator-channel", BuiltFrom: "c0ffee"}); err != nil { t.Fatal(err) } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) if err := broker.AssertMeshStreams(js); err != nil { t.Fatal(err) } @@ -91,6 +68,10 @@ func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegis t.Fatal(err) } } + // The indexer's value and the builder's done key of the module the merge commit holds. + fp := fpOf('9') + manifest := b.indexed(t, "mesh-delivery-view", "1", "novox/mesh-delivery", ".", commit, fp) + b.built(t, "mesh-delivery-view", "1", fp, manifest, sdkdelivery.BuildDone) following := following{open: open} server := link.ConnectNats(js, nil, noListener{}) @@ -104,8 +85,6 @@ func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegis done := make(chan struct{}) go func() { defer close(done); _ = server.Serve(serving) }() t.Cleanup(func() { stop(); <-done }) - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(nil) }) word := func() { t.Helper() @@ -133,37 +112,27 @@ func TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegis forge() } else { forge() - if len(asked()) != 0 { - t.Fatalf("1146/AC7: the merge asked %v before mesh-delivery's word was kept", asked()) + time.Sleep(300 * time.Millisecond) + if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known { + t.Fatal("1146/AC7: registered before mesh-delivery's word was kept") } word() } - if !c.wantRegiter { - if len(asked()) != 0 { - t.Fatalf("1146/AC7: a merge whose record holds no Approve asked %v", asked()) + if !c.wantRegister { + time.Sleep(time.Second) + if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); known { + t.Fatal("1146/AC7: a merge whose record holds no Approve registered its module") } return } - want := [3]string{"novox/mesh-delivery", "", "main"} - becomes(t, 10*time.Second, "the first build asked", func() bool { return len(asked()) >= 1 }) - if got := asked(); len(got) != 1 || got[0] != want { - t.Fatalf("1146/AC7: the merge asked %v, want %v once", got, want) - } - mu.Lock() - first := ids[0] - mu.Unlock() - build := onTrunk(first, "novox/mesh-delivery", "git", "", map[string]any{"module": "mesh-delivery-view", "version": "1"}) - build.Commit = commit - if _, _, err := takeIn(ctx, inv, build); err != nil { - t.Fatalf("1146/AC7: the outcome of the first build was refused: %v", err) - } + becomes(t, 10*time.Second, "the module registered", func() bool { + _, known := inventoryHolds(t, inv, "mesh-delivery-view") + return known + }) src, err := inv.SourceOf(ctx, "mesh-delivery-view") if err != nil || src.Repository != "novox/mesh-delivery" { t.Fatalf("1146/AC7: the module is built from %q (%v)", src.Repository, err) } - if _, known := inventoryHolds(t, inv, "mesh-delivery-view"); !known { - t.Fatal("1146/AC7: the module is not in the catalogue") - } }) } } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 7f204dac..d3f380b2 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -408,13 +408,6 @@ func (b builds) Built(ctx context.Context, result link.BuildResult) error { fmt.Printf("%s: a dry run of %s on %s, not taken in\n", result.ID, result.Repository, result.Ref) return nil } - // **A first build refused for a word the running build-agent does not read waits for one that does** (novox/hq ADR - // 0482 §5, issue 1173): its merge is kept, named, and never failed; no failed build is recorded. - if held, err := holdRefusedFirstBuild(ctx, b.inv, result); err != nil { - return err - } else if held { - return nil - } manifest, _, err := takeIn(ctx, b.inv, result) // The assignments pending on this build, made or ended (novox/hq issue 325), said in the daemon's log // after what became of the build. diff --git a/cmd/mesh-controller/new_module_asked_once_test.go b/cmd/mesh-controller/new_module_asked_once_test.go deleted file mode 100644 index d2dc87fa..00000000 --- a/cmd/mesh-controller/new_module_asked_once_test.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "reflect" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" -) - -// 1617/AC2 (unit): a new module whose first build a merge's hearing asked is not asked again when its batch is cut, -// and one whose ask failed, was never handed over or is long lost is. -func TestAModuleWhoseBuildIsComingIsNotAskedAgain(t *testing.T) { - now := time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC) - source := buildSource{Repository: "http://forge/novox/mesh-catalog.git", Seat: "git"} - asked := func(path string, mod func(*inventory.RequestOutcome)) inventory.RequestOutcome { - r := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: "b-" + path, Repository: source.Repository, - Seat: "git", Path: path, For: "merge", At: now.Add(-3 * time.Minute)}} - if mod != nil { - mod(&r) - } - return r - } - requests := []inventory.RequestOutcome{ - asked("modules/go", nil), - asked("modules/failed", func(r *inventory.RequestOutcome) { r.Heard, r.Failed = true, "no module.json" }), - asked("modules/notasked", func(r *inventory.RequestOutcome) { r.NotAsked = "the bus is silent" }), - asked("modules/lost", func(r *inventory.RequestOutcome) { r.At = now.Add(-2 * time.Hour) }), - asked("modules/byhand", func(r *inventory.RequestOutcome) { r.For = "build" }), - } - got := unaskedDirs([]string{"modules/go", "modules/failed", "modules/notasked", "modules/lost", "modules/byhand", - "modules/nodejs"}, requests, source, now) - want := []string{"modules/failed", "modules/notasked", "modules/lost", "modules/byhand", "modules/nodejs"} - if !reflect.DeepEqual(got, want) { - t.Fatalf("asked again %v, want %v", got, want) - } -} diff --git a/cmd/mesh-controller/new_module_on_merge_test.go b/cmd/mesh-controller/new_module_on_merge_test.go index a5b38109..eb693721 100644 --- a/cmd/mesh-controller/new_module_on_merge_test.go +++ b/cmd/mesh-controller/new_module_on_merge_test.go @@ -4,134 +4,109 @@ package main import ( "context" - "sync" + "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) -// asksWithPaths records every build a merge asks for, as repository, path and ref. The merge window's goroutine and -// the test's own may ask at once, so the record is written under a lock (issue 1543). -func asksWithPaths(t *testing.T) *[][3]string { - t.Helper() - var asked [][3]string - var mu sync.Mutex - was := askABuild - askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { - mu.Lock() - defer mu.Unlock() - asked = append(asked, [3]string{source.Repository, path, ref}) - return "b-" + path, nil - } - t.Cleanup(func() { askABuild = was }) - return &asked -} - -// The live case of 2026-10-07 (novox/hq issue 300), replayed: a merge of the catalogue adds one module's -// directory and touches nothing else the mesh holds. Its delivery plan said "builds new: …, sent nowhere"; -// the merge said it changed nothing any module the mesh holds is built from, and built nothing, so the -// module was never registered and `assign` refused it. The merge asks for its build, at the branch merged -// into, from the repository as the mesh spells it — and opens no plan, since nothing it holds moved. -func TestAMergeBuildsTheNewModuleItAdds(t *testing.T) { +// ADR 0501 §6 (issue 1851, A5): the live case of 2026-10-07 (novox/hq issue 300), on keys. A merge of the catalogue adds +// one module's directory and touches nothing else the mesh holds: nothing asks its first build, it opens no plan, and +// the module is registered from the builder's first done key of it, assigned nowhere, with no build request written. +func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory - asked := asksWithPaths(t) + noBuildAsked(t) + b := aBuilderOnTheBus(t) if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { t.Fatal(err) } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00", + watching, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + watchNewModules(watching, open) + + const commit = "3da80a4b00000000" + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"}, ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true} if err := (following{open: open}).SourceMoved(ctx, m); err != nil { t.Fatal(err) } - want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"} - if len(*asked) != 1 || (*asked)[0] != want { - t.Fatalf("the merge adding modules/systemd-resolved asked for %v, not %v", *asked, want) - } if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) } + // The indexer writes the module's trunk value; the builder builds it, then its key is done. + fp := fpOf('7') + manifest := b.indexed(t, "systemd-resolved", "1", "novox/mesh-catalog", "modules/systemd-resolved", commit, fp) + b.built(t, "systemd-resolved", "1", fp, manifest, sdkdelivery.BuildBuilding) + time.Sleep(300 * time.Millisecond) + if _, known := inventoryHolds(t, inv, "systemd-resolved"); known { + t.Fatal("0501 §1: registered from a key still building") + } + if obs := newModuleObservations(time.Now().Add(time.Hour)); len(obs) != 1 || obs[0].Kind != kindNewModuleWaits { + t.Fatalf("0501 §6: the new module waiting on its key past its bound is named by %+v", obs) + } + b.built(t, "systemd-resolved", "1", fp, manifest, sdkdelivery.BuildDone) + becomes(t, 10*time.Second, "the new module registered", func() bool { + _, known := inventoryHolds(t, inv, "systemd-resolved") + return known + }) + if obs := newModuleObservations(time.Now().Add(time.Hour)); len(obs) != 0 { + t.Fatalf("a registered module is still named waiting: %+v", obs) + } + requests, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: "novox/mesh-catalog", + Path: "modules/systemd-resolved"}.Name()) + if err != nil || len(requests) != 0 { + t.Fatalf("0501 §6: a build request was written: %+v (%v)", requests, err) + } } -// A merge that changes a held module and adds a new one does both: the held one walks its plan, the new -// one is asked for beside it. A merge adding nothing builds nothing new. -func TestAMergeBuildsItsNewModuleBesideItsPlan(t *testing.T) { +// ADR 0501 §7: a new module whose manifest uses a word this controller does not read is named waiting at once, never +// failed, and registered from the same key, with no second build, when a controller that reads it looks. +func TestANewModuleWithAnUnreadWordWaitsForTheControllerThatReadsIt(t *testing.T) { open := aMesh(t) ctx := t.Context() inv := open.inventory - asked := asksWithPaths(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + noBuildAsked(t) + b := aBuilderOnTheBus(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { t.Fatal(err) } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts", "modules/fresh/module.json"}, - ModuleDirs: []string{"modules/app", "modules/fresh"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + fp := fpOf('8') + raw := `{"module":"watchman","version":"1","not-a-word-yet":"5m","build":{"artifacts":[{"name":"a","kind":"archive","from":"files"}]},` + + `"resources":[{"id":"a","type":"archive","path":"/opt/watchman","artifact":"a"}]}` + digest := sdkdelivery.ManifestDigest([]byte(raw)) + value, _ := sdkdelivery.EncodeManifest(sdkdelivery.Manifest{Module: "watchman", JSON: raw}) + if _, err := b.manifests.Put(ctx, digest, value); err != nil { t.Fatal(err) } - paths := map[string]bool{} - for _, a := range *asked { - paths[a[1]] = true - } - if !paths["modules/fresh"] || !paths["modules/app"] || len(*asked) != 2 { - t.Fatalf("asked %v; want the new module and the plan's first tier", *asked) - } - plans, err := inv.OpenPlans(ctx) - if err != nil || len(plans) != 1 || plans[0].Modules["fresh"] != nil || plans[0].Modules["app"] == nil { - t.Fatalf("the plan should walk app alone: %+v %v", plans, err) - } - - *asked = nil - m2 := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c2aaaaaaaa", - Paths: []string{"README.md"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m2); err != nil { + trunk, _ := sdkdelivery.EncodeModule(sdkdelivery.Module{FP: "src1:" + fp[5:], FP2: fp, Manifest: digest, + Repository: "novox/mesh-catalog", Commit: "4444444400000000", Path: "modules/watchman"}) + if _, err := b.modules.Put(ctx, "watchman", trunk); err != nil { t.Fatal(err) } - if len(*asked) != 0 { - t.Fatalf("a merge adding no module asked for %v", *asked) + b.built(t, "watchman", "1", fp, digest, sdkdelivery.BuildDone) + takeNewModules(ctx, open, "watchman") + if _, known := inventoryHolds(t, inv, "watchman"); known { + t.Fatal("0501 §7: a manifest with an unread word was registered") } -} - -// 1617/AC1 (integration): a merge adding a module to a repository the mesh builds modules from, heard while a batch of -// that repository is still assembling (its window running), asks the module's first build when it is heard, so the -// module is registered while the batch waits; cutting the batch asks it no second time (1617/AC2). -func TestAMergeAddingAModuleAsksItsBuildBeforeItsBatchIsCut(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/c1") - asked := asksWithPaths(t) - catalog := func(commit string, made time.Time, paths ...string) link.SourceMoved { - return link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, - Head: "pr/" + commit, MergedAt: made.Format(time.RFC3339Nano), Paths: paths} + obs := newModuleObservations(time.Now()) + if len(obs) != 1 || obs[0].Kind != kindNewModuleWaits || !strings.Contains(obs[0].Summary, `"not-a-word-yet"`) { + t.Fatalf("0501 §7: the module waiting for a reader of its word is named by %+v", obs) } - hear(t, open, catalog("c1", t0, "modules/app/index.ts"), t0) - if len(*asked) != 0 { - t.Fatalf("a merge adding no module asked %v", *asked) - } - hear(t, open, lone(catalog("c2", t0.Add(10*time.Second), "modules/go/packages.json", "modules/go/module.json")), t0.Add(10*time.Second)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("the batch should still be assembling: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - want := [3]string{"novox/mesh-catalog", "modules/go", "main"} - if len(*asked) != 1 || (*asked)[0] != want { - t.Fatalf("heard with its batch assembling, the merge asked %v, not %v", *asked, want) - } - cutAt(t, open, t0.Add(10*time.Second+90*time.Second)) - n := 0 - for _, a := range *asked { - if a[1] == "modules/go" { - n++ + builds, _ := inv.Builds(ctx, "watchman", 5) + for _, b := range builds { + if !b.Worked() { + t.Fatalf("0501 §7: a failed build was recorded: %+v", b) } } - if n != 1 { - t.Fatalf("modules/go was asked %d times: %v", n, *asked) - } } diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go new file mode 100644 index 00000000..75871383 --- /dev/null +++ b/cmd/mesh-controller/new_modules.go @@ -0,0 +1,251 @@ +package main + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strings" + "sync" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A new module is registered from its first done key (novox/hq ADR 0501 §1, §6; issue 1851, A5). +// +// Nothing asks a new module's first build: the builder builds every fingerprint the indexer writes, a new module's +// included (ADR 0429 §5). The controller registers a module the catalogue does not hold from the builder's done key of the +// fingerprint the indexer's trunk value names, under the same rules of who may register as before (ADR 0501 §3): from a +// repository the catalogue builds another module from, or on the operator's Approve of the merge the trunk value names +// (a first merge, kept with mesh-delivery's word), or at the terminal (ADR 0501 §4). It is assigned nowhere. +// +// **No timer** (ADR 0426): a write of the builder or the indexer wakes it for that module (builderkeys.Watch), a first +// merge's word wakes it for that repository, and a controller that starts looks at every module once (ADR 0501 §7: a +// manifest the controller before could not read is registered when the controller that reads it runs). + +// newModulesMu keeps one take-in of new modules at a time in this process. +var newModulesMu sync.Mutex + +// newModuleWait is a new module the controller could not register yet, and why: what the first-builds condition names. +type newModuleWait struct { + Module, Repository, Commit, Why string + // Word is the manifest word this controller does not read, when that is why (ADR 0501 §7). + Word string + Since time.Time +} + +// newModulesWaiting are the new modules waiting, by module, in this process: a restart looks at every module again. +var newModulesWaiting = struct { + sync.Mutex + by map[string]newModuleWait +}{by: map[string]newModuleWait{}} + +// takeNewModules looks at the modules the indexer holds that the catalogue does not, all of them or those named, and +// registers each whose done key may register it. Each outcome is said once: a module still waiting for the same reason +// is not said again. +func takeNewModules(ctx context.Context, open *stores, only ...string) { + newModulesMu.Lock() + defer newModulesMu.Unlock() + keys, closeKeys, err := openBuilderKeys() + if err != nil { + fmt.Printf("new modules: the builder's keys cannot be read: %v\n", err) + return + } + defer closeKeys() + takeNewModulesWith(ctx, open.inventory, keys, only...) +} + +// takeNewModulesWith is takeNewModules on the buckets handed in. +func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newModuleReads, only ...string) { + entries, err := inv.Catalogued(ctx) + if err != nil { + fmt.Printf("new modules: the catalogue cannot be read: %v\n", err) + return + } + held := map[string]bool{} + for _, e := range entries { + held[e.Manifest.Module] = true + } + values, unread, err := keys.Modules(ctx) + if err != nil { + fmt.Printf("new modules: the indexer's modules cannot be read: %v\n", err) + return + } + for module, err := range unread { + if !held[module] && (len(only) == 0 || slices.Contains(only, module)) { + sayNewModuleWait(newModuleWait{Module: module, Why: "the indexer's value of it cannot be read: " + err.Error()}) + } + } + names := make([]string, 0, len(values)) + for module := range values { + if !held[module] && (len(only) == 0 || slices.Contains(only, module)) { + names = append(names, module) + } + } + sort.Strings(names) + for _, module := range names { + v := values[module] + _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: v, Ask: registrationAsk{Merge: v.Commit}}) + switch { + case err == nil: + forgetNewModuleWait(module) + fmt.Printf("%s is new in %s at %.8s: registered from the builder's done key, assigned nowhere\n", module, + v.Repository, v.Commit) + case errors.Is(err, errKeyWaits): + w := newModuleWait{Module: module, Repository: v.Repository, Commit: v.Commit, Why: err.Error()} + if word := wordIn(err.Error()); word != "" { + w.Word = word + } + sayNewModuleWait(w) + default: + sayNewModuleWait(newModuleWait{Module: module, Repository: v.Repository, Commit: v.Commit, + Why: "not registered: " + err.Error()}) + } + } +} + +// newModuleReads is what the take-in of new modules reads of the builder and the indexer. +type newModuleReads interface { + keyReads + Modules(ctx context.Context) (map[string]sdkdelivery.Module, map[string]error, error) +} + +var _ newModuleReads = builderkeys.Reader{} + +// wordIn is the manifest word a take-in's words say this controller does not read. +func wordIn(said string) string { + _, rest, ok := strings.Cut(said, "uses the word \"") + if !ok { + return "" + } + word, _, _ := strings.Cut(rest, "\"") + return word +} + +// sayNewModuleWait keeps a new module waiting and says it, once for each reason. +func sayNewModuleWait(w newModuleWait) { + newModulesWaiting.Lock() + defer newModulesWaiting.Unlock() + was, known := newModulesWaiting.by[w.Module] + if known && was.Why == w.Why && was.Commit == w.Commit { + return + } + w.Since = time.Now().UTC() + if known && was.Commit == w.Commit { + w.Since = was.Since + } + newModulesWaiting.by[w.Module] = w + fmt.Printf("%s is new at %.8s and not registered yet: %s\n", w.Module, w.Commit, firstLine(w.Why)) +} + +func forgetNewModuleWait(module string) { + newModulesWaiting.Lock() + defer newModulesWaiting.Unlock() + delete(newModulesWaiting.by, module) +} + +// kindNewModuleWaits is what a new module waiting for this controller's reader of its manifest, or waiting past its +// bound, raises (ADR 0501 §6, §7: the first-builds condition reads the key, not a build request). +const kindNewModuleWaits = "new-module-waits" + +// newModuleWaitBound is how long a new module waits for its key before it is said: the builder builds a fingerprint in +// minutes. Not a delay: its key's write registers it the moment it is done. +const newModuleWaitBound = 30 * time.Minute + +// newModuleObservations names the new modules waiting: one whose manifest uses a word this controller does not read at +// once (nothing is lost, but the operator sees what waits for which controller), any other past the bound. +func newModuleObservations(now time.Time) []conditions.Observation { + newModulesWaiting.Lock() + defer newModulesWaiting.Unlock() + var out []conditions.Observation + for _, w := range newModulesWaiting.by { + if w.Word == "" && now.Sub(w.Since) <= newModuleWaitBound { + continue + } + head := fmt.Sprintf("%s is not registered: %s", w.Module, firstLine(w.Why)) + resolved := "its key is done and its take-in registers it" + if w.Word != "" { + head = fmt.Sprintf("%s waits for a controller that reads the manifest word %q", w.Module, w.Word) + resolved = "a controller that reads the word runs and registers it from the same key, with no second build" + } + out = append(out, conditions.Observation{Scope: conditions.ScopeMerge, + ID: fmt.Sprintf("%s_%s", w.Module, short(w.Commit)), Kind: kindNewModuleWaits, Severity: conditions.Warning, + Summary: head, + Said: fmt.Sprintf("%s at %s waits since %s: %s", w.Module, w.Commit, w.Since.Format(time.RFC3339), w.Why), + Headline: fmt.Sprintf("The new module %s is not registered yet", w.Module), + Explanation: fmt.Sprintf("The merge %.8s of %s adds %s. The mesh registers it from the builder's done key of its "+ + "fingerprint, and has not yet: %s", w.Commit, w.Repository, w.Module, w.Why), + Resolved: resolved, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out +} + +// watchNewModules wakes the take-in of new modules for each module the builder or the indexer writes, until ctx ends, and +// looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work. +func watchNewModules(ctx context.Context, open *stores) { + go func() { + answerHeldMerges(ctx, open.inventory) + takeNewModules(ctx, open) + }() + keys, closeKeys, err := openBuilderKeys() + if err != nil { + fmt.Printf("new modules: the builder's keys are not watched: %v\n", err) + return + } + if err := builderkeys.Watch(ctx, keys.JS, func(module string) { wakeNewModule(ctx, open, module) }); err != nil { + closeKeys() + fmt.Printf("new modules: the builder's keys are not watched: %v\n", err) + return + } + go func() { + <-ctx.Done() + closeKeys() + }() +} + +// wakeNewModule takes module in when the catalogue does not hold it: never on the watch's own goroutine, which must keep +// draining. +func wakeNewModule(ctx context.Context, open *stores, module string) { + go func() { + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + return + } + for _, e := range entries { + if e.Manifest.Module == module { + // Known: a plan waiting on its key is woken by the plans' own look. + askToLook() + return + } + } + takeNewModules(ctx, open, module) + }() +} + +// newModulesOf are the modules the indexer holds in a repository: what a first merge's word wakes. +func newModulesOf(ctx context.Context, repository string) ([]string, error) { + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return nil, err + } + defer closeKeys() + values, _, err := keys.Modules(ctx) + if err != nil { + return nil, err + } + var out []string + for module, v := range values { + if strings.EqualFold(v.Repository, repository) { + out = append(out, module) + } + } + sort.Strings(out) + return out, nil +} diff --git a/cmd/mesh-controller/new_modules_test.go b/cmd/mesh-controller/new_modules_test.go new file mode 100644 index 00000000..cfa845ac --- /dev/null +++ b/cmd/mesh-controller/new_modules_test.go @@ -0,0 +1,33 @@ +package main + +import ( + "strings" + "testing" + "time" +) + +// ADR 0501 §6, §7 (issue 1851, A5b): the condition of a new module reads its key's take-in: one waiting for a controller +// that reads its manifest's word is named at once, one waiting for its key only past the bound, each reason said once. +func TestANewModuleWaitingIsNamedByWhatItWaitsFor(t *testing.T) { + t.Cleanup(func() { + newModulesWaiting.Lock() + newModulesWaiting.by = map[string]newModuleWait{} + newModulesWaiting.Unlock() + }) + word := `waits: watchman's manifest uses the word "key-ttl", which this controller does not read: it is registered when a controller that reads it runs` + sayNewModuleWait(newModuleWait{Module: "watchman", Commit: "c1", Why: word, Word: wordIn(word)}) + sayNewModuleWait(newModuleWait{Module: "view", Commit: "c1", Why: "waits: the build of view at aaaa is building on ace"}) + now := time.Now() + obs := newModuleObservations(now) + if len(obs) != 1 || !strings.Contains(obs[0].Summary, `"key-ttl"`) || obs[0].Kind != kindNewModuleWaits { + t.Fatalf("at once, only the unread word is named: %+v", obs) + } + if obs := newModuleObservations(now.Add(newModuleWaitBound + time.Minute)); len(obs) != 2 { + t.Fatalf("past the bound, both are named: %+v", obs) + } + forgetNewModuleWait("watchman") + forgetNewModuleWait("view") + if obs := newModuleObservations(now.Add(time.Hour)); len(obs) != 0 { + t.Fatalf("registered modules are still named: %+v", obs) + } +} diff --git a/cmd/mesh-controller/plan_looks.go b/cmd/mesh-controller/plan_looks.go index c29e4065..855d6e02 100644 --- a/cmd/mesh-controller/plan_looks.go +++ b/cmd/mesh-controller/plan_looks.go @@ -209,10 +209,6 @@ func servingPlanSaved(ctx context.Context, bus link.Bus, open *stores) func(*inv keepBuildingKey(ctx, p) if p.WalkSaid == inventory.WalkEnded { go forgetChecksOfWalk(context.WithoutCancel(ctx), open.inventory, p) - // A first build refused for a word only a newer build-agent reads is asked again (novox/hq issue 1173). - if deliversBuildAgent(p) { - go askHeldFirstBuildsAgain(context.WithoutCancel(ctx), open, p) - } } } } diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 9f38cd91..e3997d88 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -286,6 +286,9 @@ func serve(ctx context.Context) (err error) { go sweepingDeclarationFiles(ctx, inv, bus.Conn) // And the key of each plan in its build phase, which holds the build seat's checks while a delivery builds (ADR 0483 clause 2). go keepingPlansBuilding(ctx, inv, bus.Conn) + // And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by + // the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build. + watchNewModules(ctx, open) // Every call carries the lease's epoch, and its record is written only under the lease (novox/hq // to-be 45 §6). link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) }) diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index da962d91..07a3c964 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -371,12 +371,17 @@ func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMo } // **A new module is built and registered, and sent nowhere** (novox/hq issue 300): the delivery plan // says so, and assigning it is a person's act, which needs it registered first. - built := askNewModules(ctx, inv, m, from, added) + // Nothing asks its first build (novox/hq ADR 0501 §6): the builder builds every fingerprint the indexer writes, and + // the controller registers the module from its first done key (new_modules.go). + var built []string + if len(from) > 0 { + built = added + } moved := append(append([]inventory.Entry{}, touched...), packaging...) if len(moved) == 0 { if len(built) > 0 { fmt.Printf("%s/%s merged into %s (%.8s); it changed no module the mesh holds, and adds %s: "+ - "built, registered when the build lands, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit, + "registered from the builder's done key of each, and sent nowhere\n", m.Owner, m.Repo, m.Base, m.Commit, strings.Join(built, ", ")) return nil, nil } @@ -399,107 +404,6 @@ func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMo return names, nil } -// askNewModules asks the build seat for every module a merge adds to the repository — a directory holding a -// module.json that no module of the mesh is known from (touchedBy's `added`) — at the branch merged into, -// not waited for: the build's take-in registers it, as a hand `build` does, and sends it nowhere, since no -// machine is assigned it (novox/hq issue 300). Before this a merge said "it changed nothing any module -// the mesh holds is built from", the module was never registered, and `assign` refused it as unknown. -// -// The repository is spelled, and found on the seat, as the modules already built from it are; `from` is -// those of them this merge moves, so a merge read as history, or of a branch nothing follows, builds no -// new module either. Outside the plan: the plan walks modules the catalogue holds, and a new one has no -// machine to send to and nothing standing on it. What could not be asked is said, and left to `build`. -// Returns the directories asked for. -func askNewModules(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, from []inventory.Entry, - added []string) []string { - if len(added) == 0 || len(from) == 0 { - return nil - } - source := buildSource{Repository: from[0].Source.Repository, Seat: from[0].Source.Seat} - // Asked once (novox/hq issue 1617): a module is asked when its merge is heard, and again here when the batch is cut. - var requests []inventory.RequestOutcome - for _, dir := range added { - if got, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: source.Repository, Path: dir}.Name()); err == nil { - requests = append(requests, got...) - } - } - return askBuilds(ctx, inv, m, source, unaskedDirs(added, requests, source, time.Now())) -} - -// askedInFlightFor is how long a merge's build request stands as the build coming for its module: past it, a lost one -// is asked again. -const askedInFlightFor = time.Hour - -// unaskedDirs are the directories of a new module no build of a merge is on its way for: a kept request of the same -// repository and directory, asked for a merge and handed over, whose outcome has not come and is not given up on, -// is the build coming (novox/hq issue 1617). -func unaskedDirs(added []string, requests []inventory.RequestOutcome, source buildSource, now time.Time) []string { - var out []string - for _, dir := range added { - path := dir - if path == "." { - path = "" - } - coming := false - for _, r := range requests { - if r.For == "merge" && r.Repository == source.Repository && strings.Trim(r.Path, "/") == path && - r.NotAsked == "" && r.OutcomeUnknown == "" && !r.Heard && now.Sub(r.At) < askedInFlightFor { - coming = true - break - } - } - if !coming { - out = append(out, dir) - } - } - return out -} - -// askNewModulesAtHearing asks the first build of the modules a merge adds to a repository the mesh builds modules from, -// when the merge is heard, not when its batch is cut (novox/hq issue 1617): a batch waits out its window and the walk -// before it, and a module added by a merge delivered meanwhile could not be assigned. Outside the batch and its plan. -func askNewModulesAtHearing(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, entries []inventory.Entry, - read map[string][]inventory.ReadRepository, from []inventory.Entry) { - if len(from) == 0 { - return - } - _, added, _ := touchedBy(from, entries, m, read) - if asked := askNewModules(ctx, inv, m, from, added); len(asked) > 0 { - fmt.Printf("%s/%s merged into %s (%.8s) adds %s: the first build is asked now, before its batch is cut\n", - m.Owner, m.Repo, m.Base, m.Commit, strings.Join(asked, ", ")) - } -} - -// askBuilds asks the first build of each directory of a merge's repository, as askNewModules does, from the source given. -func askBuilds(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, source buildSource, - added []string) []string { - var asked []string - for _, dir := range added { - path := dir - if path == "." { - path = "" - } - id, err := askABuild(ctx, source, path, m.Base) - // Kept, asked or not, so `assign` can tell a build in flight, or a merge that could not ask for one, - // from a module nobody ever heard of (novox/hq issue 325). - request := inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path, - Ref: m.Base, Commit: m.Commit, For: "merge"} - if err != nil { - request.ID = notAskedID(m.Commit, dir) - request.NotAsked = err.Error() - recordBuildRequest(ctx, inv, request) - fmt.Printf(" %s is a new module in %s/%s and could not be built: %v — a hand `build` of it "+ - "asks again\n", dir, m.Owner, m.Repo, err) - continue - } - recordBuildRequest(ctx, inv, request) - fmt.Printf(" %s is a new module in %s/%s: build %s asked at %s; registered when it lands, assigned "+ - "nowhere\n", dir, m.Owner, m.Repo, id, m.Base) - asked = append(asked, dir) - } - return asked -} - // notAskedID is the id a merge's build that could not be asked is kept under: a key the bus takes, so a module at the // repository's root (".") is kept as one in a folder is. func notAskedID(commit, dir string) string { diff --git a/internal/builderkeys/builderkeys.go b/internal/builderkeys/builderkeys.go index b931defd..036ba35b 100644 --- a/internal/builderkeys/builderkeys.go +++ b/internal/builderkeys/builderkeys.go @@ -158,6 +158,39 @@ func (r Reader) ModuleAt(ctx context.Context, module, commit string) (delivery.M return pickAt(module, commit, values) } +// Modules is every module the indexer holds a trunk value of, by module: the value it holds now. A value that does not +// decode is left out and said in unread. +func (r Reader) Modules(ctx context.Context) (values map[string]delivery.Module, unread map[string]error, err error) { + kv, err := r.bucket(ctx, delivery.ModulesBucket) + if err != nil || kv == nil { + return nil, nil, err + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + return nil, nil, nil + } + if err != nil { + return nil, nil, fmt.Errorf("listing the indexer's modules: %w", err) + } + values, unread = map[string]delivery.Module{}, map[string]error{} + for _, key := range keys { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + continue + } + if err != nil { + return nil, nil, fmt.Errorf("the indexer's value of %s: %w", key, err) + } + m, err := delivery.DecodeModule(e.Value()) + if err != nil { + unread[key] = err + continue + } + values[key] = m + } + return values, unread, nil +} + // pickAt is, of a key's values oldest first, the newest whose commit is commit (any, with commit empty); values that // do not decode are passed over, and one of them is said when nothing else is found. func pickAt(module, commit string, values [][]byte) (delivery.Module, error) { diff --git a/internal/builderkeys/builderkeys_integration_test.go b/internal/builderkeys/builderkeys_integration_test.go index 80352ad1..57cfdadf 100644 --- a/internal/builderkeys/builderkeys_integration_test.go +++ b/internal/builderkeys/builderkeys_integration_test.go @@ -124,6 +124,11 @@ func TestTheControllerReadsTheBuildersKeysAndTheIndexersValues(t *testing.T) { if _, err := r.ModuleAt(ctx, "x", "9999999999"); !errors.Is(err, ErrNoValue) { t.Fatalf("a commit the indexer never wrote: %v", err) } + put(t, modules, "broken", []byte("{not json")) + all, unread, err := r.Modules(ctx) + if err != nil || len(all) != 1 || all["x"].Commit != "3333333333" || unread["broken"] == nil { + t.Fatalf("every module's value now: %+v %v %v", all, unread, err) + } } // The controller's account reaches the builder's and the indexer's buckets: it is granted the JetStream API whole. From 62268cf43db528d0f47cbcb98032babb5aa5739f Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:52:15 +0200 Subject: [PATCH 06/38] Take a walk's modules from the builder's done keys and ask no build but the builder's own: a key building or unreferenced holds the tier and says so, a failed key fails the walk in its words (issue 1851, A6, A7) --- .../ask_bus_look_integration_test.go | 2 + cmd/mesh-controller/ask_bus_moment_test.go | 12 + cmd/mesh-controller/away_integration_test.go | 1 + .../builder_on_the_bus_test.go | 1 + .../pending_integration_test.go | 2 + cmd/mesh-controller/plan_keys.go | 268 ++++++++++++++++++ cmd/mesh-controller/plan_keys_test.go | 190 +++++++++++++ cmd/mesh-controller/plan_retry.go | 8 +- cmd/mesh-controller/plans_building_test.go | 1 + cmd/mesh-controller/queue_integration_test.go | 1 + cmd/mesh-controller/release_plan.go | 43 ++- .../replays_integration_test.go | 1 + cmd/mesh-controller/rollout_events_test.go | 1 + .../silent_mid_tier_integration_test.go | 1 + .../tier_send_integration_test.go | 1 + .../walk_from_keys_integration_test.go | 104 +++++++ internal/inventory/plans.go | 9 + 17 files changed, 643 insertions(+), 3 deletions(-) create mode 100644 cmd/mesh-controller/plan_keys.go create mode 100644 cmd/mesh-controller/plan_keys_test.go create mode 100644 cmd/mesh-controller/walk_from_keys_integration_test.go diff --git a/cmd/mesh-controller/ask_bus_look_integration_test.go b/cmd/mesh-controller/ask_bus_look_integration_test.go index 9f78de1e..83bbee9f 100644 --- a/cmd/mesh-controller/ask_bus_look_integration_test.go +++ b/cmd/mesh-controller/ask_bus_look_integration_test.go @@ -31,6 +31,7 @@ func TestALookComesBackForATierTheBusRefusedWhole(t *testing.T) { t.Fatal(err) } bus := false + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { if !bus { @@ -95,6 +96,7 @@ func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) { if err := inv.SavePlan(ctx, &p); err != nil { t.Fatal(err) } + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "", fmt.Errorf("%w: nats: no responders available for request", inventory.ErrNoChain) diff --git a/cmd/mesh-controller/ask_bus_moment_test.go b/cmd/mesh-controller/ask_bus_moment_test.go index 42ced920..70163618 100644 --- a/cmd/mesh-controller/ask_bus_moment_test.go +++ b/cmd/mesh-controller/ask_bus_moment_test.go @@ -15,6 +15,15 @@ import ( // seconds) is not a failed build. The module is left unasked and the plan building, and the next look asks it again; only a // module's own fault, or a bus silent for ten minutes, fails the plan. +// asksOfEveryModule makes every module a test names the builder's own, the one module a walk still asks build-agent +// for (ADR 0453 §5): what a test of that ask's waits reads. +func asksOfEveryModule(t *testing.T) { + t.Helper() + was := buildersOwn + buildersOwn = func(string) bool { return true } + t.Cleanup(func() { buildersOwn = was }) +} + func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) { p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b", "c"}}, Modules: map[string]*inventory.PlanModule{}} @@ -26,6 +35,7 @@ func aTierOfThree() (*inventory.Plan, map[string]inventory.Entry) { } func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) { + asksOfEveryModule(t) p, byName := aTierOfThree() bus := false was := askABuild @@ -68,6 +78,7 @@ func TestABusThatDoesNotAnswerForAMomentDoesNotFailTheTier(t *testing.T) { } func TestABusSilentForTenMinutesFailsTheTierAsBefore(t *testing.T) { + asksOfEveryModule(t) p, byName := aTierOfThree() was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { @@ -119,6 +130,7 @@ func TestAPlanWithAModuleWaitingForTheBusIsOneTheLookComesBackTo(t *testing.T) { } func TestAModulesOwnFaultStillFailsThePlan(t *testing.T) { + asksOfEveryModule(t) p, byName := aTierOfThree() delete(byName, "a") askModule(t.Context(), p, "a", byName) diff --git a/cmd/mesh-controller/away_integration_test.go b/cmd/mesh-controller/away_integration_test.go index 72f18610..1bca1e7c 100644 --- a/cmd/mesh-controller/away_integration_test.go +++ b/cmd/mesh-controller/away_integration_test.go @@ -68,6 +68,7 @@ func TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent(t *testing.T) { var mu sync.Mutex var asks []string var sends [][]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it wasAsk := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { mu.Lock() diff --git a/cmd/mesh-controller/builder_on_the_bus_test.go b/cmd/mesh-controller/builder_on_the_bus_test.go index c20a6d6a..fe277dbd 100644 --- a/cmd/mesh-controller/builder_on_the_bus_test.go +++ b/cmd/mesh-controller/builder_on_the_bus_test.go @@ -117,6 +117,7 @@ func noBuildAsked(t *testing.T) { // the test's own may ask at once, so the record is written under a lock (issue 1543). func asksWithPaths(t *testing.T) *[][3]string { t.Helper() + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it var asked [][3]string var mu sync.Mutex was := askABuild diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index c6c5b90d..7000200a 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -305,6 +305,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "", errors.New("cannot submit a build: nats: timeout") @@ -569,6 +570,7 @@ func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() asked := 0 + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { asked++ diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go new file mode 100644 index 00000000..e3c48291 --- /dev/null +++ b/cmd/mesh-controller/plan_keys.go @@ -0,0 +1,268 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A walk takes its modules from the builder's done keys (novox/hq ADR 0503 §6, ADR 0501 §1, ADR 0498 §8; issue 1851, A6 +// and A7). +// +// From R4e a walk asks build-agent for no module but the builder's own (ADR 0453 §5). A module of its tier is marked +// taken from a key (PlanModule.Key) and settled by the look: its fingerprint is read from the indexer's trunk value +// written for the commit the walk carries of its repository; the builder's key of that fingerprint, done with every +// artifact referenced, is taken in (recorded and registered, off the plans' hold) and the module is built; a key not done +// yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words. +// **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules). + +// isBuildersOwn says a module is the builder's own: the one module build-agent still builds, asked on a merge, until R4g +// (ADR 0453 §5, ADR 0498 §7). +func isBuildersOwn(module string) bool { return buildersOwn(module) } + +// buildersOwn is isBuildersOwn's rule: a variable only so a test of the one ask left (its bus waits, its plan key) may +// name modules of its own as the builder's. +var buildersOwn = func(module string) bool { return module == sdkdelivery.BuilderModule } + +// planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's). +type planKeyReads interface { + ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error) + Build(ctx context.Context, module, fp string) (builderkeys.Key, error) +} + +// openPlanKeys opens what a walk reads of the indexer and the builder. A variable so a test needs no bus. +var openPlanKeys = func() (planKeyReads, func(), error) { + r, closeKeys, err := openBuilderKeys() + if err != nil { + return nil, nil, err + } + return r, closeKeys, nil +} + +// takeFromKey marks a module of a walk's tier as taken from the builder's key: asked of nobody (ADR 0501 §6). +func takeFromKey(p *inventory.Plan, state *inventory.PlanModule, e inventory.Entry, now time.Time) { + state.BusWaitFrom = nil + state.State = "asked" + state.AskedAt = &now + state.Key = true + state.Build, state.Fingerprint = "", "" + state.KeyAt = p.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref)) + state.Why = "waits for the builder's key" + state.Commit, state.BuiltAt, state.Made = "", nil, "" + fmt.Printf(" tier %d: %s is taken from the builder's key, asking no build\n", p.Tier, e.Manifest.Module) +} + +// keyTakes are the take-ins of keys a look started and has not seen end, by key build id, and what each said when it +// waits: kept in this process, so a look starts each once and a settle says why it waits. +var keyTakes = struct { + sync.Mutex + running map[string]bool + waits map[string]string + // taken are the keys whose take-in ended registered, or found the module already at a later build (superseded). + taken map[string]bool +}{running: map[string]bool{}, waits: map[string]string{}, taken: map[string]bool{}} + +// settleFromKeys settles every module of the tier taken from a key: built from its key's record, failed with the key's +// words or the take-in's refusal, or still waiting, saying for what. It says whether it changed the plan. It reads only +// the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look. +func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) { + return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) { + go takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id) + }) +} + +// keyRecords is what settling from keys reads of the records (*inventory.Inventory, or a test's). +type keyRecords interface { + BuildByID(ctx context.Context, id string) (inventory.Build, bool, error) + RefusedOf(ctx context.Context, ids []string) (map[string]string, error) + SourceOf(ctx context.Context, module string) (inventory.Source, error) +} + +// settleKeys is settleFromKeys on the records handed in, starting a take-in with take. +func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inventory.Plan, tier []string, + take func(module string, trunk sdkdelivery.Module, id string)) (bool, error) { + changed := false + set := func(s *inventory.PlanModule, why string) { + if s.Why != why { + s.Why = why + changed = true + } + } + for _, m := range tier { + s := p.Modules[m] + if s == nil || !s.Key || s.State != "asked" { + continue + } + trunk, err := keys.ModuleAt(ctx, m, s.KeyAt) + if errors.Is(err, builderkeys.ErrNoValue) { + set(s, fmt.Sprintf("waits for the indexer's value of %s at %s", m, orNow(s.KeyAt))) + continue + } + if err != nil { + return changed, err + } + if trunk.FP2 == "" { + why := trunk.None2 + if why == "" { + why = "the indexer wrote no fingerprint of the new form" + } + set(s, fmt.Sprintf("waits: %s at %s has no fingerprint to take a build by: %s", m, orNow(s.KeyAt), why)) + continue + } + if s.Fingerprint != trunk.FP2 { + s.Fingerprint, changed = trunk.FP2, true + } + k, err := keys.Build(ctx, m, trunk.FP2) + if err != nil { + return changed, err + } + switch { + case k.Failed(): + // **A failed key fails the walk in the builder's own words** (`digests differ:`, `not stored:`, `not pinned:`, + // `version taken:`), and a rebuild of it is a new revision the next walk takes. + s.State, s.Why = "failed", k.Build.Error + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(k.Build.Error)) + changed = true + continue + case !k.Done() || len(k.Unreferenced()) > 0: + set(s, "waits: "+k.Says()) + continue + } + id := keyBuildID(k) + refused, err := inv.RefusedOf(ctx, []string{id}) + if err != nil { + return changed, fmt.Errorf("the refused registrations cannot be read: %w", err) + } + for kept, why := range refusalsKeptFor([]string{id}) { + refused[kept] = why + } + if why, ok := refused[id]; ok { + s.State, s.Why, s.Build = "failed", why, id + p.State = inventory.PlanFailed + p.Note = fmt.Sprintf("%s failed to build in tier %d: %s", m, p.Tier, firstLine(why)) + changed = true + continue + } + b, found, err := inv.BuildByID(ctx, id) + if err != nil { + return changed, err + } + if found && b.Worked() { + keyTakes.Lock() + taken := keyTakes.taken[id] + keyTakes.Unlock() + registered := taken + if !registered { + if registered, err = registeredFrom(ctx, inv, m, b); err != nil { + return changed, err + } + } + if registered { + at := b.At + s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, b.Commit, id + s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision) + s.Why = "" + fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2)) + changed = true + continue + } + } + keyTakes.Lock() + running, waits := keyTakes.running[id], keyTakes.waits[id] + if !running { + keyTakes.running[id] = true + } + keyTakes.Unlock() + if waits != "" { + set(s, firstLine(waits)) + } else { + set(s, "taking in the builder's done key "+short(trunk.FP2)) + } + if !running { + take(m, trunk, id) + } + } + return changed, nil +} + +// registeredFrom says the module's registration stands on the build: the catalogue holds the module at the build's +// commit, or the build was registered as one with an unchanged source (no move). +func registeredFrom(ctx context.Context, inv keyRecords, module string, b inventory.Build) (bool, error) { + src, err := inv.SourceOf(ctx, module) + if errors.Is(err, inventory.ErrNoSuchModule) { + return false, nil + } + if err != nil { + return false, err + } + return sameCommitID(src.BuiltFrom, b.Commit), nil +} + +func sameCommitID(a, b string) bool { + if a == "" || b == "" { + return false + } + n := min(len(a), len(b)) + return n >= 7 && a[:n] == b[:n] +} + +func orNow(commit string) string { + if commit == "" { + return "its value now" + } + return short(commit) +} + +// takeKeyOffTheHold takes a walk's module in from its done key, off the plans' hold: a registration may sweep the +// artifact store, and the plans are held by one look at a time. Its end asks the look, which settles the module from the +// record; a refusal is kept by the key's build id, so the look fails the walk with it; a wait is said by the look. +func takeKeyOffTheHold(ctx context.Context, open *stores, module string, trunk sdkdelivery.Module, id string) { + defer func() { + keyTakes.Lock() + delete(keyTakes.running, id) + keyTakes.Unlock() + askToLook() + }() + keys, closeKeys, err := openBuilderKeys() + if err != nil { + keepKeyWait(id, "waits: the builder's keys cannot be read: "+err.Error()) + return + } + defer closeKeys() + _, _, err = takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk, + Ask: registrationAsk{Merge: trunk.Commit}}) + switch { + case err == nil, errors.Is(err, inventory.ErrSuperseded): + // Superseded is no failure: the module is already at what a later merge's key built, which a walk that carries + // this merge is answered by, as a build-agent outcome superseded was. + keepKeyWait(id, "") + keyTakes.Lock() + keyTakes.taken[id] = true + keyTakes.Unlock() + case errors.Is(err, errKeyWaits): + keepKeyWait(id, err.Error()) + default: + why := err.Error() + if rerr := open.inventory.RecordRefused(ctx, id, module, why); rerr != nil { + fmt.Printf("plans: %s's refusal could not be kept in the store, only in this controller: %v\n", module, rerr) + } + keepRefusalFor(id, why, time.Now()) + } +} + +func keepKeyWait(id, why string) { + keyTakes.Lock() + defer keyTakes.Unlock() + if why == "" { + delete(keyTakes.waits, id) + return + } + keyTakes.waits[id] = why +} diff --git a/cmd/mesh-controller/plan_keys_test.go b/cmd/mesh-controller/plan_keys_test.go new file mode 100644 index 00000000..e11da201 --- /dev/null +++ b/cmd/mesh-controller/plan_keys_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// planKeysOf is the indexer and the builder as a walk reads them, for a test: values by module and commit, keys by +// module. +type planKeysOf struct { + values map[string]sdkdelivery.Module // by module + "@" + commit + keys map[string]builderkeys.Key +} + +func (k planKeysOf) ModuleAt(_ context.Context, module, commit string) (sdkdelivery.Module, error) { + if v, ok := k.values[module+"@"+commit]; ok { + return v, nil + } + return sdkdelivery.Module{}, builderkeys.ErrNoValue +} + +func (k planKeysOf) Build(_ context.Context, module, fp string) (builderkeys.Key, error) { + if key, ok := k.keys[module]; ok && key.Fingerprint == fp { + return key, nil + } + return builderkeys.Key{Module: module, Fingerprint: fp}, nil +} + +// recordsOf is the build records and refusals a settle reads, for a test. +type recordsOf struct { + builds map[string]inventory.Build + refused map[string]string + sources map[string]inventory.Source +} + +func (r recordsOf) BuildByID(_ context.Context, id string) (inventory.Build, bool, error) { + b, ok := r.builds[id] + return b, ok, nil +} + +func (r recordsOf) RefusedOf(_ context.Context, ids []string) (map[string]string, error) { + out := map[string]string{} + for _, id := range ids { + if why, ok := r.refused[id]; ok { + out[id] = why + } + } + return out, nil +} + +func (r recordsOf) SourceOf(_ context.Context, module string) (inventory.Source, error) { + if s, ok := r.sources[module]; ok { + return s, nil + } + return inventory.Source{}, inventory.ErrNoSuchModule +} + +func aWalkOfOne(module string) (*inventory.Plan, map[string]inventory.Entry) { + p := &inventory.Plan{ID: "plan-1", State: inventory.PlanBuilding, Tiers: [][]string{{module}}, + Modules: map[string]*inventory.PlanModule{}, Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789", + Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: "c0ffee0123456789"}}}} + byName := map[string]inventory.Entry{module: {Source: inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + module, Ref: "main"}}} + return p, byName +} + +// A6 (ADR 0503 §6, ADR 0498 check §7): a walk whose module has a done key goes asked to built with no build asked: the +// ask stub is never called, the key is taken in once, and `plans` says it was taken from the builder's key. +func TestAWalkTakesADoneKeyAndAsksNoBuild(t *testing.T) { + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + t.Fatal("a walk asked build-agent for a module that is not the builder's own") + return "", nil + } + t.Cleanup(func() { askABuild = was }) + p, byName := aWalkOfOne("sudo") + askModule(t.Context(), p, "sudo", byName) + s := p.Modules["sudo"] + if s.State != "asked" || !s.Key || s.Build != "" || s.KeyAt != "c0ffee0123456789" { + t.Fatalf("the module is not taken from a key: %+v", s) + } + fp := "src2:" + strings.Repeat("a", 64) + k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 2, Build: sdkdelivery.BuilderBuild{ + State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", + Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} + keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, + keys: map[string]builderkeys.Key{"sudo": k}} + records := recordsOf{builds: map[string]inventory.Build{}, sources: map[string]inventory.Source{}} + var taken []string + take := func(m string, _ sdkdelivery.Module, id string) { taken = append(taken, m+" "+id) } + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + + changed, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take) + if err != nil || !changed || len(taken) != 1 || s.State != "asked" || s.Fingerprint != fp { + t.Fatalf("a done key not recorded yet is taken in once: %v %v %v %+v", changed, err, taken, s) + } + // A second look while the take-in runs starts no second one. + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || len(taken) != 1 { + t.Fatalf("a second look took the key in again: %v %v", taken, err) + } + // The take-in recorded and registered it: the look settles it built from the record. + id := keyBuildID(k) + records.builds[id] = inventory.Build{ID: id, Module: "sudo", Commit: "c0ffee0123456789", At: time.Now()} + records.sources["sudo"] = inventory.Source{BuiltFrom: "c0ffee0123456789"} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { + t.Fatal(err) + } + if s.State != "built" || s.Build != id || !strings.Contains(s.Made, "taken from the builder's key") { + t.Fatalf("the module is not built from its key: %+v", s) + } + if takesOutcome(*p, &inventory.PlanModule{Key: true, State: "asked", AskedAt: s.AskedAt}, time.Now(), "build-1") { + t.Fatal("a build-agent outcome answers a module taken from a key") + } +} + +// A7: a key building waits and says so, then moves on when done, with no timer and no ask; a failed key fails the walk +// in the builder's words; a value the indexer has not written yet waits, named. +func TestAWalkWaitsForAKeyAndFailsWithItsWords(t *testing.T) { + asksOfNone := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + t.Fatal("a walk asked a build") + return "", nil + } + t.Cleanup(func() { askABuild = asksOfNone }) + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + p, byName := aWalkOfOne("view") + askModule(t.Context(), p, "view", byName) + s := p.Modules["view"] + records := recordsOf{builds: map[string]inventory.Build{}} + take := func(string, sdkdelivery.Module, string) {} + keys := planKeysOf{values: map[string]sdkdelivery.Module{}, keys: map[string]builderkeys.Key{}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || + !strings.Contains(s.Why, "waits for the indexer's value of view at c0ffee01") { + t.Fatalf("no indexer value yet: %q %v", s.Why, err) + } + fp := "src2:" + strings.Repeat("c", 64) + keys.values["view@c0ffee0123456789"] = sdkdelivery.Module{FP2: fp} + keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding, By: "g14"}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil || + s.State != "asked" || !strings.Contains(s.Why, "building on g14") || p.State != inventory.PlanBuilding { + t.Fatalf("a building key: %+v %v", s, err) + } + keys.keys["view"] = builderkeys.Key{Module: "view", Fingerprint: fp, Found: true, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildFailed, Error: "not stored: tools: the store answered 500"}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], take); err != nil { + t.Fatal(err) + } + if s.State != "failed" || p.State != inventory.PlanFailed || !strings.Contains(p.Note, "not stored: tools") { + t.Fatalf("a failed key does not fail the walk in its words: %+v %q", s, p.Note) + } +} + +// A7: a take-in refused (a module from another repository, a manifest refused) fails the walk with the refusal. +func TestARefusedTakeInFailsTheWalk(t *testing.T) { + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + p, byName := aWalkOfOne("sudo") + askModule(t.Context(), p, "sudo", byName) + fp := "src2:" + strings.Repeat("d", 64) + k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ + State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", + Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} + keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp}}, + keys: map[string]builderkeys.Key{"sudo": k}} + records := recordsOf{refused: map[string]string{keyBuildID(k): "sudo is built from novox/mesh-catalog, and this build is of agent/sudo"}} + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(string, sdkdelivery.Module, string) {}); err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "agent/sudo") { + t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note) + } +} diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 9556bef5..86dc803e 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -402,7 +402,9 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { var asked []string for _, m := range failed { askModule(ctx, &p, m, byName) - if s := p.Modules[m]; s.State == "asked" { + if s := p.Modules[m]; s.State == "asked" && s.Key { + asked = append(asked, m+" from the builder's key") + } else if s.State == "asked" { asked = append(asked, m+" as "+s.Build) } } @@ -629,7 +631,9 @@ func retryTierWhole(ctx context.Context, open *stores, p *inventory.Plan, again for _, m := range again { sendAgain(p.Modules[m]) askModule(ctx, p, m, byName) - if s := p.Modules[m]; s.State == "asked" { + if s := p.Modules[m]; s.State == "asked" && s.Key { + asked = append(asked, m+" from the builder's key") + } else if s.State == "asked" { asked = append(asked, m+" as "+s.Build) } } diff --git a/cmd/mesh-controller/plans_building_test.go b/cmd/mesh-controller/plans_building_test.go index 8aab90c0..94d7d3bb 100644 --- a/cmd/mesh-controller/plans_building_test.go +++ b/cmd/mesh-controller/plans_building_test.go @@ -156,6 +156,7 @@ func TestASavedPlanPutsOrDeletesItsKey(t *testing.T) { // The plan's key is there before its first build is asked, so a check cannot take a place between the ask and the plan's save. func TestThePlansKeyIsPutBeforeItsFirstBuildIsAsked(t *testing.T) { + asksOfEveryModule(t) store := newFakeBuildingStore() var s buildingStore = store buildingKeys.Store(&s) diff --git a/cmd/mesh-controller/queue_integration_test.go b/cmd/mesh-controller/queue_integration_test.go index 91d0d484..c0cb98f3 100644 --- a/cmd/mesh-controller/queue_integration_test.go +++ b/cmd/mesh-controller/queue_integration_test.go @@ -431,6 +431,7 @@ func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { asked := asksRecorded(t) twoTiers(t, open) var refs []string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { refs = append(refs, ref) diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 4f949fca..d8ec6a75 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -390,6 +390,11 @@ func takesOutcome(p inventory.Plan, state *inventory.PlanModule, asked time.Time if p.Waiting() || state == nil || (state.Build == "" && state.AskedAt == nil) { return false } + // **A module taken from the builder's key is answered by its key alone** (ADR 0501 §1): a build-agent outcome of it + // is somebody else's build. + if state.Key { + return false + } if state.Build != "" { return state.Build == id } @@ -565,6 +570,13 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s p.Note = name + " is no longer in the catalogue" return } + // **Every module but the builder's own is taken from the builder's done key, asking no build** (novox/hq ADR 0503 + // §6, ADR 0501, ADR 0453 §5): the look settles it (settleFromKeys). + if !isBuildersOwn(name) { + sayBuilding(ctx, p.ID, now) + takeFromKey(p, state, e, now) + return + } source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} fmt.Printf(" tier %d: ", p.Tier) // The branch it follows, never a commit a build once named (novox/hq 04-ISSUES/215): the branch contains every @@ -1071,6 +1083,25 @@ func advanceOnce(ctx context.Context, open *stores, p *inventory.Plan, // the controller in its first tier: the build that produced the new one is recorded, and the // plan never hears it. The record is the fact; a build recorded after the ask is that tier's // outcome, whoever was listening. + // **Taken from the builder's keys first** (ADR 0503 §6, issue 1851): every module of the tier but the builder's own. + if keyed(p, tier) { + keys, closeKeys, err := openPlanKeys() + if err != nil { + return false, err + } + was := p.State + changed, err := settleFromKeys(ctx, open, keys, p, tier) + closeKeys() + if err != nil { + return false, err + } + if p.State == inventory.PlanFailed && was != inventory.PlanFailed { + sayUnsent(p, rollsOut) + } + if changed { + return true, nil + } + } recorded, byID, err := recordsOfAsked(ctx, inv, p, tier) if err != nil { return false, err @@ -2744,7 +2775,7 @@ func settleFromRecords(p *inventory.Plan, tier []string, recorded map[string][]i changed := false for _, m := range tier { s := p.Modules[m] - if s == nil || s.State != "asked" || s.AskedAt == nil { + if s == nil || s.State != "asked" || s.AskedAt == nil || s.Key { continue } var outcome *inventory.Build @@ -2845,3 +2876,13 @@ func stopByHand(p *inventory.Plan, how, by, why string) { } p.Delivery.Stopped, p.Delivery.StoppedWhy = by, why } + +// keyed says a module of the tier still waits on the builder's key. +func keyed(p *inventory.Plan, tier []string) bool { + for _, m := range tier { + if s := p.Modules[m]; s != nil && s.Key && s.State == "asked" { + return true + } + } + return false +} diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go index da467d3d..deb14f0b 100644 --- a/cmd/mesh-controller/replays_integration_test.go +++ b/cmd/mesh-controller/replays_integration_test.go @@ -423,6 +423,7 @@ func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { ctx := t.Context() inv := open.inventory var asked [][3]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { asked = append(asked, [3]string{source.Repository, path, ref}) diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index 2110f60f..0c072b33 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -69,6 +69,7 @@ func TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt(t *testing.T) { var mu sync.Mutex var asks []string var sends [][]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it wasAsk := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { mu.Lock() diff --git a/cmd/mesh-controller/silent_mid_tier_integration_test.go b/cmd/mesh-controller/silent_mid_tier_integration_test.go index b60784f0..dac3e72b 100644 --- a/cmd/mesh-controller/silent_mid_tier_integration_test.go +++ b/cmd/mesh-controller/silent_mid_tier_integration_test.go @@ -65,6 +65,7 @@ func TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToT var mu sync.Mutex var asks []string var sends [][]string + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it wasAsk := askABuild askABuild = func(_ context.Context, source buildSource, _, _ string) (string, error) { mu.Lock() diff --git a/cmd/mesh-controller/tier_send_integration_test.go b/cmd/mesh-controller/tier_send_integration_test.go index 9d0cdddf..b933516a 100644 --- a/cmd/mesh-controller/tier_send_integration_test.go +++ b/cmd/mesh-controller/tier_send_integration_test.go @@ -302,6 +302,7 @@ func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { } } + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it was := askABuild askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil } t.Cleanup(func() { askABuild = was }) diff --git a/cmd/mesh-controller/walk_from_keys_integration_test.go b/cmd/mesh-controller/walk_from_keys_integration_test.go new file mode 100644 index 00000000..d4609105 --- /dev/null +++ b/cmd/mesh-controller/walk_from_keys_integration_test.go @@ -0,0 +1,104 @@ +//go:build integration + +package main + +import ( + "reflect" + "strings" + "testing" + "time" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A6, A7 whole against the store and a real bus (ADR 0503 §6, ADR 0498 check §7; issue 1851): a walk whose modules the +// builder built goes from asked to built with no build asked, its keys taken in off the plans' hold, then sends its tier +// to the first machine as before; a key still building holds the tier, said, and moves on when the builder's key is done. +func TestAWalkTakesItsTierFromTheBuildersKeysAndSendsIt(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + tm := aTierMesh(t, "app1", "app2") + ctx := t.Context() + inv := tm.open.inventory + noBuildAsked(t) + b := aBuilderOnTheBus(t) + + // The walk carries c3 of the catalogue; its tier is unasked. + p := tm.plan(t) + p.Commit = "c3c3c3c3c3c3c3c3" + for _, m := range []string{"app1", "app2"} { + p.Modules[m] = &inventory.PlanModule{} + } + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + fp1, fp2 := fpOf('1'), fpOf('2') + m1 := b.indexed(t, "app1", "c3", "novox/mesh-catalog", "modules/app1", p.Commit, fp1) + m2 := b.indexed(t, "app2", "c3", "novox/mesh-catalog", "modules/app2", p.Commit, fp2) + b.built(t, "app1", "c3", fp1, m1, sdkdelivery.BuildDone) + b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildBuilding) + + advancePlans(ctx, tm.open) + becomes(t, 10*time.Second, "app1 taken from its key", func() bool { + advancePlans(ctx, tm.open) + return tm.plan(t).Modules["app1"].State == "built" + }) + p = tm.plan(t) + if s := p.Modules["app2"]; s.State != "asked" || !s.Key || !strings.Contains(s.Why, "building on ace") { + t.Fatalf("A7: app2, whose key is building, is %+v", s) + } + if len(tm.sent) != 0 { + t.Fatalf("A7: the tier was sent while app2 waits on its key: %v", tm.sent) + } + if s := p.Modules["app1"]; !strings.Contains(s.Made, "taken from the builder's key") || s.Fingerprint != fp1 { + t.Fatalf("A6: plans does not say app1 was taken from the builder's key: %+v", s) + } + + b.built(t, "app2", "c3", fp2, m2, sdkdelivery.BuildDone) + becomes(t, 10*time.Second, "the tier sent to the first machine", func() bool { + advancePlans(ctx, tm.open) + return len(tm.sent) > 0 + }) + p = tm.plan(t) + if !reflect.DeepEqual(tm.sent[0], []string{"anchor"}) { + t.Fatalf("A6: the tier taken from keys was sent %v, not to the first machine", tm.sent) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + for _, m := range []string{"app1", "app2"} { + if current[m].Commit != p.Commit || !isKeyBuildID(p.Modules[m].Build) { + t.Fatalf("A6: %s stands on %+v (%s), not the key's build at %s", m, current[m], p.Modules[m].Build, p.Commit) + } + } +} + +// A7: a failed key fails the walk with the builder's words, and nothing is sent. +func TestAFailedKeyFailsTheWalkInTheBuildersWords(t *testing.T) { + tm := aTierMesh(t, "app1") + ctx := t.Context() + inv := tm.open.inventory + noBuildAsked(t) + b := aBuilderOnTheBus(t) + p := tm.plan(t) + p.Commit = "c4c4c4c4c4c4c4c4" + p.Modules["app1"] = &inventory.PlanModule{} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + fp := fpOf('4') + manifest := b.indexed(t, "app1", "c4", "novox/mesh-catalog", "modules/app1", p.Commit, fp) + b.built(t, "app1", "c4", fp, manifest, sdkdelivery.BuildFailed) + for i := 0; i < 3; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if p.State != inventory.PlanFailed || !strings.Contains(p.Note, "digests differ: ace sha256:1, g14 sha256:2") { + t.Fatalf("A7: the walk is %s: %q", p.State, p.Note) + } + if len(tm.sent) != 0 { + t.Fatalf("A7: a failed walk sent %v", tm.sent) + } +} diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index 109b6104..90af4a0c 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -388,6 +388,15 @@ type PlanModule struct { // data, or "waits: …" with the recreate words, the data change named, the person's or the bus's hold, or why it // could not be judged. Recorded map[string]string `json:"recorded,omitempty"` + // Key says the walk takes this module from the builder's done key and asked no build (novox/hq ADR 0501, ADR 0453 + // §5; issue 1851): from R4e every module but the builder's own. Its outcome is the key's, never a build-agent outcome. + Key bool `json:"key,omitempty"` + // KeyAt is the commit of its repository whose indexer's trunk value names the fingerprint taken (ADR 0501 §2); empty + // for a module of a repository the walk carries no commit of (a dependent through a base), taken at the value now. + KeyAt string `json:"key_at,omitempty"` + // Fingerprint is the fingerprint of the new form (`src2:`) the walk takes the module at, read from that trunk value; + // empty until the indexer has written it. + Fingerprint string `json:"fingerprint,omitempty"` } // PlanGate is one module's rollout record at its gate (to-be 45 §8): the component, the first machine, From 7743bd0d4b34c43f7a1dc36d720458a83a946703 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:53:45 +0200 Subject: [PATCH 07/38] Delete the walk's reuse of a check's build: a walk takes keys and asks build-agent only for the builder's own module (issue 1851, A8) --- cmd/mesh-controller/build.go | 1 - cmd/mesh-controller/check_builds.go | 2 +- .../check_builds_integration_test.go | 17 +--- cmd/mesh-controller/check_builds_test.go | 44 ---------- cmd/mesh-controller/release_plan.go | 88 ++----------------- cmd/mesh-controller/same_program.go | 4 +- 6 files changed, 14 insertions(+), 142 deletions(-) diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index f78f79f1..326dcf12 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -511,7 +511,6 @@ func buildOneAsked(ctx context.Context, source buildSource, path, ref string, wa Seats: seatBases(ctx), DryRun: dryRun, } - request.Reuse = reuseAsked(ctx, dryRun) fmt.Printf("asked for %s", source) if source.Seat != "" { fmt.Printf(" (%s)", repository) diff --git a/cmd/mesh-controller/check_builds.go b/cmd/mesh-controller/check_builds.go index 7cf571de..b1f78938 100644 --- a/cmd/mesh-controller/check_builds.go +++ b/cmd/mesh-controller/check_builds.go @@ -53,7 +53,7 @@ func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p invento if p.WalkSaid != inventory.WalkEnded || p.Delivery == nil { return } - for _, pull := range reuseAskOf(&p).Pulls { + for _, pull := range pullsOfWalk(&p) { said, err := forgetChecks(ctx, inv, builder.PullRef{Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number}) if err != nil { fmt.Fprintf(os.Stderr, "%s: its check builds stay until the week's sweep: %v\n", p.ID, err) diff --git a/cmd/mesh-controller/check_builds_integration_test.go b/cmd/mesh-controller/check_builds_integration_test.go index 36a78eb6..1e21463c 100644 --- a/cmd/mesh-controller/check_builds_integration_test.go +++ b/cmd/mesh-controller/check_builds_integration_test.go @@ -14,23 +14,12 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) { +func TestAWalkNamesThePullRequestsOfItsMergesOnce(t *testing.T) { p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ {Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3}, {Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}} - asked := reuseAskOf(p) - if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) { - t.Fatalf("the walk names %+v", asked.Pulls) - } - walk := withReuse(t.Context(), asked) - if reuseAsked(walk, false) != asked { - t.Fatal("a walk's ask does not carry its leave") - } - if reuseAsked(walk, true) != nil { - t.Fatal("a dry run carries a leave to take a check's build") - } - if reuseAsked(t.Context(), false) != nil { - t.Fatal("an ask that is not a walk's carries a leave") + if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(pullsOfWalk(p), want) { + t.Fatalf("the walk names %+v", pullsOfWalk(p)) } } diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go index 8c4d0ccf..1e670644 100644 --- a/cmd/mesh-controller/check_builds_test.go +++ b/cmd/mesh-controller/check_builds_test.go @@ -104,44 +104,6 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) { // build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and // is named in testdata/store-tests. -// A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an -// earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does -// not name (novox/hq ADR 0350 §2, issue 703). -func TestAWalksLeaveNamesTheBuildsItsPullRequestsVerdictsMade(t *testing.T) { - was := checkedBuildsOf - forgetWalkVerdicts() - t.Cleanup(func() { checkedBuildsOf = was; forgetWalkVerdicts() }) - var asked [][]link.PullRef - pull := link.PullRef{Owner: "novox", Repo: "shop", Number: 7} - atHead := link.CheckedBuild{Pull: pull, Check: "build-2", Head: "bbbbbbbbbbbb", Built: link.CheckBuilt{Module: "app", Key: "r1:2"}} - earlier := link.CheckedBuild{Pull: pull, Check: "build-1", Head: "aaaaaaaaaaaa", Built: link.CheckBuilt{Module: "app", Key: "r1:1"}} - checkedBuildsOf = func(_ context.Context, pulls []link.PullRef) ([]link.CheckedBuild, error) { - asked = append(asked, pulls) - return []link.CheckedBuild{atHead, earlier}, nil - } - pulls := []link.PullRef{pull} - heads := map[link.PullRef]string{pull: "bbbbbbbbbbbb0123456789"} - for range 3 { - if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, heads); !reflect.DeepEqual(got.Builds, - []link.CheckedBuild{atHead}) { - t.Fatalf("the leave names %+v, not only the merged head's build", got.Builds) - } - } - if !reflect.DeepEqual(asked, [][]link.PullRef{pulls}) { - t.Fatalf("the verdicts were read %d times for one walk: %v", len(asked), asked) - } - // A merge whose head was not said takes nothing. - if got := reuseWithVerdicts(t.Context(), "plan-1", &link.ReuseAsk{Pulls: pulls}, nil); got.Builds != nil { - t.Fatalf("a merge of no said head names %+v", got.Builds) - } - checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { - return nil, errors.New("no bus") - } - if got := reuseWithVerdicts(t.Context(), "plan-2", &link.ReuseAsk{Pulls: pulls}, heads); got.Builds != nil { - t.Fatalf("verdicts that could not be read named %+v", got.Builds) - } -} - // A check is handed only the builds earlier checks of its own pull request at its own head made: another pull // request's build, or an earlier head's, is never kept (novox/hq issue 703). None when they cannot be read. func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) { @@ -171,12 +133,6 @@ func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) { } } -func forgetWalkVerdicts() { - walkVerdicts.Lock() - defer walkVerdicts.Unlock() - walkVerdicts.read = map[string]verdictsRead{} -} - // A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request // merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703). func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) { diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index d8ec6a75..57e43e81 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -415,35 +415,13 @@ func init() { } } -// reuseKey carries a walk's leave to take a check's build from askModule to the ask it makes (novox/hq ADR 0350 §2). -type reuseKey struct{} - -// withReuse is ctx carrying a walk's leave to take a check's build. -func withReuse(ctx context.Context, r *link.ReuseAsk) context.Context { - return context.WithValue(ctx, reuseKey{}, r) -} - -// reuseIn is the leave ctx carries; nil for every ask but a walk's. -func reuseIn(ctx context.Context) *link.ReuseAsk { - r, _ := ctx.Value(reuseKey{}).(*link.ReuseAsk) - return r -} - -// reuseAsked is the leave an ask carries: a walk's, and never a dry run's — a dry run registers nothing, so it -// takes nothing either (novox/hq ADR 0350 §2, §6). -func reuseAsked(ctx context.Context, dryRun bool) *link.ReuseAsk { - if dryRun { +// pullsOfWalk are the pull requests of the merges a walk answers, each once: what its end lets go of (forgetChecksOfWalk) +// and what a check of one was (same_program.go). +func pullsOfWalk(p *inventory.Plan) []link.PullRef { + if p.Delivery == nil { return nil } - return reuseIn(ctx) -} - -// reuseAskOf is a walk's leave to take a check's build: the pull requests of the merges it answers. -func reuseAskOf(p *inventory.Plan) *link.ReuseAsk { - r := &link.ReuseAsk{} - if p.Delivery == nil { - return r - } + var out []link.PullRef seen := map[link.PullRef]bool{} for _, m := range p.Delivery.Merges { pull, ok := pullOf(m) @@ -452,10 +430,10 @@ func reuseAskOf(p *inventory.Plan) *link.ReuseAsk { } if !seen[pull] { seen[pull] = true - r.Pulls = append(r.Pulls, pull) + out = append(out, pull) } } - return r + return out } // checkedBuildsOf are the builds the build seat's checks of those pull requests said they made for their merges, @@ -496,53 +474,6 @@ func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string { return heads } -// walkVerdicts are the checks' builds a walk read, by the walk's plan: read once per walk (novox/hq issue 649), not -// once per module it asks for. Kept a day; a walk lasts minutes. -var walkVerdicts = struct { - sync.Mutex - read map[string]verdictsRead -}{read: map[string]verdictsRead{}} - -type verdictsRead struct { - at time.Time - builds []link.CheckedBuild -} - -// reuseWithVerdicts is a walk's leave with the builds its pull requests' verdicts name: the only builds the builder -// may take (ADR 0350 §2), and **only those of a check of the head each merge merged** (novox/hq issue 703): a build -// a check made at an earlier head of the same pull request is never registered. Read once per walk. When they -// cannot be read the leave names none, and the walk builds, saying why. -func reuseWithVerdicts(ctx context.Context, walk string, r *link.ReuseAsk, heads map[link.PullRef]string) *link.ReuseAsk { - walkVerdicts.Lock() - read, known := walkVerdicts.read[walk] - walkVerdicts.Unlock() - if !known || walk == "" { - builds, err := checkedBuildsOf(ctx, r.Pulls) - if err != nil { - fmt.Printf(" the checks' builds of %d pull request(s) could not be read, so the walk builds: %v\n", len(r.Pulls), err) - return r - } - read = verdictsRead{at: time.Now(), builds: builds} - if walk != "" { - walkVerdicts.Lock() - for id, w := range walkVerdicts.read { - if time.Since(w.at) > 24*time.Hour { - delete(walkVerdicts.read, id) - } - } - walkVerdicts.read[walk] = read - walkVerdicts.Unlock() - } - } - r.Builds = nil - for _, b := range read.builds { - if link.SameHead(b.Head, heads[b.Pull]) { - r.Builds = append(r.Builds, b) - } - } - return r -} - // pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled // (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none. func pullOf(m inventory.PlanMerge) (link.PullRef, bool) { @@ -587,13 +518,10 @@ func askModule(ctx context.Context, p *inventory.Plan, name string, byName map[s if p.Delivery != nil && p.Delivery.Alone && carried != "" { ref = carried } - // **The walk takes the check's build of the same key** (novox/hq ADR 0350 §2): its ask says so, with the pull - // requests it answers, whose checks name the part of a differing key. // **The plan's key is there before its first build is** (ADR 0483 clause 2): a check taken between the ask and the plan's save // would hold a place for its whole run. sayBuilding(ctx, p.ID, now) - id, err := askABuild(withReuse(ctx, reuseWithVerdicts(ctx, p.ID, reuseAskOf(p), mergedHeadsOf(p))), source, - e.Source.Path, ref) + id, err := askABuild(ctx, source, e.Source.Path, ref) if err != nil { // **A bus that does not answer for a moment is not a failed build** (issue 1334): the module stays unasked and the // tier's next look asks it again. A fault of the module's own fails the plan at once, as does a bus silent too long. diff --git a/cmd/mesh-controller/same_program.go b/cmd/mesh-controller/same_program.go index 6c79c26b..fa1260d2 100644 --- a/cmd/mesh-controller/same_program.go +++ b/cmd/mesh-controller/same_program.go @@ -30,7 +30,7 @@ func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[st return nil } } - pulls := reuseAskOf(plan).Pulls + pulls := pullsOfWalk(plan) heads := mergedHeadsOf(plan) out := map[string]string{} for name := range plan.Modules { @@ -97,7 +97,7 @@ func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inv if plan.Delivery == nil || len(plan.Modules) == 0 { return nil } - pulls := reuseAskOf(plan).Pulls + pulls := pullsOfWalk(plan) if len(pulls) == 0 { return nil } From 9da79ef8c375358a6bb8e9aaf5d782e79062ffe0 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 17:58:21 +0200 Subject: [PATCH 08/38] Move no module whose fingerprint is unchanged, and let a key's build stand on the held build-agent build of the same digests at the switch (issue 1851, A9) --- cmd/mesh-controller/check_builds_test.go | 15 -- .../key_takein_integration_test.go | 30 ++++ cmd/mesh-controller/release_plan.go | 23 --- cmd/mesh-controller/same_program.go | 133 ++++++++---------- .../same_program_integration_test.go | 35 ++--- cmd/mesh-controller/same_program_test.go | 64 ++------- internal/inventory/source.go | 48 ++++++- internal/inventory/standing_switch_test.go | 49 +++++++ 8 files changed, 208 insertions(+), 189 deletions(-) create mode 100644 internal/inventory/standing_switch_test.go diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go index 1e670644..f0219db1 100644 --- a/cmd/mesh-controller/check_builds_test.go +++ b/cmd/mesh-controller/check_builds_test.go @@ -132,18 +132,3 @@ func TestACheckIsHandedOnlyItsOwnHeadsBuilds(t *testing.T) { t.Fatalf("verdicts that could not be read named %+v", got) } } - -// A walk's merged heads are the forge's word for each merge; a merge announced without its head, or a pull request -// merged at two heads in one walk, has none, and its walk takes no check's build (novox/hq issue 703). -func TestAWalksMergedHeadsAreTheForgesWord(t *testing.T) { - merge := func(number int, head string) inventory.PlanMerge { - return inventory.PlanMerge{Repository: "novox/shop", Number: number, Head: head} - } - p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ - merge(7, "aaaaaaaaaaaa1"), merge(8, ""), merge(9, "bbbbbbbbbbbb"), merge(9, "cccccccccccc")}}} - heads := mergedHeadsOf(p) - shop := func(n int) link.PullRef { return link.PullRef{Owner: "novox", Repo: "shop", Number: n} } - if heads[shop(7)] != "aaaaaaaaaaaa1" || heads[shop(8)] != "" || heads[shop(9)] != "" { - t.Fatalf("merged heads %v", heads) - } -} diff --git a/cmd/mesh-controller/key_takein_integration_test.go b/cmd/mesh-controller/key_takein_integration_test.go index 39df46e7..24ba2d98 100644 --- a/cmd/mesh-controller/key_takein_integration_test.go +++ b/cmd/mesh-controller/key_takein_integration_test.go @@ -190,3 +190,33 @@ func TestAKeysRecordKeepsTheKeepSetAndReadsItsManifest(t *testing.T) { t.Fatalf("the key's record does not read its manifest's context: %s", text) } } + +// A9 (ADR 0453 §6, ADR 0499 consequences): at the switch, a key's build whose every artifact has the digest of the +// build-agent build the mesh holds (`src1:`) is no move: it is registered with that build's artifacts, and a walk sends +// nothing for it. +func TestAKeyWithTheHeldBuildsDigestsIsNoMove(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + withTrunkList(t, true) + keys, trunk := aKeyFor("sudo", "6", fpOf('6'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + k := keys.keys["sudo"] + held := inventory.Build{ID: "build-sudo-old", Module: "sudo", Repository: "novox/mesh-catalog", Path: "modules/sudo", + Commit: "0ld0ld0ld0ld0ld0", SourceFingerprint: "src1:" + strings.Repeat("9", 64), + Made: []inventory.Artifact{{Name: "a", Kind: "archive", Reference: k.Build.Artifacts[0].Reference}}} + held.Manifest, _ = json.Marshal(catalogue.Manifest{Module: "sudo", Version: "held"}) + if err := inv.RecordBuild(ctx, held); err != nil { + t.Fatal(err) + } + _, kept, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}) + if err != nil { + t.Fatal(err) + } + same, err := inv.SameSourceCommits(ctx, "sudo", kept.ID) + if err != nil || !same["0ld0ld0ld0ld0ld0"] { + t.Fatalf("A9: the key's build does not stand on the held build of the same digests: %v %v", same, err) + } + if shelf, _ := inv.Catalogue(ctx); shelf["sudo"].Version != "held" { + t.Fatalf("A9: sudo is registered as %q, not with the held build's artifacts", shelf["sudo"].Version) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 57e43e81..8cc11981 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -451,29 +451,6 @@ var checkedBuildsOf = func(ctx context.Context, pulls []link.PullRef) ([]link.Ch return link.ReadCheckBuilds(ctx, js, buildSeatHeld(ctx), pulls, time.Now().Add(-builder.CheckBuildsKept)) } -// mergedHeadsOf are the head commits the walk's merges merged, by pull request (novox/hq issue 703): a pull request -// merged twice in one walk, or whose head the announcement did not say, has none, and takes no check's build. -func mergedHeadsOf(p *inventory.Plan) map[link.PullRef]string { - heads := map[link.PullRef]string{} - if p.Delivery == nil { - return heads - } - seen := map[link.PullRef]bool{} - for _, m := range p.Delivery.Merges { - pull, ok := pullOf(m) - if !ok { - continue - } - if seen[pull] && heads[pull] != m.Head { - heads[pull] = "" - continue - } - seen[pull] = true - heads[pull] = m.Head - } - return heads -} - // pullOf is a merge's pull request: owner and repository, the last two parts of however the repository is spelled // (owner/repo, a path on the git seat, a URL), and its number. False when the merge names none. func pullOf(m inventory.PlanMerge) (link.PullRef, bool) { diff --git a/cmd/mesh-controller/same_program.go b/cmd/mesh-controller/same_program.go index fa1260d2..00892b95 100644 --- a/cmd/mesh-controller/same_program.go +++ b/cmd/mesh-controller/same_program.go @@ -8,45 +8,22 @@ import ( "strings" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) -// **A module moves only when its program changes** (novox/hq issue 721). A merge check builds every module the -// delivery plan moves and compares each with the build the mesh holds of it; a walk is cut without a module whose -// check found its whole image the same as that build, while the mesh still holds it, and says so among the modules -// it left unchanged. A merge of one sentence in a package a program imports but never reaches moves no build of it. +// **A module moves only when its source changes** (novox/hq issue 721; from R4e by its fingerprint, ADR 0453 §6, ADR +// 0499; issue 1851, A9). A walk is cut without a module whose fingerprint at the commit the walk carries is the +// fingerprint of the build the mesh holds of it, and says so among the modules it left unchanged: a merge of a file no +// module reads moves nothing. A build the mesh holds of the old form (build-agent's `src1:`) has no fingerprint to +// compare: the module stays in the walk, and the rule of the standing build (equal digests, no move) judges it once its +// key is taken. -// sameAsRunning are the modules of a walk its checks found the same as the build the mesh holds, each with what its -// check said. **Only when every pull request the walk answers was checked at the head it merged, and every one of -// those checks built the module and found it the same as the build the mesh still holds**: a merge the walk answers -// without a pull request, a check of another head, a check that built it differently, or a held build that moved -// since, leaves the module in the walk. -func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[string]string) map[string]string { - if plan.Delivery == nil || len(plan.Delivery.Merges) == 0 { - return nil - } - for _, m := range plan.Delivery.Merges { - if _, ok := pullOf(m); !ok { - return nil - } - } - pulls := pullsOfWalk(plan) - heads := mergedHeadsOf(plan) +// sameByFingerprint are the modules whose fingerprint now (`src2:`, the indexer's trunk value at the walk's commit) is +// the fingerprint of the build the mesh holds, each with why; one whose either is unknown, or of the old form, is not. +func sameByFingerprint(now, held map[string]string) map[string]string { out := map[string]string{} - for name := range plan.Modules { - said := "" - for _, pull := range pulls { - b, ok := checkedBuildOf(builds, pull, heads[pull], name) - if !ok || !stillHeld(b.Built, name, held) { - said = "" - break - } - if said == "" { - said = b.Built.Same - } - } - if said != "" { - out[name] = said + for m, fp := range now { + if strings.HasPrefix(fp, "src2:") && held[m] == fp { + out[m] = fmt.Sprintf("unchanged: its fingerprint %s is the build the mesh holds", short(fp)) } } if len(out) == 0 { @@ -55,65 +32,67 @@ func sameAsRunning(plan *inventory.Plan, builds []link.CheckedBuild, held map[st return out } -// checkedBuildOf is the newest build a check of the pull request at head made of the module; false when none did. -func checkedBuildOf(builds []link.CheckedBuild, pull link.PullRef, head, module string) (link.CheckedBuild, bool) { - if head == "" { - return link.CheckedBuild{}, false +// heldFingerprint is the source fingerprint of the build the mesh holds of a module: the newest build of the commit +// it is registered at. +func heldFingerprint(ctx context.Context, inv *inventory.Inventory, module string) (string, error) { + src, err := inv.SourceOf(ctx, module) + if err != nil { + return "", err } - for _, b := range builds { - if b.Pull == pull && b.Built.Module == module && link.SameHead(b.Head, head) { - return b, true + fps, err := inv.BuildSourceFingerprints(ctx, module) + if err != nil { + return "", err + } + for commit, fp := range fps { + if sameCommitID(commit, src.BuiltFrom) { + return fp, nil } } - return link.CheckedBuild{}, false + return "", nil } -// stillHeld is whether a check found every artifact of the module the same as a build the mesh still holds. -func stillHeld(b link.CheckBuilt, module string, held map[string]string) bool { - if b.Same == "" || b.NotBuilt != "" || len(b.Made) == 0 || len(b.SameAs) != len(b.Made) { - return false - } - for _, m := range b.Made { - was := pinnedDigest(b.SameAs[m.Name]) - if was == "" || was != pinnedDigest(held[module+"/"+m.Name]) { - return false - } - } - return true -} - -// pinnedDigest is a reference's pinned digest, empty when it is not pinned. -func pinnedDigest(ref string) string { - if i := strings.LastIndex(ref, "@"); i >= 0 { - return ref[i+1:] - } - return "" -} - -// leaveSameAsRunning takes from a walk being cut every module its checks found the same as the build the mesh holds -// (sameAsRunning), and every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read leaves the walk as it was**, said. +// leaveSameAsRunning takes from a walk being cut every module whose fingerprint is unchanged (sameByFingerprint), and +// every dependent in the walk only for one of those (withDependentsOf): what was taken, with why. **Anything not read +// leaves the walk as it was**, said. func leaveSameAsRunning(ctx context.Context, inv *inventory.Inventory, plan *inventory.Plan, moved []string, edges []inventory.Edge) map[string]string { - if plan.Delivery == nil || len(plan.Modules) == 0 { + if len(plan.Modules) == 0 { return nil } - pulls := pullsOfWalk(plan) - if len(pulls) == 0 { - return nil - } - builds, err := checkedBuildsOf(ctx, pulls) + entries, err := inv.Catalogued(ctx) if err != nil { - fmt.Printf(" the checks' builds could not be read, so every module moves: %v\n", err) + fmt.Printf(" the catalogue could not be read, so every module moves: %v\n", err) // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before return nil } - held, err := inv.RegisteredHeld(ctx) + keys, closeKeys, err := openPlanKeys() if err != nil { - fmt.Printf(" what the mesh holds could not be read, so every module moves: %v\n", err) + fmt.Printf(" the indexer's fingerprints could not be read, so every module moves: %v\n", err) // empty-on-error: said above; nothing left out of the walk is the safe answer, every module moves as before return nil } - same := withDependentsOf(sameAsRunning(plan, builds, held), plan, moved, edges) + defer closeKeys() + byName := map[string]inventory.Entry{} + for _, e := range entries { + byName[e.Manifest.Module] = e + } + now, held := map[string]string{}, map[string]string{} + for m := range plan.Modules { + e, known := byName[m] + if !known || isBuildersOwn(m) { + continue + } + v, err := keys.ModuleAt(ctx, m, plan.CommitOn(e.Source.Repository, followedBranch(e.Source.Ref))) + if err != nil || v.FP2 == "" { + continue // not written yet, or none: the module moves, and its key is waited for in the walk + } + fp, err := heldFingerprint(ctx, inv, m) + if err != nil { + continue + } + now[m], held[m] = v.FP2, fp + } + same := withDependentsOf(sameByFingerprint(now, held), plan, moved, edges) if len(same) == 0 { return nil } diff --git a/cmd/mesh-controller/same_program_integration_test.go b/cmd/mesh-controller/same_program_integration_test.go index 9e608dbd..0a4f5131 100644 --- a/cmd/mesh-controller/same_program_integration_test.go +++ b/cmd/mesh-controller/same_program_integration_test.go @@ -3,45 +3,38 @@ package main import ( - "context" "strings" "testing" "time" "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) -func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { +// R4e A9 (ADR 0453 §6, ADR 0499; issue 1851), against the store and a real bus: a merge of a file no module reads moves +// nothing. Its module's fingerprint at the merge, as the indexer wrote it, is the fingerprint of the build the mesh holds, +// so the walk is cut done with nothing to walk, and its record names the module unchanged and why — the walk +// mesh-delivery reads as delivered, never stranded; no build is asked. +func TestAWalkWhoseEveryFingerprintIsUnchangedIsDone(t *testing.T) { open := windowed(t) endsItsLine(t) - asksRecorded(t) + noBuildAsked(t) + b := aBuilderOnTheBus(t) ctx := t.Context() inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}, - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { + fp := fpOf('5') + if err := inv.RecordBuild(ctx, inventory.Build{ID: "key-held", Repository: "novox/one", Ref: "main", + Module: "one", Commit: "c0", On: "ace", SourceFingerprint: fp, + Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}}); err != nil { t.Fatal(err) } - // What runs: the module registered with that build's image (RegisteredHeld reads the module table). if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { t.Fatal(err) } - pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7} - was := checkedBuildsOf - checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { - return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k", - Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}}, - Same: "unchanged: its program is the same as running build one aaaa", - SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil - } - t.Cleanup(func() { checkedBuildsOf = was }) code := repoMerge("one", "one1aaaaaaaa", t0) - code.Number, code.HeadSHA = 7, "headaaaaaaaa1111" + b.indexed(t, "one", "1", "novox/one", "", code.Commit, fp) hear(t, open, code, t0.Add(time.Second)) cutAt(t, open, t0.Add(2*time.Minute)) ws, _ := walks(t, open) @@ -52,7 +45,7 @@ func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) + if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "is the build the mesh holds") { + t.Fatalf("the walk does not say one's fingerprint is unchanged: %+v", w.Delivery) } } diff --git a/cmd/mesh-controller/same_program_test.go b/cmd/mesh-controller/same_program_test.go index 11afe41d..39b4a467 100644 --- a/cmd/mesh-controller/same_program_test.go +++ b/cmd/mesh-controller/same_program_test.go @@ -5,66 +5,26 @@ import ( "testing" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) -// TestAWalkLeavesOutAModuleWhoseProgramDidNotChange is novox/hq issue 721: mesh-controller #335 changed one sentence -// that mesh-builder never reaches; its check found build-agent the same as the build the mesh holds, so the walk moves -// mesh-controller alone and says build-agent is unchanged. -func TestAWalkLeavesOutAModuleWhoseProgramDidNotChange(t *testing.T) { - pull := link.PullRef{Owner: "novox", Repo: "mesh-controller", Number: 335} - head := "34050dc6aaaabbbbccccdddd" - aPlan := func() *inventory.Plan { - return &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}}, - Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}}, - Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{{Repository: "novox/mesh-controller", - Commit: "c2ec0a60", Number: 335, Head: head}}}} - } - held := map[string]string{"build-agent/agent": "store/build-agent@sha256:aaaa", - "mesh-controller/controller": "store/mesh-controller@sha256:cccc"} - builds := []link.CheckedBuild{ - {Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "build-agent", Key: "k1", - Made: []link.MadeArtifact{{Name: "agent", Kind: "image", Reference: "scratch/build-agent@sha256:bbbb"}}, - Same: "unchanged: its program is the same as running build agent aaaa", - SameAs: map[string]string{"agent": "10.0.0.1:5000/build-agent@sha256:aaaa"}}}, - {Pull: pull, Head: head[:12], Built: link.CheckBuilt{Module: "mesh-controller", Key: "k2", - Made: []link.MadeArtifact{{Name: "controller", Kind: "image", Reference: "scratch/mesh-controller@sha256:dddd"}}}}, - } - - p := aPlan() - same := sameAsRunning(p, builds, held) - if len(same) != 1 || same["build-agent"] == "" { +// TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange is novox/hq issue 721 on fingerprints (R4e A9, ADR 0453 §6): +// mesh-controller #335 changed one sentence that mesh-builder never reaches; the indexer's fingerprint of build-agent at +// the merge is the fingerprint of the build the mesh holds, so the walk moves mesh-controller alone and says build-agent +// is unchanged. A held build of the old form, or a fingerprint not written, leaves the module in the walk. +func TestAWalkLeavesOutAModuleWhoseFingerprintDidNotChange(t *testing.T) { + a, b := "src2:"+strings.Repeat("a", 64), "src2:"+strings.Repeat("b", 64) + now := map[string]string{"build-agent": a, "mesh-controller": b, "old": a, "unwritten": ""} + held := map[string]string{"build-agent": a, "mesh-controller": a, "old": "src1:" + strings.Repeat("a", 64)} + same := sameByFingerprint(now, held) + if len(same) != 1 || !strings.Contains(same["build-agent"], "is the build the mesh holds") { t.Fatalf("the walk leaves out %v, want build-agent alone", same) } + p := &inventory.Plan{Tiers: [][]string{{"mesh-controller", "build-agent"}}, + Modules: map[string]*inventory.PlanModule{"mesh-controller": {}, "build-agent": {}}} withhold(p, []string{"build-agent"}, nil) if len(p.Tiers) != 1 || len(p.Tiers[0]) != 1 || p.Tiers[0][0] != "mesh-controller" { t.Fatalf("the walk moves %v, want mesh-controller alone", p.Tiers) } - - // **The held build moved since the check**: what it was compared with no longer runs. - moved := map[string]string{"build-agent/agent": "store/build-agent@sha256:eeee"} - if same := sameAsRunning(aPlan(), builds, moved); len(same) != 0 { - t.Fatalf("a module compared with a build the mesh no longer holds is left out: %v", same) - } - // **A check of another head** says nothing of this merge. - other := aPlan() - other.Delivery.Merges[0].Head = "ffffffffffffffff" - if same := sameAsRunning(other, builds, held); len(same) != 0 { - t.Fatalf("a check of another head left out %v", same) - } - // **Another pull request the walk answers** that did not find it the same leaves it in. - two := aPlan() - two.Delivery.Merges = append(two.Delivery.Merges, inventory.PlanMerge{Repository: "novox/mesh-catalog", - Commit: "1234", Number: 9, Head: "abcdefabcdefabcdef"}) - if same := sameAsRunning(two, builds, held); len(same) != 0 { - t.Fatalf("a module another merge moves is left out: %v", same) - } - // **A merge with no pull request** leaves the walk as it is. - direct := aPlan() - direct.Delivery.Merges[0].Number = 0 - if same := sameAsRunning(direct, builds, held); len(same) != 0 { - t.Fatalf("a walk answering a merge without a pull request left out %v", same) - } } // TestADependentOfABaseThatCameOutTheSameIsLeftOut is ADR 0267 rule 3 (novox/hq issue 607): a dependent is in a walk diff --git a/internal/inventory/source.go b/internal/inventory/source.go index f58714b8..a498b07b 100644 --- a/internal/inventory/source.go +++ b/internal/inventory/source.go @@ -2,6 +2,8 @@ package inventory import ( "context" + "encoding/json" + "strings" "github.com/novox/mesh-controller/internal/chain" ) @@ -43,6 +45,11 @@ func (i *Inventory) sourceRows(ctx context.Context, module string) (map[string][ // standing is, of a module's builds newest first, the index of the build that stands for the build at // `from`: the oldest of the unbroken run of builds behind it with its source fingerprint. +// +// **Across the switch to the new form** (novox/hq ADR 0453 §6, ADR 0499; R4e A9): a build of the new form (`src2:`, taken +// from the builder's key) continues the run of the build before it of the old form (`src1:` or none, build-agent's) +// when every artifact of the two has the same digest: the fingerprints cannot be compared, the programs can, and equal +// digests are no move. Past it the run goes on by the old fingerprint. func standing(builds []sourceRow, from int) int { at := from fp := builds[from].fingerprint @@ -50,14 +57,53 @@ func standing(builds []sourceRow, from int) int { return at } for j := from + 1; j < len(builds); j++ { - if builds[j].fingerprint != fp || builds[j].failedGate { + if builds[j].failedGate { break } + if builds[j].fingerprint != fp { + if !newForm(fp) || newForm(builds[j].fingerprint) || !sameDigests(builds[at].made, builds[j].made) { + break + } + fp = builds[j].fingerprint + } at = j } return at } +// newForm says a source fingerprint is of the new form (ADR 0453 §6). +func newForm(fp string) bool { return strings.HasPrefix(fp, "src2:") } + +// sameDigests says two builds' artifacts, as recorded (JSON of []Artifact), are the same programs: the same names, each +// with the same digest, wherever each was pushed. A build naming an artifact with no digest is never the same. +func sameDigests(a, b []byte) bool { + digests := func(raw []byte) map[string]string { + var made []Artifact + if json.Unmarshal(raw, &made) != nil || len(made) == 0 { + return nil + } + out := map[string]string{} + for _, m := range made { + d := digestIn(m.Reference) + if d == "" { + return nil + } + out[m.Name] = d + } + return out + } + da, db := digests(a), digests(b) + if da == nil || db == nil || len(da) != len(db) { + return false + } + for name, d := range da { + if db[name] != d { + return false + } + } + return true +} + // StandingBuild is the build that stands for a module's build (novox/hq issue 280): the oldest build of // the unbroken run of builds with its source fingerprint, at or before it — itself when its fingerprint // is empty or it starts the run. Answers its id and the manifest it was recorded with; empty when the diff --git a/internal/inventory/standing_switch_test.go b/internal/inventory/standing_switch_test.go new file mode 100644 index 00000000..5d1aa7b8 --- /dev/null +++ b/internal/inventory/standing_switch_test.go @@ -0,0 +1,49 @@ +package inventory + +import ( + "encoding/json" + "testing" +) + +func madeOf(t *testing.T, refs map[string]string) []byte { + t.Helper() + var made []Artifact + for name, ref := range refs { + made = append(made, Artifact{Name: name, Kind: "bundle", Reference: ref}) + } + raw, err := json.Marshal(made) + if err != nil { + t.Fatal(err) + } + return raw +} + +// R4e A9 (ADR 0453 §6, ADR 0499): at the switch, a build taken from the builder's key (`src2:`) whose every artifact has +// the digest of the build-agent build before it (`src1:`) is no move: it stands on that build, wherever each was pushed. +// One digest that differs is a move, as is a build of the old form after one of the new. +func TestAKeysBuildWithTheOldBuildsDigestsIsNoMove(t *testing.T) { + const d1, d2 = "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "sha256:2222222222222222222222222222222222222222222222222222222222222222" + old := sourceRow{id: "build-1", fingerprint: "src1:aa", made: madeOf(t, map[string]string{ + "tools": "artifact-store://x/tools/blobs/" + d1, "img": "artifact-store://x/img@" + d2})} + key := sourceRow{id: "key-bb-1", fingerprint: "src2:bb", made: madeOf(t, map[string]string{ + "tools": "artifact-store://x/tools/blobs/" + d1, "img": "registry.internal:5000/x/img@" + d2})} + if at := standing([]sourceRow{key, old}, 0); at != 1 { + t.Fatalf("the key's build with the same digests stands on %d, want the old build", at) + } + moved := key + moved.made = madeOf(t, map[string]string{"tools": "artifact-store://x/tools/blobs/" + d2, "img": "x/img@" + d2}) + if at := standing([]sourceRow{moved, old}, 0); at != 0 { + t.Fatalf("a key's build with another digest stands on %d, want itself (a move)", at) + } + // Two key builds of one fingerprint are one run, as two builds of one fingerprint always were. + again := key + again.id = "key-bb-2" + if at := standing([]sourceRow{again, key, old}, 0); at != 2 { + t.Fatalf("a run of one new fingerprint on an old build of the same digests stands on %d", at) + } + // An old build after a new one never continues the new one's run. + if at := standing([]sourceRow{old, key}, 0); at != 0 { + t.Fatalf("an old build continues a new one's run: %d", at) + } +} From 0aec975d1930e44888ab96d82738f0051882a1ff Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:02:00 +0200 Subject: [PATCH 09/38] Keep an assignment of a module whose key is coming pending on that key and ask no build; nothing asks a request again at start (issue 1851, A10) --- .../builder_on_the_bus_test.go | 7 + cmd/mesh-controller/chain.go | 46 +----- cmd/mesh-controller/mesh_for_test.go | 8 + cmd/mesh-controller/new_modules.go | 20 ++- cmd/mesh-controller/pending.go | 138 +++++++++++++++--- .../pending_integration_test.go | 130 +++++------------ cmd/mesh-controller/pending_keys_test.go | 39 +++++ 7 files changed, 222 insertions(+), 166 deletions(-) create mode 100644 cmd/mesh-controller/pending_keys_test.go diff --git a/cmd/mesh-controller/builder_on_the_bus_test.go b/cmd/mesh-controller/builder_on_the_bus_test.go index fe277dbd..b17ff06e 100644 --- a/cmd/mesh-controller/builder_on_the_bus_test.go +++ b/cmd/mesh-controller/builder_on_the_bus_test.go @@ -51,6 +51,10 @@ func aBuilderOnTheBus(t *testing.T) builderOnTheBus { servingBus.Store(js) t.Cleanup(func() { servingBus.Store(nil) }) withTrunkList(t, true) + // The indexer on this bus is read, not the "not held" a mesh with no bus reads (aMeshOnTheStore). + wasIndexed := indexedModule + indexedModule = theIndexedModule + t.Cleanup(func() { indexedModule = wasIndexed }) return b } @@ -130,3 +134,6 @@ func asksWithPaths(t *testing.T) *[][3]string { t.Cleanup(func() { askABuild = was }) return &asked } + +// theIndexedModule is the look at the indexer as the controller makes it, kept before any test replaces it. +var theIndexedModule = indexedModule diff --git a/cmd/mesh-controller/chain.go b/cmd/mesh-controller/chain.go index 0f648a9d..fa28d8ae 100644 --- a/cmd/mesh-controller/chain.go +++ b/cmd/mesh-controller/chain.go @@ -51,55 +51,13 @@ func attachTheChain(ctx context.Context, inv *inventory.Inventory, server *link. fmt.Printf("chain: the builds are not laid out under their modules yet, and none is read until they are: %v\n", err) go layBuildsOutSoon(ctx, inv) } - // And the requests the move of delivery 1 found never handed over, asked again once (askAgain), once it is serving. - go askAgainSoon(ctx, inv) + // The requests the move of delivery 1 found never handed over are not asked again: the controller asks no build + // (novox/hq ADR 0501; issue 1851, A10), and the builder builds every fingerprint the indexer writes. // And what delivery 1 recorded of what each machine was sent, copied onto the keys of the assignments (CopySentBuilds). go copySentBuildsSoon(ctx, inv) return nil } -// askAgain asks once more for each build request the move found never handed over (novox/hq issue 981 note 7 Q5): the -// requests a merge could not ask for, which a person's hand `build` would otherwise have to repeat. **At most once**: a -// request is claimed before it is asked and settled after, so a crash between loses the ask, which is where it stood. -func askAgain(ctx context.Context, inv *inventory.Inventory) { - pending, err := inv.RequestsToAskAgain(ctx) - if err != nil { - fmt.Printf("chain: the requests never handed over cannot be read: %v\n", err) - return - } - for _, r := range pending { - if claimed, err := inv.ClaimToAskAgain(ctx, r.ID); err != nil || !claimed { - if err != nil { - fmt.Printf("chain: %s cannot be claimed to be asked again: %v\n", r.ID, err) - } - continue - } - id, err := askABuild(ctx, buildSource{Repository: r.Repository, Seat: r.Seat}, r.Path, r.Ref) - if err != nil { - if serr := inv.AskedAgainAs(ctx, r.ID, "not asked again: "+err.Error()); serr != nil { - fmt.Printf("chain: %s was not asked again, and that could not be kept: %v\n", r.ID, serr) - } - fmt.Printf("chain: %s was never handed over and cannot be asked again: %v\n", r.ID, err) - continue - } - recordBuildRequest(ctx, inv, inventory.BuildRequest{ID: id, Repository: r.Repository, Seat: r.Seat, Path: r.Path, - Ref: r.Ref, Commit: r.Commit, For: r.For}) - if err := inv.AskedAgainAs(ctx, r.ID, id); err != nil { - fmt.Printf("chain: %s was asked again as %s, and that could not be kept: %v\n", r.ID, id, err) - } - fmt.Printf("chain: %s, never handed over, is asked again as %s\n", r.ID, id) - } -} - -// askAgainSoon waits for the controller to be serving, which an ask needs, and then asks again. -func askAgainSoon(ctx context.Context, inv *inventory.Inventory) { - select { - case <-ctx.Done(): - case <-time.After(time.Minute): - askAgain(ctx, inv) - } -} - // kindSentBuildsNotCopied is the condition of the one-time copy of node.sent_builds onto the assignments' keys that has not // succeeded: until it has, a module an assignment pulled in may be missing from what a machine is read to run. const kindSentBuildsNotCopied = "sent-builds-not-copied" diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index 68be9da6..43f73c75 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -13,6 +13,7 @@ import ( "github.com/novox/mesh-controller/internal/conditions" "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/overlay" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" @@ -58,6 +59,13 @@ func aMeshOnTheStore(t *testing.T) *stores { // A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is // open, and a mesh with no bus would otherwise read as one whose conditions cannot be read. withConditionsInMemory(t) + // The indexer holds no module unless a test puts one on its bus (aBuilderOnTheBus): an assignment's look at it + // reads "not held" on a mesh with no bus of its own. + wasIndexed := indexedModule + indexedModule = func(context.Context, string) (sdkdelivery.Module, bool, error) { + return sdkdelivery.Module{}, false, nil + } + t.Cleanup(func() { indexedModule = wasIndexed }) for _, m := range provided { if err := open.inventory.Provide(t.Context(), m); err != nil { t.Fatal(err) diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go index 75871383..58d88cd3 100644 --- a/cmd/mesh-controller/new_modules.go +++ b/cmd/mesh-controller/new_modules.go @@ -57,15 +57,21 @@ func takeNewModules(ctx context.Context, open *stores, only ...string) { return } defer closeKeys() - takeNewModulesWith(ctx, open.inventory, keys, only...) + for _, module := range takeNewModulesWith(ctx, open.inventory, keys, only...) { + // An assignment waiting for it is made now (ADR 0501 §6; issue 1851, A10). + for _, line := range settlePendingOnKey(ctx, open, module) { + fmt.Println(line) + } + } } -// takeNewModulesWith is takeNewModules on the buckets handed in. -func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newModuleReads, only ...string) { +// takeNewModulesWith is takeNewModules on the buckets handed in; it answers the modules it registered. +func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newModuleReads, only ...string) []string { entries, err := inv.Catalogued(ctx) if err != nil { fmt.Printf("new modules: the catalogue cannot be read: %v\n", err) - return + // empty-on-error: said above; nothing is registered this time, and the next write of a key looks again + return nil } held := map[string]bool{} for _, e := range entries { @@ -74,7 +80,8 @@ func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newM values, unread, err := keys.Modules(ctx) if err != nil { fmt.Printf("new modules: the indexer's modules cannot be read: %v\n", err) - return + // empty-on-error: said above; nothing is registered this time, and the next write of a key looks again + return nil } for module, err := range unread { if !held[module] && (len(only) == 0 || slices.Contains(only, module)) { @@ -88,12 +95,14 @@ func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newM } } sort.Strings(names) + var registered []string for _, module := range names { v := values[module] _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: module, Trunk: v, Ask: registrationAsk{Merge: v.Commit}}) switch { case err == nil: forgetNewModuleWait(module) + registered = append(registered, module) fmt.Printf("%s is new in %s at %.8s: registered from the builder's done key, assigned nowhere\n", module, v.Repository, v.Commit) case errors.Is(err, errKeyWaits): @@ -107,6 +116,7 @@ func takeNewModulesWith(ctx context.Context, inv *inventory.Inventory, keys newM Why: "not registered: " + err.Error()}) } } + return registered } // newModuleReads is what the take-in of new modules reads of the builder and the indexer. diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index a5ec7359..724a6b1a 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -9,6 +9,9 @@ import ( "strings" "time" + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -85,8 +88,30 @@ const ( unknownModule = iota buildInFlight knownNotBuilt + // keyComing is a module the indexer holds and the catalogue does not yet: registered from the builder's done key + // of it (ADR 0501 §6), which nobody asks. + keyComing ) +// keyPendingPrefix opens the build a pending assignment waits for when it waits for a module's key: no build is asked +// (ADR 0501 §5, §6; issue 1851, A10), so it names the module's key, not a build id. +const keyPendingPrefix = "key:" + +// indexedModule is the indexer's trunk value of a module the catalogue does not hold, found when it holds one. A +// variable so a test needs no bus. +var indexedModule = func(ctx context.Context, module string) (sdkdelivery.Module, bool, error) { + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return sdkdelivery.Module{}, false, err + } + defer closeKeys() + v, err := keys.ModuleAt(ctx, module, "") + if errors.Is(err, builderkeys.ErrNoValue) { + return sdkdelivery.Module{}, false, nil + } + return v, err == nil, err +} + // notHeld is where a module the catalogue does not hold stands: its case, the build request it was read // from, and the sentence that says it. type notHeld struct { @@ -96,7 +121,24 @@ type notHeld struct { } // whereIs looks for a module the catalogue does not hold among the build requests the controller keeps. -func whereIs(ctx context.Context, inv *inventory.Inventory, module string, now time.Time) (notHeld, error) { +// notHeldReads is what whereIs reads of the records (*inventory.Inventory, or a test's). +type notHeldReads interface { + RequestsNamed(ctx context.Context, name string) ([]inventory.RequestOutcome, error) + Catalogue(ctx context.Context) (map[string]catalogue.Manifest, error) + RequestedNames(ctx context.Context) ([]string, error) +} + +func whereIs(ctx context.Context, inv notHeldReads, module string, now time.Time) (notHeld, error) { + // **Its key is coming** (ADR 0501 §6): the indexer holds it, and the controller registers it from the builder's done + // key of it. Read before the build requests of before R4e, which nothing writes any more. + if v, held, err := indexedModule(ctx, module); err != nil { + return notHeld{}, fmt.Errorf("whether the indexer holds %s cannot be read: %w", module, err) + } else if held { + return notHeld{kind: keyComing, request: inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ + ID: keyPendingPrefix + module, Repository: v.Repository, Seat: gitSeat, Path: keyPath(v)}}, + said: fmt.Sprintf("%s is not registered yet: the indexer holds it in %s at %.8s, and it is registered from "+ + "the builder's done key of it, which no one asks", module, v.Repository, v.Commit)}, nil + } requests, err := inv.RequestsNamed(ctx, module) if err != nil { return notHeld{}, err @@ -169,6 +211,12 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts return "", err } switch where.kind { + case keyComing: + lead := where.said + if opts.Build { + lead += "\n build \"true\" asks nothing: the builder builds every fingerprint the indexer writes (ADR 0501 §6)" + } + return keepPending(ctx, open, node, module, where.request, waiting, lead) case buildInFlight: lead := where.said if opts.Build { @@ -183,30 +231,20 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts also = fmt.Sprintf("\n %s already has a pending assignment of %s, waiting for build %s; build "+ "\"true\" asks for a new build and makes it wait for that one", node, module, waiting.Build) } - return "", fmt.Errorf("%w: %s%s\n assign it again with build \"true\" to ask for its build and keep this "+ - "assignment until the build registers it; or `build` it (repository %s, path %s) and assign it "+ - "once it is registered", inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path)) + return "", fmt.Errorf("%w: %s%s\n it is registered from the builder's done key once a merge adds it to %s "+ + "(path %s) and the indexer holds it, or at the controller's terminal; assign it then", + inventory.ErrNoSuchModule, where.said, also, last.Repository, orRoot(last.Path)) } - if waiting != nil && waiting.State == inventory.PendingApplying { - return "", fmt.Errorf("%s is being assigned %s now; nothing was asked", node, module) - } - source := buildSource{Repository: last.Repository, Seat: last.Seat} - id, err := askABuild(ctx, source, last.Path, last.Ref) - if err != nil { - return "", fmt.Errorf("%s\n its build could not be asked for: %w; nothing was assigned", where.said, err) - } - asked := inventory.RequestOutcome{BuildRequest: inventory.BuildRequest{ID: id, Repository: last.Repository, - Seat: last.Seat, Path: last.Path, Ref: last.Ref, For: "assign", At: now.UTC()}} - recordBuildRequest(ctx, inv, asked.BuildRequest) - lead := fmt.Sprintf("%s\n build %s of %s is asked for now; %s can be assigned once it registers it", - where.said, id, requestSource(asked), module) - return keepPending(ctx, open, node, module, asked, waiting, lead) + // **No build is asked** (ADR 0501 §5, §6): a module the indexer does not hold has no key coming. + return "", fmt.Errorf("%w: %s\n build \"true\" asks no build: the controller asks none (ADR 0501 §5); a module "+ + "is registered from the builder's done key once a merge adds it to a repository the catalogue builds from, or "+ + "at the controller's terminal, and the indexer holds no %s", inventory.ErrNoSuchModule, where.said, module) } answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) + - "\n a module in a repository is built with `build` (its repository and path), then assigned" + "\n a module in a repository is registered from the builder's done key once a merge adds it, then assigned" if opts.Build { - answer += "\n build \"true\" asks for nothing here: the controller has no build request saying where a " + - "module of that name is" + answer += "\n build \"true\" asks for nothing: the controller asks no build (ADR 0501 §5), and the indexer " + + "holds no module of that name" } return "", fmt.Errorf("%w%s", inventory.ErrNoSuchModule, answer) } @@ -316,7 +354,7 @@ func clock(t time.Time) string { return t.Local().Format("2006-01-02 15:04:05 MS // closestNames is up to three names near the one asked: registered modules and directories the controller // asked to build. -func closestNames(ctx context.Context, inv *inventory.Inventory, name string) []string { +func closestNames(ctx context.Context, inv notHeldReads, name string) []string { var candidates []string if shelf, err := inv.Catalogue(ctx); err == nil { for m := range shelf { @@ -535,6 +573,19 @@ func settlePending(ctx context.Context, open *stores, now time.Time) []string { } continue } + if strings.HasPrefix(p.Build, keyPendingPrefix) { + why, err := keyPendingEnded(ctx, p.Module) + if err != nil { + say("the key %s waits for cannot be read: %v", p.Module, err) + continue + } + if why != "" { + if line := expirePending(ctx, inv, p, why); line != "" { + said = append(said, line) + } + } + continue + } requests, err := inv.RequestsNamed(ctx, p.Module) if err != nil { say("the build requests for %s could not be read: %v", p.Module, err) @@ -744,3 +795,46 @@ func withdrawPending(ctx context.Context, inv *inventory.Inventory, node, module } return "", false, nil } + +// keyPendingEnded is why a pending assignment waiting for a module's key ends: the indexer no longer holds the module, +// or the builder's key of its fingerprint failed; empty while it is coming. A variable so a test needs no bus. +var keyPendingEnded = func(ctx context.Context, module string) (string, error) { + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return "", err + } + defer closeKeys() + v, err := keys.ModuleAt(ctx, module, "") + if errors.Is(err, builderkeys.ErrNoValue) { + return fmt.Sprintf("the indexer no longer holds %s", module), nil + } + if err != nil || v.FP2 == "" { + return "", err + } + k, err := keys.Build(ctx, module, v.FP2) + if err != nil { + return "", err + } + if k.Failed() { + return k.Says(), nil + } + return "", nil +} + +// settlePendingOnKey assigns, now, every pending assignment of a module its done key just registered (ADR 0501 §6): +// what an outcome of a build did for one waiting on that build. +func settlePendingOnKey(ctx context.Context, open *stores, module string) []string { + waiting, err := open.inventory.Pending(ctx, time.Time{}) + if err != nil { + return []string{fmt.Sprintf("the pending assignments could not be read after %s was registered: %v", module, err)} + } + var said []string + for _, p := range waiting { + if p.Module == module { + if line := applyPending(ctx, open, p, keyPendingPrefix+module); line != "" { + said = append(said, line) + } + } + } + return said +} diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index 7000200a..e45d5207 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -13,6 +13,7 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -36,11 +37,13 @@ func aCatalogueMesh(t *testing.T) *stores { return open } +// mergeAdding keeps the build request a merge adding a module asked before R4e (issue 300): nothing asks one since (ADR +// 0501 §6), and the pending assignments still read the ones kept. These tests are of what an assignment does with one. func mergeAdding(t *testing.T, open *stores, dir, commit string) { t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, - Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { + if err := open.inventory.RecordBuildRequest(t.Context(), inventory.BuildRequest{ID: "b-" + dir, + Repository: "novox/mesh-catalog", Seat: "git", Path: dir, Ref: "main", Commit: commit, For: "merge", + At: time.Now().UTC()}); err != nil { t.Fatal(err) } } @@ -225,101 +228,10 @@ func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { } } -func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { - t.Fatal(err) - } - asked := asksWithPaths(t) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil { - t.Fatal(err) - } - if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { - t.Fatalf("asked %v", *asked) - } - for _, want := range []string{"known and not registered", "no space left on device", - "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - requests, _ := inv.RequestsNamed(ctx, "sensors") - if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { - t.Fatalf("the request is not kept as assign's: %+v", requests) - } -} - -func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { - t.Fatal(err) - } - // Waiting on the failed build, its expiry not yet settled. - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", - Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { - t.Fatal(err) - } - _, err := assign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { - t.Fatalf("known and not built with a pending assignment waiting: %v", err) - } - asked := asksWithPaths(t) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { - t.Fatalf("assign with build: %q %v", said, err) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { - t.Fatalf("pending: %+v", pending) - } - // The new build in flight now: build "true" again asks nothing. - if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || - !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { - t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) - } -} - func TestAFailedAskIsNotABuildInFlight(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() inv := open.inventory - asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { - return "", errors.New("cannot submit a build: nats: timeout") - } - t.Cleanup(func() { askABuild = was }) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { - t.Fatalf("a merge that could not ask: %v", err) - } - if len(pendingOf(t, open)) != 0 { - t.Fatal("a pending assignment waits on a build never asked") - } - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { t.Fatal(err) @@ -327,7 +239,7 @@ func TestAFailedAskIsNotABuildInFlight(t *testing.T) { if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { t.Fatal(err) } - _, err = assign(ctx, open, "laptop", "gauges") + _, err := assign(ctx, open, "laptop", "gauges") if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { t.Fatalf("a build not handed over: %v", err) } @@ -737,3 +649,31 @@ func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) { t.Fatalf("past the bound: %+v", rows) } } + +// A10 (ADR 0501 §6; issue 1851): an assignment of a module the indexer holds and the catalogue does not yet waits for +// the builder's done key of it, asking nothing, and is made when that key registers the module. +func TestAnAssignmentWaitsForTheKeyAndAsksNothing(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + noBuildAsked(t) + b := aBuilderOnTheBus(t) + fp := fpOf('a') + manifest := b.indexed(t, "sensors", "1", "novox/mesh-catalog", "modules/sensors", "3da80a4b00aa0000", fp) + b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildBuilding) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil || !strings.Contains(said, "registered from the builder's done key") || !strings.Contains(said, "kept as pending") { + t.Fatalf("A10: the assignment of a module whose key is coming: %q %v", said, err) + } + if p := pendingOf(t, open); len(p) != 1 || p[0].Build != keyPendingPrefix+"sensors" { + t.Fatalf("A10: pending %+v", p) + } + settlePending(ctx, open, time.Now().Add(time.Hour)) + if p := pendingOf(t, open); p[0].State != inventory.PendingWaiting { + t.Fatalf("A10: a pending assignment waiting for a building key expired: %+v", p) + } + b.built(t, "sensors", "1", fp, manifest, sdkdelivery.BuildDone) + takeNewModules(ctx, open, "sensors") + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatalf("A10: the key registered sensors and the pending assignment was not made: %+v", pendingOf(t, open)) + } +} diff --git a/cmd/mesh-controller/pending_keys_test.go b/cmd/mesh-controller/pending_keys_test.go new file mode 100644 index 00000000..4423f0be --- /dev/null +++ b/cmd/mesh-controller/pending_keys_test.go @@ -0,0 +1,39 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A10 (ADR 0501 §6; issue 1851): a module the catalogue does not hold and the indexer does is one whose key is coming: +// an assignment of it waits for that key, named by the module, with no build asked and no build request read. +func TestAModuleTheIndexerHoldsIsWaitedForByItsKey(t *testing.T) { + was := indexedModule + indexedModule = func(_ context.Context, module string) (sdkdelivery.Module, bool, error) { + return sdkdelivery.Module{Repository: "novox/mesh-catalog", Commit: "3da80a4b00aa", Path: "modules/" + module}, true, nil + } + t.Cleanup(func() { indexedModule = was }) + where, err := whereIs(t.Context(), noRequests{}, "sensors", time.Now()) + if err != nil { + t.Fatal(err) + } + if where.kind != keyComing || where.request.ID != keyPendingPrefix+"sensors" || where.request.Path != "modules/sensors" || + !strings.Contains(where.said, "registered from the builder's done key") { + t.Fatalf("a module whose key is coming is read as %+v", where) + } +} + +// noRequests is records with no build request and no module: what a mesh after R4e reads of a new module. +type noRequests struct{} + +func (noRequests) RequestsNamed(context.Context, string) ([]inventory.RequestOutcome, error) { + return nil, nil +} +func (noRequests) Catalogue(context.Context) (map[string]catalogue.Manifest, error) { return nil, nil } +func (noRequests) RequestedNames(context.Context) ([]string, error) { return nil, nil } From 7f6d418205f0bfe4d5c795a2e68697fe9d38ffc4 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:03:14 +0200 Subject: [PATCH 10/38] Mark the walks the integration tests drive around the one ask left as the builder's own, and expect no ask of a moved module (issue 1851, A5, A6) --- .../key_takein_integration_test.go | 6 ++-- .../one_group_one_plan_test.go | 17 ++++------ cmd/mesh-controller/queue_test.go | 1 + .../replays_integration_test.go | 32 ------------------- 4 files changed, 11 insertions(+), 45 deletions(-) diff --git a/cmd/mesh-controller/key_takein_integration_test.go b/cmd/mesh-controller/key_takein_integration_test.go index 24ba2d98..1b68c58d 100644 --- a/cmd/mesh-controller/key_takein_integration_test.go +++ b/cmd/mesh-controller/key_takein_integration_test.go @@ -66,10 +66,12 @@ func TestADoneKeyOfTheTrunkRegistersUnderTheSameRules(t *testing.T) { t.Fatalf("0501 §1: sudo is %q", shelf["sudo"].Version) } got, found, err := inv.BuildByID(ctx, kept.ID) - if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference || - got.SourceFingerprint != fpOf('a') { + if err != nil || !found || len(got.Made) != 1 || got.Made[0].Reference != kept.Made[0].Reference { t.Fatalf("0498 §8: the recorded build is %+v (%v)", got, err) } + if fps, err := inv.BuildSourceFingerprints(ctx, "sudo"); err != nil || fps["c0ffee0123456789"] != fpOf('a') { + t.Fatalf("0498 §8: the recorded build's fingerprint is %v (%v), not the key's", fps, err) + } // Recording the same key again is one record. if _, _, err := takeInKey(ctx, inv, keys, keyTake{Module: "sudo", Trunk: trunk, Ask: registrationAsk{Merge: trunk.Commit}}); err != nil { t.Fatal(err) diff --git a/cmd/mesh-controller/one_group_one_plan_test.go b/cmd/mesh-controller/one_group_one_plan_test.go index 14c7647a..60fd5201 100644 --- a/cmd/mesh-controller/one_group_one_plan_test.go +++ b/cmd/mesh-controller/one_group_one_plan_test.go @@ -109,8 +109,8 @@ func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *t } // Issue 1049: a move delivered half before its second merge. The controller adds a module the catalogue removes, in one -// group: the merges are one batch and one plan in either order, the module is built once and from the controller's -// repository, and the catalogue's removal does not forget it. +// group: the merges are one batch and one plan in either order, no build of the module is asked (it is registered from +// the builder's key, ADR 0501 §6), and the catalogue's removal does not forget it. func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOne(t *testing.T) { for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { t.Run(name, func(t *testing.T) { @@ -149,17 +149,12 @@ func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOn if len(ws) != 1 || len(bs) != 0 { t.Fatalf("a move: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) } - var fromController int + // **No build of it is asked from either repository** (ADR 0501 §6, issue 1851): the move is registered from the + // builder's done key of route-proxy in the controller's repository, on the operator's Approve of the group. for _, a := range *asked { - if strings.Contains(a[0], "mesh-controller") && a[1] == "modules/route-proxy" { - fromController++ + if a[1] == "modules/route-proxy" { + t.Errorf("a build of route-proxy was asked: %v", a) } - if strings.Contains(a[0], "mesh-catalog") && a[1] == "modules/route-proxy" { - t.Errorf("route-proxy was built from the repository it left: %v", a) - } - } - if fromController != 1 { - t.Fatalf("route-proxy was built %d time(s) from the controller's repository: %v", fromController, *asked) } entries, err := open.inventory.Catalogued(ctx) if err != nil { diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index eb997872..43219812 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -24,6 +24,7 @@ import ( // asksRecorded makes every ask a plan makes return the next id in a row, and says which were asked. func asksRecorded(t *testing.T) *[]string { t.Helper() + asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): these tests are of what a walk does around it var asked []string was := askABuild askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go index deb14f0b..37a4531d 100644 --- a/cmd/mesh-controller/replays_integration_test.go +++ b/cmd/mesh-controller/replays_integration_test.go @@ -418,38 +418,6 @@ func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T } } -func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - var asked [][3]string - asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it - was := askABuild - askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { - asked = append(asked, [3]string{source.Repository, path, ref}) - return "build-" + path, nil - } - t.Cleanup(func() { askABuild = was }) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00", - Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"}, - ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"} - if len(asked) != 1 || asked[0] != want { - t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want) - } - if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { - t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) - } -} - func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { open := aMesh(t) ctx := t.Context() From f76da2ce9c7233c77dde5c00f8a9c534b00fe23b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:05:49 +0200 Subject: [PATCH 11/38] =?UTF-8?q?Ask=20the=20checker=20for=20a=20try=20of?= =?UTF-8?q?=20a=20branch=20head,=20not=20build-agent,=20and=20read=20its?= =?UTF-8?q?=20word=20in=20today's=20answer=20(issue=201851,=20A11,=20ADR?= =?UTF-8?q?=200501=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/try.go | 205 +++++++++----------- cmd/mesh-controller/try_integration_test.go | 39 ++++ cmd/mesh-controller/try_test.go | 97 +++------ 3 files changed, 159 insertions(+), 182 deletions(-) diff --git a/cmd/mesh-controller/try.go b/cmd/mesh-controller/try.go index a120385c..d306975e 100644 --- a/cmd/mesh-controller/try.go +++ b/cmd/mesh-controller/try.go @@ -18,35 +18,31 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// A branch is tried before its pull request (novox/hq issue 686). +// A branch is tried before its pull request (novox/hq issue 686), by the checker (novox/hq ADR 0501 §5; issue 1851). // -// A repository's store tests skip without a store, and no agent may start a database on its machine. So a -// builder pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time: -// each failure there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312). +// A repository's store tests skip without a store, and no agent may start a database on its machine. So a builder +// pushed, a reviewer read, and only the pull request's merge check ran those tests for the first time: each failure +// there cost a round of fifteen to thirty minutes (mesh-controller #283, #310, #312). // -// `try / ` asks the build seat for the repository's own check of a pushed branch or -// commit, as the merge check runs it (internal/builder/check.go): the ref merged into the trunk's head, the -// trunk's merge-check.sh, each part in its toolchain's container, against a throwaway store and bus. It is a -// check and nothing more: +// `try / ` asks the checker (mesh-delivery-checker's `checker_try`) to judge the commit the ref +// names merged onto the trunk, as it judges a pull request's head: the build modules' fetch, digest and test steps, +// the unit level and then the integration level, each level's kept verdicts taken. From R4e build-agent is asked no +// try. It is a check and nothing more: // -// - **no pull request, so no status**: its outcome is never said as `checked`, which is what the forge's -// holder sets statuses from (isTry, in Built); -// - **nothing registered, nothing built**: it names no module of the graph, so the merge gate does not run and -// no build is made for a merge (novox/hq ADR 0350's scratch builds are a pull request's check's alone); -// - **a check in the build queue**: after deliveries (link.KindOf, novox/hq issue 558), and bounded by the -// check's own bound (checkTimeout); -// - **the code a pull request's check would run anyway**: only a repository of the mesh, whose every pull -// request the build seat already checks, in containers that hold no runtime socket. +// - **no pull request, so no status**: the checker owes the forge nothing for a try; +// - **nothing registered, nothing built for delivery**: a try writes no verdict of a group and no preview; +// - **bounded**: one try of a head at a time — a second ask answers the first's id — and at most four open, the +// checker's own rule; a repository the checker judges no level of (hq) is refused by the checker, and said. // -// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id ` reads what -// came of it from the build's log, where the build seat says it whole (link.StepTried): holding its answer up -// to tryWait for a verdict, like `plans wait` (novox/hq issue 229). Nothing is kept beside the log. +// A check outlasts a seat verb's one answer, so the ask answers its id at once and `try --id ` reads what came of +// it from the checker (`checker_try_read`), holding its answer up to tryWait for a verdict, like `plans wait`. // tryWait is how long reading a try holds its answer for the verdict. A variable so a test need not wait. var tryWait = 20 * time.Second // A ref as the forge is asked for it — a branch or a commit, never an option — and a `go test -run` pattern. var ( + tryID = regexp.MustCompile(`^try-[0-9]+$`) tryRef = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._/-]*$`) tryTests = regexp.MustCompile(`^[A-Za-z0-9_^$|().*+?/\[\]-]+$`) ) @@ -77,34 +73,70 @@ func isTry(result link.BuildResult) bool { return c != nil && (c.Try || (c.Number == 0 && c.Group == "")) } -// asATry makes a check's ask a try's: the repository's own script alone, for no pull request. -func asATry(request link.BuildRequest, tests string, storeOnly bool) link.BuildRequest { - c := *request.Check - c.Number, c.Try, c.Tests, c.StoreOnly = 0, true, tests, storeOnly - c.Paths, c.Modules, c.New, c.Dependents, c.Manifests, c.Builds = nil, nil, nil, nil, nil, nil - c.Judge, c.Plan, c.Group, c.Members = "", nil, "", nil - request.Check = &c - return request +// checkerModule is the module whose tools judge a try (mesh-delivery-checker). +const checkerModule = "mesh-delivery-checker" + +// checkerTry is the checker's answer to a try's ask: the try's id. +type checkerTry struct { + ID string `json:"id"` } -// maxTries is how many tries may wait or run at once, mesh-wide: each holds a build machine for minutes. -const maxTries = 4 +// checkerTryRead is the checker's word on a try (`checker_try_read`): its state queued, running, pass, fail or error +// (could not run), what it said, the head and trunk judged, how long it took, what failed, and the machines it ran on. +type checkerTryRead struct { + ID string `json:"id"` + Repository string `json:"repository"` + Head string `json:"head"` + Trunk string `json:"trunk"` + Branch string `json:"branch,omitempty"` + State string `json:"state"` + Said string `json:"said"` + Took string `json:"took,omitempty"` + Failed []string `json:"failed,omitempty"` + Machines []string `json:"machines,omitempty"` +} -// triesIn reads the build queue for a new try of one commit: the id of a try of that commit already waiting or -// running — the answer to a second ask — and whether there is room for another. -func triesIn(q link.Queue, owner, repo, commit string) (same string, room bool) { - n := 0 - for _, a := range q.Asks { - if !link.IsTry(a.Request) || (a.State != link.AskWaiting && a.State != link.AskInFlight) { - continue +// askChecker asks one of the checker's tools and reads its result into into. A variable so a test needs no checker. +var askChecker = func(ctx context.Context, tool string, args map[string]any, into any) error { + js, err := aBus() + if err != nil { + return err + } + defer js.Close() + raw, _ := json.Marshal(args) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, checkerModule, tool, raw, 30*time.Second) + if err != nil { + return fmt.Errorf("%s.%s: %w", checkerModule, tool, err) + } + if answer.Error != "" { + return fmt.Errorf("%s.%s: %s", checkerModule, tool, answer.Error) + } + var wrapped struct { + Result json.RawMessage `json:"result"` + } + if err := json.Unmarshal(answer.Result, &wrapped); err == nil && len(wrapped.Result) > 0 { + return json.Unmarshal(wrapped.Result, into) + } + return json.Unmarshal(answer.Result, into) +} + +// tryOfChecker is the checker's word on a try as `try` answers it, today's answer unchanged (novox/hq issue 686): queued +// is waiting, and the machines it ran on are where. +func tryOfChecker(r checkerTryRead) tryAnswer { + a := tryAnswer{ID: r.ID, State: r.State, Said: r.Said, Repository: r.Repository, Commit: r.Head, Trunk: r.Trunk, + Took: r.Took, Failed: r.Failed, On: strings.Join(r.Machines, ", "), Base: r.Branch} + switch r.State { + case "queued": + a.State = "waiting" + if a.Said == "" { + a.Said = "it waits for the checker; ask again" } - n++ - c := a.Request.Check - if same == "" && strings.EqualFold(c.Owner, owner) && strings.EqualFold(c.Repo, repo) && a.Request.Ref == commit { - same = a.ID + case "running": + if a.Said == "" { + a.Said = "still running; ask again" } } - return same, n < maxTries + return a } // forgeCommit asks the forge, through its module's tools on the bus, which commit a branch or commit of a @@ -168,9 +200,9 @@ func tryCommand(ctx context.Context, args []string) error { if len(positionals) != 0 { return usage } - // Before it is a token of a bus subject: an id, never a pattern. - if !link.SafeBuildID(*id) { - return fmt.Errorf("%q is not a try's id", *id) + // An id, never a pattern: the checker's `try-`. + if !tryID.MatchString(*id) { + return fmt.Errorf("%q is not a try's id, try-", *id) } return tryRead(ctx, *id) } @@ -220,56 +252,33 @@ func tryCommand(ctx context.Context, args []string) error { return fmt.Errorf("%s/%s is no repository of the mesh — no module is built from it and its owner is not "+ "the core's — and only the mesh's are tried on the build seat; nothing was asked", owner, repo) } - request, err := checkRequestFor(ctx, open, p, checkScope{Mesh: true, From: mesh.From}, entries) - if err != nil { - return err + // **Asked of the checker, never of build-agent** (ADR 0501 §5): the head merged onto the trunk, as a pull request's. + askArgs := map[string]any{"repository": strings.ToLower(owner + "/" + repo), "head": commit, "trunk": trunk} + if *tests != "" { + askArgs["tests"] = *tests } - request = asATry(request, *tests, *storeOnly) - seat := buildSeatHeld(ctx) - // **Bounded in number** (novox/hq issue 686): one try of a commit at a time, and maxTries in all. - js, err := dialTheBus() - if err != nil { - return err + if *storeOnly { + askArgs["level"] = "integration" } - queue, err := link.ReadQueue(ctx, js, seat) - js.Close() - if err != nil { - return err + var asked checkerTry + if err := askChecker(ctx, "checker_try", askArgs, &asked); err != nil { + return fmt.Errorf("the checker did not take a try of %s/%s at %.12s; nothing was asked: %w", owner, repo, commit, err) } - same, room := triesIn(queue, owner, repo, commit) - if same != "" { - return printTry(tryAnswer{ID: same, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit, - On: seat, Said: fmt.Sprintf("a try of %s/%s at %.12s already waits or runs, as %s: nothing new was asked. "+ - "Read it with try id=%s", owner, repo, commit, same, same)}) - } - if !room { - return fmt.Errorf("%d tries already wait or run on the build seat, the most at once (`queue` lists them); "+ - "nothing was asked — read one of yours, or ask again when one has ended", maxTries) - } - ask, err := askOverOn(seat) - if err != nil { - return err - } - defer ask.Close() - if err := ask.Ask(ctx, request); err != nil { - return err - } - return printTry(tryAnswer{ID: request.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit, - Base: trunk, On: seat, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: its own "+ - "merge-check.sh, as a check in the build queue, bounded at %s; no status is set and nothing is registered. "+ - "Read it with try id=%s — a full run of this kind takes minutes", seat, owner, repo, commit, ref, trunk, - checkTimeout, request.ID)}) + return printTry(tryAnswer{ID: asked.ID, State: "asked", Repository: owner + "/" + repo, Ref: ref, Commit: commit, + Base: trunk, On: checkerModule, Said: fmt.Sprintf("asked %s to try %s/%s at %.12s (%s) merged into %s: the head "+ + "judged as a pull request's, unit level then integration level; no status is set and nothing is registered. "+ + "Read it with try id=%s", checkerModule, owner, repo, commit, ref, trunk, asked.ID)}) } // tryRead answers what came of a try, holding the answer up to tryWait for its verdict. func tryRead(ctx context.Context, id string) error { until := time.Now().Add(tryWait) for { - lines, err := buildLines(ctx, id) - if err != nil { - return err + var r checkerTryRead + if err := askChecker(ctx, "checker_try_read", map[string]any{"id": id}, &r); err != nil { + return printTry(tryAnswer{ID: id, State: "error", Said: "the checker's word on it cannot be read: " + err.Error()}) } - a := tryAnswerOf(id, lines) + a := tryOfChecker(r) if (a.State != "waiting" && a.State != "running") || !time.Now().Add(2*time.Second).Before(until) { return printTry(a) } @@ -279,34 +288,6 @@ func tryRead(ctx context.Context, id string) error { } } -// tryAnswerOf reads a try's answer from what the build seat said of it: its outcome under StepTried; `failed` -// for one that could not run — an error, never a pass; else how far it is. -func tryAnswerOf(id string, lines []link.BuildLine) tryAnswer { - a := tryAnswer{ID: id, State: "waiting", Said: "nothing is said of " + id + " yet: it waits in the build queue " + - "(`queue` says where), or no try has that id; ask again"} - for _, l := range lines { - switch l.Step { - case link.StepTried: - var t link.Tried - if err := json.Unmarshal([]byte(l.Message), &t); err != nil || t.Verdict == "" { - return tryAnswer{ID: id, State: "error", Said: "the build seat's word on it cannot be read: " + l.Message} - } - return tryAnswer{ID: id, State: t.Verdict, Said: t.Summary, Took: t.Took, Commit: t.Head, Trunk: t.Trunk, - Failed: t.Failed} - case "failed": - return tryAnswer{ID: id, State: "error", Said: "it could not run: " + l.Message} - case "checked": - // A check that is no try, or a build seat from before the verb: its verdict is not a try's. - return tryAnswer{ID: id, State: "error", Said: "the build seat checked it and said no try's outcome — " + - "it predates the verb (novox/hq issue 686), or " + id + " is no try: " + l.Message + "; `builds` with " + - "the id says every line"} - } - a.State, a.Last = "running", l.Message - a.Said = "still running; ask again. `builds` with the id says every line so far" - } - return a -} - func printTry(a tryAnswer) error { fmt.Fprintln(os.Stderr, a.Said) for _, l := range a.Failed { diff --git a/cmd/mesh-controller/try_integration_test.go b/cmd/mesh-controller/try_integration_test.go index c3b930e8..5885470c 100644 --- a/cmd/mesh-controller/try_integration_test.go +++ b/cmd/mesh-controller/try_integration_test.go @@ -4,8 +4,13 @@ package main import ( "context" + "encoding/json" + "reflect" + "strings" "testing" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -37,3 +42,37 @@ func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) } } + +// ADR 0501 §5 (issue 1851): a try is asked of the checker, the head merged onto the trunk, never of build-agent: +// --store-only is the integration level, --tests passes on, the ask answers the checker's id. +func TestATryIsAskedOfTheChecker(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + noBuildAsked(t) + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "sudo", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + wasForge, wasChecker := forgeCommit, askChecker + forgeCommit = func(context.Context, string, string, string) (string, string, error) { + return strings.Repeat("c", 40), "main", nil + } + var asked []map[string]any + askChecker = func(_ context.Context, tool string, args map[string]any, into any) error { + if tool != "checker_try" { + t.Errorf("asked %s", tool) + } + asked = append(asked, args) + return json.Unmarshal([]byte(`{"id":"try-1"}`), into) + } + t.Cleanup(func() { forgeCommit, askChecker = wasForge, wasChecker }) + if err := tryCommand(ctx, []string{"novox/Mesh-Catalog", "fix/x", "--store-only", "--tests", "^TestX$"}); err != nil { + t.Fatal(err) + } + want := map[string]any{"repository": "novox/mesh-catalog", "head": strings.Repeat("c", 40), "trunk": "main", + "level": "integration", "tests": "^TestX$"} + if len(asked) != 1 || !reflect.DeepEqual(asked[0], want) { + t.Fatalf("the checker was asked %v, want %v", asked, want) + } +} diff --git a/cmd/mesh-controller/try_test.go b/cmd/mesh-controller/try_test.go index 49ef75a8..e696945b 100644 --- a/cmd/mesh-controller/try_test.go +++ b/cmd/mesh-controller/try_test.go @@ -15,29 +15,6 @@ import ( // A branch is tried before its pull request (novox/hq issue 686): the repository's own check asked for a pushed // branch or commit, with no pull request. -// What a try asks is a check — after deliveries in the build queue, bounded as one — of the repository's own -// script alone: no module of the graph, so no gate; nothing built for a merge, so no reusable image. -func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) { - request := link.BuildRequest{ID: "build-1", Ref: "c0ffee", Check: &link.CheckRequest{Owner: "novox", Repo: "a", - Number: 7, Base: "main", Modules: []string{"a"}, New: []string{"b"}, Manifests: []string{"module.json"}, - Dependents: []string{"c"}, Builds: []link.CheckBuild{{Module: "a"}}, Judge: link.JudgeSelf, - Plan: &link.ChangePlan{}}} - tried := asATry(request, "^TestStore", true) - c := tried.Check - if !c.Try || c.Number != 0 || c.Tests != "^TestStore" || !c.StoreOnly { - t.Fatalf("not marked a try, or it kept a pull request's number: %+v", c) - } - if len(c.Modules)+len(c.New)+len(c.Manifests)+len(c.Dependents)+len(c.Builds) != 0 || c.Judge != "" || c.Plan != nil { - t.Fatalf("a try runs the gate or builds for a merge: %+v", c) - } - if c.Base != "main" || c.Owner != "novox" || tried.Ref != "c0ffee" { - t.Fatalf("a try lost what it merges into what: %+v", tried) - } - if link.KindOf(tried) != link.KindCheck { - t.Fatal("a try does not count as a check in the build queue") - } -} - // A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and // nothing is recorded or registered (the handler holds no store here, and would fail reaching for one). @@ -76,58 +53,38 @@ func TestATryOfAnUnknownRefIsRefusedLoudly(t *testing.T) { } } -// The answer of a try carries the lines that name what failed (novox/hq issue 460's shape) and how long it took. -func TestATrysAnswerCarriesTheFailingLines(t *testing.T) { - tried, _ := json.Marshal(link.Tried{Verdict: "fail", Summary: "its merge-check.sh failed: TestX", Took: "5m3s", - Head: "c0ffee", Trunk: "ca3f43c", Failed: []string{"--- FAIL: TestX (0.01s)", " x_test.go:12: got 1, want 2", - "FAIL\tgithub.com/novox/a/cmd\t3.1s"}}) - lines := []link.BuildLine{{Seq: 1, Step: "check", Message: "novox/a#0 at c0ffee"}, - {Seq: 2, Step: "output", Message: "ok github.com/novox/a/internal 1.2s"}, - {Seq: 3, Step: link.StepTried, Message: string(tried)}, {Seq: 4, Step: "checked", Message: "pass: x"}} - a := tryAnswerOf("build-1", lines) - if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 3 || a.Failed[0] != "--- FAIL: TestX (0.01s)" || - a.Trunk != "ca3f43c" || a.ID != "build-1" { - t.Fatalf("the answer lost the verdict, the time or the failing lines: %+v", a) +// The answer of a try is the checker's word, in today's shape (ADR 0501 §5; issue 1851): its verdict, the failing +// line, how long it took and where; queued is waiting; one that could not run is an error, never a pass. +func TestATrysAnswerIsTheCheckersWord(t *testing.T) { + var asked []string + was := askChecker + askChecker = func(_ context.Context, tool string, args map[string]any, into any) error { + asked = append(asked, tool) + raw, _ := json.Marshal(checkerTryRead{ID: "try-1", Repository: "novox/a", Head: "c0ffee", Trunk: "ca3f43c", + State: "fail", Said: "the unit level failed: TestX", Took: "5m3s", + Failed: []string{"unit: a/cmd.TestX: got 1, want 2"}, Machines: []string{"ace"}}) + return json.Unmarshal(raw, into) } - // Not answered yet: said as running, with how far it is — never as a pass. - if a := tryAnswerOf("build-1", lines[:2]); a.State != "running" || a.Last == "" { - t.Fatalf("a try still running: %+v", a) + t.Cleanup(func() { askChecker = was }) + if err := tryRead(t.Context(), "try-1"); err != nil { + t.Fatal(err) } - if a := tryAnswerOf("build-1", nil); a.State != "waiting" { - t.Fatalf("a try nothing was said of: %+v", a) + if len(asked) != 1 || asked[0] != "checker_try_read" { + t.Fatalf("a try is read from %v, not the checker", asked) } - // One that could not run is an error with the build seat's word, never a pass. - if a := tryAnswerOf("build-1", append(lines[:1:1], link.BuildLine{Seq: 2, Step: "failed", - Message: "cannot clone"})); a.State != "error" || !strings.Contains(a.Said, "cannot clone") { + a := tryOfChecker(checkerTryRead{ID: "try-1", State: "fail", Took: "5m3s", Head: "c0ffee", Trunk: "ca3f43c", + Failed: []string{"unit: a/cmd.TestX: got 1, want 2"}, Machines: []string{"ace"}}) + if a.State != "fail" || a.Took != "5m3s" || len(a.Failed) != 1 || a.Commit != "c0ffee" || a.Trunk != "ca3f43c" || a.On != "ace" { + t.Fatalf("the answer lost the verdict, the time, the failing line or where: %+v", a) + } + if a := tryOfChecker(checkerTryRead{ID: "try-2", State: "queued"}); a.State != "waiting" || a.Said == "" { + t.Fatalf("a try the checker has not started: %+v", a) + } + if a := tryOfChecker(checkerTryRead{ID: "try-3", State: "error", Said: "no node holds the go build module"}); a.State != "error" { t.Fatalf("a try that could not run: %+v", a) } -} - -// One try of a commit at a time, and four in all (novox/hq issue 686): a second ask is answered the first's id, -// a fifth is refused; a try that ended, and a pull request's check, count for nothing. -func TestTriesAreBoundedInNumber(t *testing.T) { - ask := func(id, repo, ref, state string, try bool) link.QueuedAsk { - return link.QueuedAsk{ID: id, State: state, Request: link.BuildRequest{ID: id, Ref: ref, - Check: &link.CheckRequest{Owner: "novox", Repo: repo, Try: try}}} - } - q := link.Queue{Asks: []link.QueuedAsk{ - ask("t1", "a", "c1", link.AskInFlight, true), ask("t2", "a", "c2", link.AskWaiting, true), - ask("t3", "b", "c1", link.AskDead, true), ask("p1", "a", "c9", link.AskWaiting, false)}} - if same, room := triesIn(q, "novox", "a", "c2"); same != "t2" || !room { - t.Fatalf("a second try of a commit: %q %v", same, room) - } - if same, room := triesIn(q, "novox", "b", "c1"); same != "" || !room { - t.Fatalf("a dead try, or another repository's, was taken for this one: %q %v", same, room) - } - if same, _ := triesIn(q, "novox", "a", "c9"); same != "" { - t.Fatalf("a pull request's check was taken for a try: %q", same) - } - q.Asks = append(q.Asks, ask("t4", "c", "c3", link.AskWaiting, true), ask("t5", "d", "c4", link.AskWaiting, true)) - if _, room := triesIn(q, "novox", "e", "c5"); room { - t.Fatal("a fifth try was let in") - } - // An id is checked before it is a token of a bus subject. - for _, id := range []string{"x.>", "*", "a b"} { + // An id is checked before it is asked of anybody. + for _, id := range []string{"x.>", "*", "a b", "build-1"} { if err := tryCommand(context.Background(), []string{"--id", id}); err == nil || !strings.Contains(err.Error(), "not a try's id") { t.Errorf("try --id %q: %v", id, err) From c7039d81e12c73719256cd1f62ccf545c8169950 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:09:43 +0200 Subject: [PATCH 12/38] =?UTF-8?q?Refuse=20build,=20rebuild=20and=20replay?= =?UTF-8?q?=20--register=20by=20name,=20naming=20what=20replaced=20them,?= =?UTF-8?q?=20and=20delete=20the=20hand=20builds=20(issue=201851,=20A11a,?= =?UTF-8?q?=20ADR=200501=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ask_bus_look_integration_test.go | 5 - cmd/mesh-controller/build.go | 237 ------------------ cmd/mesh-controller/dependents_test.go | 10 - cmd/mesh-controller/key_takein.go | 3 - cmd/mesh-controller/main.go | 24 +- cmd/mesh-controller/not_asked_id_test.go | 24 -- cmd/mesh-controller/order_test.go | 42 ---- cmd/mesh-controller/plan_retry.go | 95 ------- cmd/mesh-controller/queue.go | 202 +-------------- cmd/mesh-controller/queue_integration_test.go | 62 ----- cmd/mesh-controller/queue_test.go | 87 +------ cmd/mesh-controller/seatverbs.go | 51 ++-- cmd/mesh-controller/seatverbs_test.go | 93 +++---- cmd/mesh-controller/try.go | 35 --- cmd/mesh-controller/upgrades.go | 63 +---- .../walk_from_keys_integration_test.go | 2 +- .../walk_said_integration_test.go | 25 +- .../walk_wait_rebuild_754_integration_test.go | 63 ----- 18 files changed, 91 insertions(+), 1032 deletions(-) delete mode 100644 cmd/mesh-controller/not_asked_id_test.go delete mode 100644 cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go diff --git a/cmd/mesh-controller/ask_bus_look_integration_test.go b/cmd/mesh-controller/ask_bus_look_integration_test.go index 83bbee9f..d4b1eee4 100644 --- a/cmd/mesh-controller/ask_bus_look_integration_test.go +++ b/cmd/mesh-controller/ask_bus_look_integration_test.go @@ -116,11 +116,6 @@ func TestALookFailsTheTierTheBusRefusedPastTheBound(t *testing.T) { t.Fatalf("a bus silent past its bound did not fail the plan in words: %s %q", got.State, got.Note) } - // Rebuilt by hand while the bus still reloads: said as waiting, not as a failure and not with an empty reason. - joined, said, err := joinAPlan(ctx, open, "a") - if err != nil || !joined || !strings.Contains(said, "not asked yet: the bus did not answer") { - t.Fatalf("a rebuild left for the bus is said as %v %q %v", joined, said, err) - } // And retried by hand: the module is named as waiting, not as asked again. got, _ = inv.PlanByID(ctx, "plan-bus") if got.State == inventory.PlanFailed { diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index 326dcf12..a9c640cc 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -28,112 +28,6 @@ import ( // cheapest next step. That is how novox/hq ADR 0001 records `hal/sdk` reaching 34,636: // nothing in it was wrong, and no one edit was the one that should have been a new file. -// buildCommand builds a module from its source and records what came out. -// -// **Run where there is a container runtime**, which is why it is a command rather than something -// the control plane does on its own: building needs to run things on a machine, and what the -// control plane may send a machine is bounded by the declaration language. This is the shape the -// builder module will take when it is given work over the broker; today a person runs it, and the -// mesh records the result the same way either way. -// buildOn rebuilds every module the mesh holds that stands on the named module's artifacts — the -// rebuild a changed base needs, which nothing else asks for: their sources did not move, and -// "behind" does not see a base that did (novox/hq 04-ISSUES/131). Bases first among them too. -func buildOn(ctx context.Context, base string, wait time.Duration) error { - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - held, err := open.inventory.Catalogued(ctx) - if err != nil { - return err - } - against, err := open.inventory.BuiltAgainst(ctx) - if err != nil { - return err - } - var on []inventory.Entry - for _, e := range held { - if standsOnModule(e, base, against) { - on = append(on, e) - } - } - if len(on) == 0 { - fmt.Printf("nothing the mesh holds stands on %s\n", base) - return nil - } - on = orderByBases(on, against) - fmt.Printf("%d module(s) stand on %s:\n", len(on), base) - var failed []string - for _, e := range on { - fmt.Printf("--- %s\n", e.Manifest.Module) - source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} - if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil { - fmt.Printf(" %v\n", err) - failed = append(failed, e.Manifest.Module) - } - } - if len(failed) > 0 { - return fmt.Errorf("%d of %d could not be built: %s", len(failed), len(on), strings.Join(failed, ", ")) - } - fmt.Printf("\n%d module(s) rebuilt on %s. `push --behind` sends them on\n", len(on), base) - return nil -} - -func buildCommand(ctx context.Context, args []string) error { - set := flag.NewFlagSet("build", flag.ContinueOnError) - ref := set.String("ref", "", "the branch, tag or commit to build") - // A module is a repository and a path within it (novox/hq ADR 0069). Empty is the repository's - // root, which is the ordinary case and why this is a flag rather than a second argument. - path := set.String("path", "", "the module's directory inside the repository") - wait := set.Duration("wait", 10*time.Minute, "how long to wait for a builder to answer") - dryRun := set.Bool("dry-run", false, "build and print the manifest, recording nothing") - // Every module whose source has moved, rather than one named repository. - // - // **The mirror of `push --behind`, and the same argument** (novox/hq ADR 0010): the mesh - // already knows which modules are behind their source, so making a person read that list and - // retype each repository is asking them to be the loop. Naming a repository and asking which - // ones need building are different requests, so they are not combined. - behind := set.Bool("behind", false, "every module the mesh holds older than its source has") - on := set.String("on", "", "rebuild every module that stands on this module's artifacts — the rebuild a changed base needs") - // A repository on the mesh's own forge, named by its path there (novox/hq ADR 0111). Without it - // the repository is external, cloned exactly as given — see source.go. - self := set.Bool("self", false, "the repository is a path on the forge holding the git seat") - positionals, err := parseAround(set, args) - if err != nil { - return err - } - if *on != "" { - if len(positionals) != 0 || *behind || *self { - return errors.New("build --on names a base and nothing else") - } - return buildOn(ctx, *on, *wait) - } - - if *behind { - if len(positionals) != 0 || *self { - return errors.New("build or build --behind, not both: one names a " + - "repository and the other asks which need building") - } - return buildBehind(ctx, *wait) - } - if len(positionals) != 1 { - return errors.New("build [--self] [--path P] [--ref R] [--wait D] [--dry-run] | build --behind | build --on ") - } - source := buildSource{Repository: positionals[0]} - if *self { - if err := onASeat(source.Repository); err != nil { - return err - } - source.Seat = gitSeat - } - - if *dryRun { - return buildAndShow(ctx, source, *path, *ref, *wait) - } - return buildOne(ctx, source, *path, *ref, *wait) -} - // buildFrom turns what a builder said into what the mesh keeps. // // **By digest and path, never by where it was pushed** (novox/hq 04-ISSUES/102). The builder @@ -355,86 +249,6 @@ func builderCommand(ctx context.Context, args []string) error { return issueOnTheNewBus(ctx, inv, m, node, address) } -// buildBehind builds every module the mesh holds older than its source has. -// -// **This is the loop novox/hq ADR 0010 replaced a pipeline with, closed.** The mesh already -// records where each module came from and what its source last had; until this, a person read -// that list and retyped each repository — which is a person being the loop, and the thing a -// pipeline was doing before it was taken away. -// -// Each is built and recorded on its own. **One failing does not stop the others**, for the same -// reason one broken module no longer blocks a machine's whole declaration: a mesh where one bad -// repository holds back nine good ones is a mesh where nobody dares add the tenth. -func buildBehind(ctx context.Context, wait time.Duration) error { - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - - held, err := inv.Catalogued(ctx) - if err != nil { - return err - } - var stale []inventory.Entry - for _, e := range held { - if !e.Source.Current() { - stale = append(stale, e) - } - } - if len(stale) == 0 { - // Said rather than doing nothing quietly: "nothing needed building" and "this did not - // run" must never look the same. - fmt.Println("every module the mesh holds is what its source last had") - return nil - } - - fmt.Printf("%d module(s) behind their source:\n", len(stale)) - for _, e := range stale { - fmt.Printf(" %s %s < %s\n", - e.Manifest.Module, short(e.Source.BuiltFrom), short(e.Source.Head)) - } - fmt.Println() - - // Bases first: a module built before the module it stands on is built against the old one - // and reports success (novox/hq 04-ISSUES/131). - against, err := inv.BuiltAgainst(ctx) - if err != nil { - return err - } - stale = orderByBases(stale, against) - - var failed []string - for _, e := range stale { - fmt.Printf("--- %s\n", e.Manifest.Module) - // Its own recorded ref, not its head commit: a module tracking a branch should be built - // from that branch, and pinning to the commit the mesh happened to notice would quietly - // turn a tracked branch into a pin. - source := buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} - if err := buildOne(ctx, source, e.Source.Path, e.Source.Ref, wait); err != nil { - fmt.Printf(" %v\n", err) - failed = append(failed, e.Manifest.Module) - } - } - - if len(failed) > 0 { - return fmt.Errorf("%d of %d could not be built: %s", - len(failed), len(stale), strings.Join(failed, ", ")) - } - fmt.Printf("\n%d module(s) built. `push --behind` sends them to the machines running them\n", - len(stale)) - return nil -} - -// buildOne asks a build machine for one repository and records everything that came back. -// -// Separated from the command so `--behind` can walk a list without a second path to the same act. -func buildOne(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error { - _, err := buildOneAsked(ctx, source, path, ref, wait, false, "build") - return err -} - // recordAsked keeps a build request once it is asked (novox/hq issue 325), with who asked it, in a store // opened for the purpose: the asks reach here from processes that hold none. func recordAsked(ctx context.Context, r inventory.BuildRequest) { @@ -867,57 +681,6 @@ func publishable(result link.BuildResult, follows string) error { return nil } -// buildAndShow builds and prints the manifest without recording anything. -func buildAndShow(ctx context.Context, source buildSource, path, ref string, wait time.Duration) error { - repository, err := cloneFrom(ctx, source) - if err != nil { - return err - } - if err := verbMayAsk(ctx, source, repository, path); err != nil { - return err - } - ident, err := openIdentity(ctx) - if err != nil { - return err - } - defer ident.Close() - server, err := connectLink(ctx, nil, nil, nil) - if err != nil { - return err - } - defer server.Close() - - ask, err := askOverOn(buildSeatHeld(ctx)) - if err != nil { - return err - } - defer ask.Close() - - result, err := ask.Submit(ctx, link.BuildRequest{ - ID: link.NewBuildID(time.Now()), - Repository: repository, Path: path, Ref: ref, - Held: heldBy(ctx), Seats: seatBases(ctx), - DryRun: true, - }, wait) - if err != nil { - return err - } - if result.Failed != "" { - return fmt.Errorf("%s could not build %s:\n%s", result.On, result.Repository, result.Failed) - } - manifest, err := catalogue.ParseManifest(result.Manifest) - if err != nil { - return fmt.Errorf("%s built %s and what came back is not a manifest: %w", - result.On, result.Repository, err) - } - body, err := json.MarshalIndent(manifest, "", " ") - if err != nil { - return err - } - fmt.Println(string(body)) - return nil -} - // answers is what the three questions came back with, read once. type answers struct { wrong []inventory.Doing diff --git a/cmd/mesh-controller/dependents_test.go b/cmd/mesh-controller/dependents_test.go index 05c306b1..bc9386a9 100644 --- a/cmd/mesh-controller/dependents_test.go +++ b/cmd/mesh-controller/dependents_test.go @@ -35,16 +35,6 @@ func TestAMergeOfABaseTakesWhatStandsOnItAlong(t *testing.T) { t.Fatalf("%s was rebuilt and stands on nothing that moved (%v)", n, names) } } - // The dependents come in base order when the merge orders them: the runtime, then shop, then - // the plugin that stands on shop. - ordered := orderByBases(append([]inventory.Entry{entry("mesh-tools")}, got...), against) - pos := map[string]int{} - for i, e := range ordered { - pos[e.Manifest.Module] = i - } - if !(pos["mesh-tools"] < pos["shop"] && pos["shop"] < pos["shop-plugin"]) { - t.Fatalf("not in base order: %v", ordered) - } // Nothing moved: nothing follows. if more := dependentsOf(nil, entries, against); len(more) != 0 { t.Fatalf("with nothing moved, %d module(s) were rebuilt", len(more)) diff --git a/cmd/mesh-controller/key_takein.go b/cmd/mesh-controller/key_takein.go index 435cfacb..b418c049 100644 --- a/cmd/mesh-controller/key_takein.go +++ b/cmd/mesh-controller/key_takein.go @@ -57,9 +57,6 @@ func keyBuildID(k builderkeys.Key) string { return fmt.Sprintf("key-%s-%d", part, k.Revision) } -// isKeyBuildID says a recorded build was taken from a builder's key. -func isKeyBuildID(id string) bool { return strings.HasPrefix(id, "key-") } - // readArchive is the archive a key's reference names, read from the artifact store and checked against its digest. A // variable so a test needs no store. var readArchive = func(ctx context.Context, inv *inventory.Inventory, reference string) ([]byte, error) { diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index d3f380b2..e3e346f7 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -73,8 +73,9 @@ func run() error { } switch args[0] { - case "build": - return buildCommand(ctx, args[1:]) + case "build", "rebuild": + // **The hand build goes with the asks** (novox/hq ADR 0501 §5): refused by name, naming the replacement. + return removedByName(args[0]) case "builder": return builderCommand(ctx, args[1:]) case "board": @@ -98,8 +99,6 @@ func run() error { return cancelCommand(ctx, args[1:]) case "clear": return clearCommand(ctx, args[1:]) - case "rebuild": - return rebuildCommand(ctx, args[1:]) case "replay": return replayCommand(ctx, args[1:]) case "kill": @@ -475,3 +474,20 @@ func connectionName(command, verb string) string { } return "mesh-controller command " + command } + +// removedByName refuses a command the controller no longer has, naming what replaced it (novox/hq ADR 0501 §4, §5; +// ADR 0429 §18; ADR 0503 §3; issue 1851, A11, and issue 1318). +func removedByName(command string) error { + replacement := map[string]string{ + "build": "the controller asks no build (ADR 0501 §5): the builder builds every fingerprint the indexer writes, a " + + "module is registered from its done key, and a branch is tried with `try`; a module from a repository the " + + "catalogue does not build is registered at the controller's terminal with `register`", + "rebuild": "the controller asks no build (ADR 0501 §5): a failed key is built again by the builder as a new " + + "revision (ADR 0429 §18), and a walk's Retry waits for it", + "replay --register": "a build is registered only from the builder's done key of a trunk fingerprint (ADR 0501 " + + "§1, §2); `replay` without --register still shows what a build of a recorded commit makes", + "forget-checks": "build-agent makes no check build from R4e (ADR 0503 §3), so there is nothing to let go of", + "delivery-plan": "a delivery's plan is read with `delivery-order` and the checker's verdict (ADR 0503 §3)", + }[command] + return fmt.Errorf("`%s` is removed: %s", command, replacement) +} diff --git a/cmd/mesh-controller/not_asked_id_test.go b/cmd/mesh-controller/not_asked_id_test.go deleted file mode 100644 index 557589f4..00000000 --- a/cmd/mesh-controller/not_asked_id_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "testing" - - "github.com/novox/mesh-controller/internal/chain" -) - -// 1146: a merge's ask that failed is kept under an id the bus takes as a key, for a module at the repository's root -// (".") as for one in a folder. -func TestTheIdOfABuildNotAskedIsAKeyTheBusTakes(t *testing.T) { - for dir, want := range map[string]string{ - ".": "not-asked-0177ea7a", - "modules/mesh-delivery-view": "not-asked-0177ea7a-modules-mesh-delivery-view", - } { - id := notAskedID("0177ea7a00000000", dir) - if id != want { - t.Errorf("notAskedID(%q) = %q, want %q", dir, id, want) - } - if !chain.ValidKey("req." + id) { - t.Errorf("notAskedID(%q) = %q: the bus takes no key req.%s", dir, id, id) - } - } -} diff --git a/cmd/mesh-controller/order_test.go b/cmd/mesh-controller/order_test.go index 9bd36d56..5aa455b1 100644 --- a/cmd/mesh-controller/order_test.go +++ b/cmd/mesh-controller/order_test.go @@ -28,48 +28,6 @@ func stoodOn(edges map[string][]string) map[string][]string { return out } -// A module built before the module it stands on is built against the old one and reports success -// (novox/hq 04-ISSUES/131). So bases come first, however the set arrived. -func TestBasesAreBuiltBeforeWhatStandsOnThem(t *testing.T) { - in := []inventory.Entry{entry("app"), entry("runtime"), entry("other"), entry("base")} - edges := stoodOn(map[string][]string{"app": {"runtime"}, "runtime": {"base"}}) - got := orderByBases(in, edges) - pos := map[string]int{} - for i, e := range got { - pos[e.Manifest.Module] = i - } - if !(pos["base"] < pos["runtime"] && pos["runtime"] < pos["app"]) { - t.Fatalf("bases not first: %v", pos) - } - if len(got) != 4 { - t.Fatalf("an entry was lost or doubled: %d", len(got)) - } - // A base outside the set is not waited for: it is not being rebuilt. - got = orderByBases([]inventory.Entry{entry("app")}, stoodOn(map[string][]string{"app": {"elsewhere"}})) - if len(got) != 1 { - t.Fatalf("a dependency outside the set changed the set: %v", got) - } -} - -// A module registered from its manifest and never built still names its bases there; once built, -// the recorded edge is what says so. Both are read, and a module never stands on itself. -func TestWhatStandsOnAModuleIsReadFromItsBuildOrItsManifest(t *testing.T) { - built := entry("gitea") - edges := stoodOn(map[string][]string{"gitea": {"mesh-tools"}}) - if !standsOnModule(built, "mesh-tools", edges) { - t.Fatal("a recorded edge was not read") - } - if standsOnModule(built, "gitea", edges) || standsOnModule(built, "postgres", edges) { - t.Fatal("an edge was invented") - } - fresh := inventory.Entry{Manifest: catalogue.Manifest{Module: "plex", Build: &catalogue.Build{ - On: []catalogue.BuildsOn{{Arg: "RUNTIME_BASE", Module: "mesh-tools", Artifact: "runtime"}}, - }}} - if !standsOnModule(fresh, "mesh-tools", nil) { - t.Fatal("a manifest's own base was not read") - } -} - // A merge names a repository the way the forge does; a source is recorded the way a build was // asked for. The two meet on owner/repo and branch, whichever form the record took. func TestAMergeMatchesTheSourcesBuiltFromIt(t *testing.T) { diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 86dc803e..6cca81db 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -442,101 +442,6 @@ func retryWords(p inventory.Plan, failed, asked, waiting []string) (note, said s return note, said } -// joinAPlan asks a module again for the plan that holds it unbuilt or failed, if one does: open plans -// first, then the most recent failed one that nothing newer supersedes. Says whether it joined one. -func joinAPlan(ctx context.Context, open *stores, module string) (bool, string, error) { - inv := open.inventory - release, err := inv.HoldPlans(ctx, true) - if err != nil { - return false, "", err - } - defer release() - openPlans, err := inv.OpenPlans(ctx) - if err != nil { - return false, "", err - } - recent, err := inv.RecentPlans(ctx, 20) - if err != nil { - return false, "", err - } - p, found := planHolding(module, openPlans, recent) - if !found { - return false, "", nil - } - // **A walk that waits for its delivery's word asks nothing** (novox/hq ADR 0239, issue 754): a rebuild does not - // join it, and its outcome is not the walk's. The walk asks its own tier once it has its word. - if p.Waiting() { - return false, fmt.Sprintf("not joining %s (%s at %s): it waits for %s's word, so this build is not its "+ - "outcome; once let go, the walk asks %s itself", p.ID, p.Repository, short(p.Commit), p.Delivery.Awaits, - module), nil - } - entries, err := inv.Catalogued(ctx) - if err != nil { - return false, "", err - } - byName := map[string]inventory.Entry{} - for _, e := range entries { - byName[e.Manifest.Module] = e - } - was := p.State - askModule(ctx, &p, module, byName) - s := p.Modules[module] - resumed(&p, fmt.Sprintf("tier %d: %s rebuilt by hand", p.Tier, module)) - if err := inv.SavePlan(ctx, &p); err != nil { - return false, "", err - } - if s.State == "" && s.BusWaitFrom != nil { - return true, busWaitWords(p, []string{module}), nil - } - if s.State != "asked" { - return true, "", fmt.Errorf("%s could not be asked for %s: %s", module, p.ID, s.Why) - } - said := fmt.Sprintf("rebuild asked as %s, joining %s at tier %d (%s at %s): the plan takes this build as %s's outcome", - s.Build, p.ID, p.Tier, p.Repository, short(p.Commit), module) - switch { - case was == inventory.PlanFailed && p.State == inventory.PlanBuilding: - said += "; the plan had failed and builds again from this tier" - case was == inventory.PlanFailed: - said += "; the plan stays failed while " + strings.Join(failedIn(p), ", ") + " failed too — `plans retry " + p.ID + "` asks them" - } - return true, said, nil -} - -// planHolding is the plan a rebuild of a module joins: an open plan whose current tier holds it not -// yet built, else the newest failed plan whose current tier does, and that no open plan of its -// repository supersedes. -func planHolding(module string, openPlans, recent []inventory.Plan) (inventory.Plan, bool) { - holds := func(p inventory.Plan) bool { - if p.Tier >= len(p.Tiers) { - return false - } - for _, m := range p.Tiers[p.Tier] { - if m == module { - s := p.Modules[m] - return s == nil || s.State != "built" - } - } - return false - } - for _, p := range openPlans { - if holds(p) { - return p, true - } - } - sorted := append([]inventory.Plan(nil), recent...) - sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Created.After(sorted[j].Created) }) - for _, p := range sorted { - if p.State != inventory.PlanFailed || !holds(p) { - continue - } - if _, superseded := newerOpenPlan(p, openPlans); superseded { - continue - } - return p, true - } - return inventory.Plan{}, false -} - // retryRollouts sends each module whose rollout stopped to the machines it was first sent to, again, // records that send as the first anew, and sets the plan rolling: from there it goes on as the plan // would have — the rest sent once those report they applied it, the next tier after (ADR 0218). diff --git a/cmd/mesh-controller/queue.go b/cmd/mesh-controller/queue.go index f82e2919..a11ade45 100644 --- a/cmd/mesh-controller/queue.go +++ b/cmd/mesh-controller/queue.go @@ -477,89 +477,6 @@ func clearQueue(ctx context.Context, js *broker.JetStream, open *stores, seat st return b.String(), nil } -// rebuildCommand asks the module's current source again — or, given a build's id, that build's -// repository, path and ref — under a new id. -func rebuildCommand(ctx context.Context, args []string) error { - if len(args) != 1 { - return errors.New("rebuild ") - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - inv := open.inventory - entries, err := inv.Catalogued(ctx) - if err != nil { - return err - } - var source buildSource - var path, ref, module string - if b, found, err := inv.BuildByID(ctx, args[0]); err != nil { - return err - } else if found { - source, module = sourceOfBuild(b, entries) - path, ref = b.Path, b.Ref - // **A build at a commit is rebuilt at what its module follows now** (novox/hq ADR 0219, issue - // 219): asked now, a rebuild of an old commit would be the newest ask of the module and roll - // that commit out over everything since. Building that commit again is `replay`, which says - // what it would do and refuses to register it while anything newer is asked or registered. - if e, known := entryNamed(entries, module); known && ref != "" && followedBranch(ref) == "" { - ref = followedBranch(e.Source.Ref) - follows := ref - if follows == "" { - follows = "the repository's default branch" - } - fmt.Printf("%s was built at commit %s; a rebuild asks what %s follows now, %s — `replay %s` "+ - "builds that commit\n", b.ID, short(b.Ref), module, follows, b.ID) - } - } else if e, known := entryNamed(entries, args[0]); known { - // As a plan asks it: the branch it follows, never a commit a build once named (issue 215). - source = buildSource{Repository: e.Source.Repository, Seat: e.Source.Seat} - path, ref, module = e.Source.Path, followedBranch(e.Source.Ref), e.Manifest.Module - } else { - return fmt.Errorf("%s is neither a module the catalogue holds nor a build the mesh recorded", args[0]) - } - - // **A module a plan holds unbuilt, or failed, joins that plan** rather than running beside it: the - // plan would ask it again, or stay failed on an outcome a rebuild has replaced. A walk waiting for its word - // is not joined (novox/hq issue 754): the rebuild runs beside it, and says so. - var beside string - if module != "" { - joined, said, err := joinAPlan(ctx, open, module) - if err != nil { - return err - } - if joined { - fmt.Println(said) - return nil - } - beside = said - } - id, err := askABuild(ctx, source, path, ref) - if err != nil { - return err - } - recordAsked(ctx, inventory.BuildRequest{ID: id, Repository: source.Repository, Seat: source.Seat, Path: path, - Ref: ref, For: "rebuild"}) - if beside != "" { - fmt.Printf("rebuild asked as %s, beside the walk: %s\n", id, beside) - return nil - } - fmt.Printf("rebuild asked as %s\n", id) - return nil -} - -// entryNamed is the catalogue's entry for a module. -func entryNamed(entries []inventory.Entry, name string) (inventory.Entry, bool) { - for _, e := range entries { - if e.Manifest.Module == name { - return e, true - } - } - return inventory.Entry{}, false -} - // sourceOfBuild is where a recorded build's repository is asked from: the catalogued module's own // source when the build is of one — on its seat, so the outcome registers it as the mesh records it // (ADR 0111) — else the repository as it was cloned. @@ -576,14 +493,17 @@ func sourceOfBuild(b inventory.Build, entries []inventory.Entry) (buildSource, s // replayCommand asks a recorded build's repository and path again at the commit it built. func replayCommand(ctx context.Context, args []string) error { set := flag.NewFlagSet("replay", flag.ContinueOnError) - register := set.Bool("register", false, "register what it builds, as any build is") - older := set.Bool("older", false, "with --register: even though a newer build of the module is registered") + register := set.Bool("register", false, "removed (novox/hq ADR 0501 §5): refused by name") positionals, err := parseAround(set, args) if err != nil { return err } + // **Registering a replay goes with the asks** (novox/hq ADR 0429 §18, ADR 0501 §5): refused by name. + if *register { + return removedByName("replay --register") + } if len(positionals) != 1 { - return errors.New("replay [--register [--older]]") + return errors.New("replay ") } open, err := openStores(ctx) if err != nil { @@ -603,86 +523,15 @@ func replayCommand(ctx context.Context, args []string) error { return err } source, module := sourceOfBuild(b, entries) - var history []inventory.Build - if module != "" { - if history, err = inv.Builds(ctx, module, 100); err != nil { - return err - } + if b.Commit == "" { + return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+ + "nothing to replay", b.ID) } - // What is asked of the module and not yet answered, when the replay would be registered. - var outstanding []string - if *register { - js, err := dialTheBus() - if err != nil { - return err - } - q, err := link.ReadQueue(ctx, js, buildSeatHeld(ctx)) - js.Close() - if err != nil { - return err - } - plans, err := inv.OpenPlans(ctx) - if err != nil { - return err - } - outstanding = outstandingFor(module, b.Repository, b.Path, q, plans) - } - if err := replayRefusal(b, module, history, *register, *older, outstanding); err != nil { - return err - } - asker := "" - if *register { - asker = "replay" - } - id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, !*register, asker) + id, err := buildOneAsked(ctx, source, b.Path, b.Commit, 0, true, "") if err != nil { return err } - fmt.Println(replaySaid(b, module, id, *register)) - return nil -} - -// replayRefusal is why a replay is not asked, or nothing (novox/hq ADR 0219, issue 207). -// -// A replay re-asks a recorded build at the commit it built, under a new id — and an id is when it -// was asked, which is what orders builds of one module (issue 219). So a registered replay of an -// older commit is newer than everything since, and would roll that older commit out as the module's -// current version: what issue 207 recorded happening by accident. It is therefore a dry run unless -// --register says otherwise, and --register is refused when a newer build of a different commit is -// registered, unless --older says that is the point. -// -// **And refused while anything newer is outstanding**, --older or not: an ask of the module in the -// queue, or an open plan holding it unbuilt. Asked now, the replay would be newer than those asks, -// and their builds — made from newer source — would be recorded and never registered (issue 219 -// orders by ask), the plan waiting on them sending the older commit instead. -func replayRefusal(b inventory.Build, module string, history []inventory.Build, register, older bool, - outstanding []string) error { - if b.Commit == "" { - return fmt.Errorf("%s recorded no commit — it failed before it knew what it was building, so there is "+ - "nothing to replay; `rebuild %s` asks its repository, path and ref again", b.ID, b.ID) - } - if older && !register { - return errors.New("--older only says what --register may do; a dry run registers nothing") - } - if register && len(outstanding) > 0 { - return fmt.Errorf("%s is asked and not yet answered — %s — and a registered replay asked now would "+ - "replace what those build (novox/hq issue 219). Wait for them, or `replay %s` without --register to look", - orNone(module), strings.Join(outstanding, "; "), b.ID) - } - if !register || older { - return nil - } - for _, h := range history { - if h.ID == b.ID || !h.Worked() || h.Commit == b.Commit { - continue - } - if h.AskedOrAt().After(b.AskedOrAt()) { - return fmt.Errorf("a newer build of %s is registered — %s, from %s — and registering a replay of %s "+ - "would roll that older commit out as %s's current version (novox/hq issue 207). `replay %s` "+ - "without --register looks at it; --register --older registers it anyway", - module, h.ID, short(h.Commit), short(b.Commit), module, b.ID) - } - } + fmt.Println(replaySaid(b, module, id, false)) return nil } @@ -822,32 +671,3 @@ func holdersAmong(entries []inventory.Entry, seat string) []string { sort.Strings(out) return out } - -// outstandingFor is everything asked of a module and not yet answered: its asks in the build queue, -// by repository and path, and every open plan holding it not yet built. -func outstandingFor(module, repository, path string, q link.Queue, plans []inventory.Plan) []string { - var out []string - for _, a := range q.Asks { - if repositoryMatches(a.Repository, repository) && a.Path == path { - out = append(out, fmt.Sprintf("%s %s in the queue", a.ID, a.State)) - } - } - if module == "" { - return out - } - for _, p := range plans { - if !p.Open() { - continue - } - for _, tier := range p.Tiers { - for _, m := range tier { - if m == module { - if st := p.Modules[m]; st == nil || st.State != "built" { - out = append(out, fmt.Sprintf("%s holds it not yet built", p.ID)) - } - } - } - } - } - return out -} diff --git a/cmd/mesh-controller/queue_integration_test.go b/cmd/mesh-controller/queue_integration_test.go index c0cb98f3..5bb334f2 100644 --- a/cmd/mesh-controller/queue_integration_test.go +++ b/cmd/mesh-controller/queue_integration_test.go @@ -71,44 +71,6 @@ func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) { } } -func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := rebuildCommand(ctx, []string{"a"}); err != nil { - t.Fatal(err) - } - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" { - t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"]) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - // b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone. - if err := rebuildCommand(ctx, []string{"b"}); err != nil { - t.Fatal(err) - } - if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil { - t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"]) - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } -} - func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) { t.Helper() url := testbus.URL(t) @@ -425,30 +387,6 @@ func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) { } } -func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - var refs []string - asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it - was := askABuild - askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { - refs = append(refs, ref) - return was(c, source, path, ref) - } - if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a", - Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil { - t.Fatal(err) - } - if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil { - t.Fatal(err) - } - if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 { - t.Fatalf("asked %v at %v", *asked, refs) - } -} - func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) { js, ask := aBuildQueue(t) open := aMesh(t) diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index 43219812..da2fea1e 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -101,89 +101,6 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) { // `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone. -// A failed plan an open plan of its repository supersedes is not joined: the open one holds the module. -func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) { - at := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - failed := inventory.Plan{ID: "plan-old", Repository: "novox/a", Branch: "main", Created: at, State: inventory.PlanFailed, - Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "failed"}}} - newer := inventory.Plan{ID: "plan-new", Repository: "novox/a", Branch: "main", Created: at.Add(time.Hour), - State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}} - if _, found := planHolding("a", []inventory.Plan{newer}, []inventory.Plan{newer, failed}); found { - t.Fatal("joined a failed plan a newer open one supersedes") - } - if p, found := planHolding("a", nil, []inventory.Plan{failed}); !found || p.ID != "plan-old" { - t.Fatalf("did not join the failed plan holding it: %v %s", found, p.ID) - } - built := failed - built.Modules = map[string]*inventory.PlanModule{"a": {State: "built"}} - if _, found := planHolding("a", nil, []inventory.Plan{built}); found { - t.Fatal("joined a plan that has the module built") - } -} - -// replay is a dry run unless registered, and registering an older commit than one registered since -// is refused unless --older says it is meant (novox/hq issue 207). -func TestReplayRefusesToRollAnOlderCommitOutUnlessToldTo(t *testing.T) { - asked := time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC) - old := inventory.Build{ID: "build-old", Module: "a", Commit: "0ldc0mm1t", Asked: asked} - newer := inventory.Build{ID: "build-new", Module: "a", Commit: "n3wc0mm1t", Asked: asked.Add(time.Hour)} - history := []inventory.Build{newer, old} - - if err := replayRefusal(old, "a", history, false, false, nil); err != nil { - t.Errorf("a dry run was refused: %v", err) - } - err := replayRefusal(old, "a", history, true, false, nil) - if err == nil || !strings.Contains(err.Error(), "build-new") || !strings.Contains(err.Error(), "--older") { - t.Errorf("registering an older commit than the one registered was not refused: %v", err) - } - if err := replayRefusal(old, "a", history, true, true, nil); err != nil { - t.Errorf("--register --older was refused: %v", err) - } - if err := replayRefusal(newer, "a", history, true, false, nil); err != nil { - t.Errorf("registering the newest build again was refused: %v", err) - } - // A newer build of the same commit, or one that failed, is not a newer version to roll back from. - same := inventory.Build{ID: "build-same", Module: "a", Commit: old.Commit, Asked: asked.Add(2 * time.Hour)} - broken := inventory.Build{ID: "build-broken", Module: "a", Failed: "no", Asked: asked.Add(3 * time.Hour)} - if err := replayRefusal(old, "a", []inventory.Build{broken, same, old}, true, false, nil); err != nil { - t.Errorf("refused for a newer build of the same commit or a failed one: %v", err) - } - if err := replayRefusal(inventory.Build{ID: "build-x", Failed: "clone"}, "", nil, false, false, nil); err == nil { - t.Error("a build that recorded no commit was replayed") - } - if err := replayRefusal(old, "a", history, false, true, nil); err == nil { - t.Error("--older without --register was taken") - } - // **Nothing newer outstanding**, --older or not: an ask of the module in the queue, or an open plan - // holding it unbuilt, would be replaced by a replay asked now (issue 219). - q := link.Queue{Asks: []link.QueuedAsk{ - {ID: "build-queued", Repository: "https://forge.example/novox/a.git", State: link.AskWaiting}, - {ID: "build-elsewhere", Repository: "https://forge.example/novox/b.git", State: link.AskWaiting}, - {ID: "build-other-path", Repository: "https://forge.example/novox/a.git", Path: "sub", State: link.AskWaiting}, - }} - plans := []inventory.Plan{ - {ID: "plan-holds", State: inventory.PlanBuilding, Tiers: [][]string{{"x"}, {"a"}}, Modules: map[string]*inventory.PlanModule{}}, - {ID: "plan-built", State: inventory.PlanRolling, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}}, - {ID: "plan-failed", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}}, - } - outstanding := outstandingFor("a", "novox/a", "", q, plans) - if len(outstanding) != 2 || !strings.Contains(outstanding[0], "build-queued") || !strings.Contains(outstanding[1], "plan-holds") { - t.Fatalf("outstanding: %v", outstanding) - } - if err := replayRefusal(old, "a", history, true, true, outstanding); err == nil || !strings.Contains(err.Error(), "build-queued") { - t.Errorf("registered over an outstanding ask: %v", err) - } - if err := replayRefusal(old, "a", history, false, false, outstanding); err != nil { - t.Errorf("a dry run was refused for what is outstanding: %v", err) - } - if said := replaySaid(old, "a", "build-1", false); !strings.Contains(said, "dry run") || !strings.Contains(said, "--register") { - t.Errorf("a dry replay does not say what it is: %q", said) - } - if said := replaySaid(old, "a", "build-1", true); !strings.Contains(said, "registered") || !strings.Contains(said, "rolled out") { - t.Errorf("a registered replay does not say what it does: %q", said) - } -} - // A plan waiting on builds of a seat whose every holder is paused says so and is not late; a seat // paused on some holders only is not a reason the plan is waiting. func TestAPlanWaitingOnAPausedSeatSaysSoAndIsNotLate(t *testing.T) { @@ -241,9 +158,9 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) { {"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}}, {"clear", nil, []string{"clear"}}, {"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}}, - {"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild", "gitea"}}, + {"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild"}}, {"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}}, - {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register", "--older"}}, + {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register"}}, {"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}}, {"pause", nil, []string{"pause"}}, {"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}}, diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index b7b00d09..c07fe9ba 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -86,6 +86,14 @@ type verbArguments struct { misread []string // arguments the command line read that the schema does not declare: a bug here } +// passOver takes every argument given as read: a removed verb is refused by name whatever was asked of it (novox/hq ADR +// 0501 §5), so no argument of a table of before is refused as passed over in its place. +func (a *verbArguments) passOver() { + for k := range a.given { + a.used[k] = true + } +} + // controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the // arguments are checked against the table compiled beside argvFor, not a row a newer or older build // wrote. @@ -522,21 +530,19 @@ func (a *verbArguments) commandLine() ([]string, error) { } return []string{"clear"}, nil case "rebuild": - if err := need("what"); err != nil { - return nil, err - } - return []string{"rebuild", str("what")}, nil + // Removed (ADR 0501 §5): the command refuses it by name, naming its replacement, whatever was asked. + a.passOver() + return []string{"rebuild"}, nil case "replay": if err := need("id"); err != nil { return nil, err } argv := []string{"replay", str("id")} + // A table of before still offers register: passed on, so the command refuses it by name (ADR 0501 §5). if on("register") { argv = append(argv, "--register") } - if on("older") { - argv = append(argv, "--older") - } + a.passOver() return argv, nil case "pause", "resume": if n := str("node"); n != "" { @@ -1076,34 +1082,9 @@ func (a *verbArguments) commandLine() ([]string, error) { } return []string{"module", "issue", str("module"), "--node", str("node")}, nil case "build": - // The command's three shapes (`build --on`, `build --behind`, `build `), one per - // call: each branch reads only its own argument, so another given beside it is refused as - // passed over rather than dropped. Asked and not waited for, the same as a single build. - if b := str("on"); b != "" { - return []string{"build", "--on", b, "--wait", "0"}, nil - } - if on("behind") { - return []string{"build", "--behind", "--wait", "0"}, nil - } - if a.given["repository"] == "" { - // No shape named: the command says its usage, which names all three — the answer the - // caller needs, and the same as `rotate` given neither of its shapes. - return []string{"build"}, nil - } - // Not waited for: a tool call cannot hold a connection for the minutes a build takes; the - // daemon takes the outcome in when it comes and the id follows the build (issue 176). A - // repository given without a scheme is a path on the forge holding the git seat. - argv := []string{"build", str("repository"), "--wait", "0"} - if !strings.Contains(str("repository"), "://") && !strings.HasPrefix(str("repository"), "git@") { - argv = append(argv, "--self") - } - if p := str("path"); p != "" { - argv = append(argv, "--path", p) - } - if r := str("ref"); r != "" { - argv = append(argv, "--ref", r) - } - return argv, nil + // Removed (ADR 0501 §5): the command refuses it by name, naming its replacement, whatever was asked. + a.passOver() + return []string{"build"}, nil } return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for", verb, catalogue.ControllerSeatName) diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 361fdc49..49756a89 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "errors" "fmt" "os" @@ -21,66 +22,6 @@ func TestBuildsWithAnIdReadsThatBuildsLog(t *testing.T) { } } -// The build tool takes a repository as a URL or as its path on the forge holding the git seat, and -// says which it was given, so the command reads the path as a seat source rather than handing it to -// git as written (novox/hq issue 176). And it never waits: the id follows the build. -func TestTheBuildToolTellsAForgePathFromAURL(t *testing.T) { - argv, _ := argvFor("build", map[string]any{"repository": "novox/mesh-catalog", "path": "modules/x"}) - if line := strings.Join(argv, " "); !strings.Contains(line, "--self") || !strings.Contains(line, "--wait 0") { - t.Fatalf("a forge path is a seat source, not waited for; got %q", line) - } - argv, _ = argvFor("build", map[string]any{"repository": "https://example.tld/o/r.git"}) - if line := strings.Join(argv, " "); strings.Contains(line, "--self") { - t.Fatalf("a URL is cloned as given; got %q", line) - } -} - -// The build tool has the command's three shapes: a repository, a base whose dependents are rebuilt -// (`--on`), or everything behind its source (`--behind`) — each asked, not waited for, one per call. -// Given neither, the command line is what it was: nothing new reaches it. -func TestTheBuildToolTakesOnAndBehind(t *testing.T) { - cases := []struct { - args map[string]any - want string - }{ - {map[string]any{"on": "mesh-tools"}, "build --on mesh-tools --wait 0"}, - {map[string]any{"behind": "true"}, "build --behind --wait 0"}, - {map[string]any{"behind": true}, "build --behind --wait 0"}, - {map[string]any{"repository": "novox/x"}, "build novox/x --wait 0 --self"}, - {map[string]any{"repository": "novox/x", "behind": "false"}, "build novox/x --wait 0 --self"}, - {map[string]any{"repository": "novox/x", "path": "modules/x", "ref": "main"}, - "build novox/x --wait 0 --self --path modules/x --ref main"}, - } - for _, c := range cases { - argv, err := argvFor("build", c.args) - if err != nil { - t.Errorf("%v: %v", c.args, err) - continue - } - if got := strings.Join(argv, " "); got != c.want { - t.Errorf("%v became %q, not %q", c.args, got, c.want) - } - } - // Naming no shape runs the bare command, which answers its usage naming all three. - if argv, err := argvFor("build", map[string]any{}); err != nil || strings.Join(argv, " ") != "build" { - t.Errorf("a build naming nothing became %v, %v", argv, err) - } - // One shape per call: a second beside it is refused, not dropped, and so is half of one. - for _, refused := range []map[string]any{ - {"path": "modules/x"}, - {"on": "mesh-tools", "repository": "novox/x"}, - {"on": "mesh-tools", "behind": "true"}, - {"behind": "true", "repository": "novox/x"}, - {"behind": "true", "path": "modules/x"}, - {"on": "mesh-tools", "ref": "main"}, - {"behind": "yes"}, - } { - if argv, err := argvFor("build", refused); err == nil { - t.Errorf("%v was accepted as %v", refused, argv) - } - } -} - // `rotate` is one verb with two shapes (ADR 0114, issue 180): a pair credential by provision, or a // module's own secret by machine, module and name. func TestRotateTakesAProvisionOrAnOwnSecret(t *testing.T) { @@ -171,9 +112,35 @@ func TestActsDoNotBlockTheCall(t *testing.T) { if strings.Join(argv, " ") != "push one --wait 0s --why w" || !answersFirst(argv) { t.Fatalf("a named push with no setting waits, or holds its caller: %v", argv) } - argv, _ = argvFor("build", map[string]any{"repository": "novox/x", "path": "modules/x"}) - if strings.Join(argv, " ") != "build novox/x --wait 0 --self --path modules/x" { - t.Fatalf("build: %v", argv) +} + +// A11a (ADR 0501 §5; issue 1851): the build, rebuild and replay --register verbs reach a command that refuses them by +// name, naming the replacement, whatever a table of before asked of them; the command lines refuse the same. +func TestTheRemovedVerbsAreRefusedByName(t *testing.T) { + for _, c := range []struct { + verb string + args map[string]any + want string + }{ + {"build", map[string]any{"repository": "novox/x", "path": "modules/x"}, "build"}, + {"build", map[string]any{"on": "mesh-tools"}, "build"}, + {"rebuild", map[string]any{"what": "gitea"}, "rebuild"}, + {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, "replay build-1 --register"}, + } { + argv, err := argvFor(c.verb, c.args) + if err != nil || strings.Join(argv, " ") != c.want { + t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want) + } + } + for _, removed := range []string{"build", "rebuild"} { + err := removedByName(removed) + if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 0501") { + t.Errorf("%s: %v", removed, err) + } + } + err := replayCommand(context.Background(), []string{"build-1", "--register"}) + if err == nil || !strings.Contains(err.Error(), "`replay --register` is removed") { + t.Errorf("replay --register: %v", err) } } diff --git a/cmd/mesh-controller/try.go b/cmd/mesh-controller/try.go index d306975e..7dcd4ed3 100644 --- a/cmd/mesh-controller/try.go +++ b/cmd/mesh-controller/try.go @@ -12,9 +12,6 @@ import ( "strings" "time" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/link" ) @@ -300,35 +297,3 @@ func printTry(a tryAnswer) error { fmt.Println(string(out)) return nil } - -// buildLines is everything the build seat said of one build, as the bus keeps it, in order. -func buildLines(ctx context.Context, id string) ([]link.BuildLine, error) { - js, err := aBus() - if err != nil { - return nil, fmt.Errorf("cannot reach the bus to read a build's log: %w", err) - } - defer js.Close() - subject := link.BuildLogOf("*", id) - sub, err := js.Context().PullSubscribe(subject, "", - nats.BindStream(broker.EventsStream), nats.DeliverAll(), nats.AckNone()) - if err != nil { - return nil, fmt.Errorf("cannot read %s from the bus: %w", subject, err) - } - defer func() { _ = sub.Unsubscribe() }() - var lines []link.BuildLine - for { - batch, err := sub.Fetch(200, nats.MaxWait(2*time.Second)) - if err != nil && !errors.Is(err, nats.ErrTimeout) && !errors.Is(err, context.DeadlineExceeded) { - return nil, fmt.Errorf("reading a build's log: %w", err) - } - for _, msg := range batch { - var line link.BuildLine - if json.Unmarshal(msg.Data, &line) == nil { - lines = append(lines, line) - } - } - if len(batch) < 200 { - return lines, nil - } - } -} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 07a3c964..f2140362 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -404,15 +404,6 @@ func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMo return names, nil } -// notAskedID is the id a merge's build that could not be asked is kept under: a key the bus takes, so a module at the -// repository's root (".") is kept as one in a folder is. -func notAskedID(commit, dir string) string { - if dir == "." || dir == "" { - return "not-asked-" + short(commit) - } - return fmt.Sprintf("not-asked-%s-%s", short(commit), strings.ReplaceAll(strings.Trim(dir, "/"), "/", "-")) -} - // actingOnMerges keeps one merge acted on at a time (novox/hq issue 266). var actingOnMerges sync.Mutex @@ -1085,50 +1076,6 @@ func anyInside(paths []string, dir string) bool { return false } -// orderByBases is the entries with every base before what stands on it: a module whose build stood -// on another's artifact comes after that module. Entries outside the set are not waited for — they -// are not being rebuilt. Stable for what has no order between it. -// -// `against` is what each module's newest build stood on (inventory.BuiltAgainst): the edges are -// derived from builds, not declared, because a recorded manifest no longer carries `build.on`. -func orderByBases(entries []inventory.Entry, against map[string][]string) []inventory.Entry { - inSet := map[string]bool{} - for _, e := range entries { - inSet[e.Manifest.Module] = true - } - var out []inventory.Entry - placed := map[string]bool{} - var place func(e inventory.Entry, seen map[string]bool) - place = func(e inventory.Entry, seen map[string]bool) { - name := e.Manifest.Module - if placed[name] || seen[name] { - return - } - seen[name] = true - for _, base := range entries { - if base.Manifest.Module != name && inSet[base.Manifest.Module] && standsOnModule(e, base.Manifest.Module, against) { - place(base, seen) - } - } - placed[name] = true - out = append(out, e) - } - for _, e := range entries { - place(e, map[string]bool{}) - } - return out -} - -// standsOn is whether anything in the set is built on the named module's artifacts. -func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool { - for _, e := range entries { - if standsOnModule(e, module, against) { - return true - } - } - return false -} - // standsOnModule is whether an entry's build stood on the named module: by what its newest build // recorded it was handed (`artifact-store:///@…`, the module's own artifact), or // — for a module registered from a manifest and not yet built — by the base its manifest names. @@ -1152,6 +1099,16 @@ func standsOnModule(e inventory.Entry, module string, against map[string][]strin return false } +// standsOn is whether anything in the set is built on the named module's artifacts. +func standsOn(entries []inventory.Entry, module string, against map[string][]string) bool { + for _, e := range entries { + if standsOnModule(e, module, against) { + return true + } + } + return false +} + // isHistory is whether a merge made at mergedAt predates the last time the source was seen. A merge // with no time on it is taken as news: refusing it would silence a forge that says less. func isHistory(mergedAt string, seen time.Time) bool { diff --git a/cmd/mesh-controller/walk_from_keys_integration_test.go b/cmd/mesh-controller/walk_from_keys_integration_test.go index d4609105..bc8960c1 100644 --- a/cmd/mesh-controller/walk_from_keys_integration_test.go +++ b/cmd/mesh-controller/walk_from_keys_integration_test.go @@ -69,7 +69,7 @@ func TestAWalkTakesItsTierFromTheBuildersKeysAndSendsIt(t *testing.T) { t.Fatal(err) } for _, m := range []string{"app1", "app2"} { - if current[m].Commit != p.Commit || !isKeyBuildID(p.Modules[m].Build) { + if current[m].Commit != p.Commit || !strings.HasPrefix(p.Modules[m].Build, "key-") { t.Fatalf("A6: %s stands on %+v (%s), not the key's build at %s", m, current[m], p.Modules[m].Build, p.Commit) } } diff --git a/cmd/mesh-controller/walk_said_integration_test.go b/cmd/mesh-controller/walk_said_integration_test.go index 37190421..f53ed71c 100644 --- a/cmd/mesh-controller/walk_said_integration_test.go +++ b/cmd/mesh-controller/walk_said_integration_test.go @@ -206,35 +206,12 @@ func TestAWalkEndedByCancelOrClearIsSaidEnded(t *testing.T) { } } -func TestAWalkRestartedByARebuildIsSaidStarted(t *testing.T) { - open := aMesh(t) - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-rebuild", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.CancelledByHand}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(t.Context(), &failed); err != nil { - t.Fatal(err) - } - if err := asTyped(t, "rebuild", "a"); err != nil { - t.Fatal(err) - } - if started := next(inventory.WalkStarted); started["walk"] != failed.ID { - t.Fatalf("the rebuilt walk's start: %v", started) - } -} - func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { open := aMesh(t) ctx := t.Context() asksRecorded(t) twoTiers(t, open) - for _, again := range [][]string{{"plans", "retry"}, {"rebuild", "a"}} { + for _, again := range [][]string{{"plans", "retry"}} { before := time.Now().UTC().Add(-time.Hour) walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go deleted file mode 100644 index ae027544..00000000 --- a/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go +++ /dev/null @@ -1,63 +0,0 @@ -//go:build integration - -package main - -import ( - "testing" - "time" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -func TestARebuildInTheWaitIsNotTheWalksBuild(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - created := time.Now().UTC().Add(-10 * time.Minute) - plan := inventory.Plan{ID: "plan-754", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - // A rebuild of a, beside the waiting walk: not joined, and its outcome not taken. - joined, said, err := joinAPlan(ctx, open, "a") - if err != nil || joined || said == "" { - t.Fatalf("a rebuild joined a waiting walk: %v %q %v", joined, said, err) - } - planBuilt(ctx, open, "a", "c0ffee", "", time.Now().UTC(), "build-rebuild") - p, _ := open.inventory.PlanByID(ctx, plan.ID) - if s := p.Modules["a"]; s != nil && s.State != "" { - t.Fatalf("the waiting walk took a rebuild as its build: %+v", s) - } - if len(*asked) != 0 { - t.Fatalf("a waiting walk asked %v", *asked) - } - - // Let go: both are asked, and their outcomes complete the tier. - if _, err := letGo(ctx, open.inventory, plan.ID, "operator", "test"); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, _ = open.inventory.PlanByID(ctx, plan.ID) - for _, m := range []string{"a", "b"} { - if s := p.Modules[m]; s == nil || s.State != "asked" || s.Build == "" { - t.Fatalf("let go, %s is %+v", m, s) - } - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } - for _, m := range []string{"a", "b"} { - id := p.Modules[m].Build - at, _ := link.BuildAskedAt(id) - planBuilt(ctx, open, m, "c0ffee", "", at, id) - } - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Open() && p.Tier == 0 { - t.Fatalf("the walk did not get past its tier: %s at %d, %q", p.State, p.Tier, p.Note) - } -} From b71425867f520cabe4efd1174956769bd596a05d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:11:50 +0200 Subject: [PATCH 13/38] =?UTF-8?q?Register=20a=20module=20from=20a=20reposi?= =?UTF-8?q?tory=20the=20catalogue=20does=20not=20build=20at=20the=20contro?= =?UTF-8?q?ller's=20terminal=20alone,=20from=20its=20done=20trunk=20key,?= =?UTF-8?q?=20naming=20exactly=20its=20repository=20and=20path=20(issue=20?= =?UTF-8?q?1851,=20A11b,=20ADR=200501=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/main.go | 2 + cmd/mesh-controller/register.go | 76 ++++++++++++++++++++++++++++ cmd/mesh-controller/register_test.go | 38 ++++++++++++++ cmd/mesh-controller/seatverbs.go | 3 ++ 4 files changed, 119 insertions(+) create mode 100644 cmd/mesh-controller/register.go create mode 100644 cmd/mesh-controller/register_test.go diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index e3e346f7..e56bcac1 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -73,6 +73,8 @@ func run() error { } switch args[0] { + case "register": + return registerCommand(ctx, args[1:]) case "build", "rebuild": // **The hand build goes with the asks** (novox/hq ADR 0501 §5): refused by name, naming the replacement. return removedByName(args[0]) diff --git a/cmd/mesh-controller/register.go b/cmd/mesh-controller/register.go new file mode 100644 index 00000000..1c7a727b --- /dev/null +++ b/cmd/mesh-controller/register.go @@ -0,0 +1,76 @@ +package main + +import ( + "context" + "errors" + "fmt" + "strings" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" +) + +// The terminal's exception becomes a registration, not a build (novox/hq ADR 0501 §4; issue 1851, A11b). +// +// `register / `, at the controller's terminal alone: the operator names the module, +// its repository and its path, and the controller registers it from the builder's done key of the fingerprint the +// indexer's trunk value of that module names — the registration a build asked at the terminal used to make for a +// repository the catalogue does not build (ADR 0432 §3, ADR 0420 §7). No verb reaches it (terminalOnlyCommands), and it +// asks no build: a key not done is waited for, and said. +func registerCommand(ctx context.Context, args []string) error { + if len(args) != 3 { + return errors.New("register / (the path \".\" for a module at the repository's root)") + } + if !startedAtTheTerminal() { + return terminalRefusal("register is run at the controller's terminal only, never through a verb: it registers " + + "a module from a repository the catalogue does not build, and whoever may call a verb includes agents " + + "(novox/hq ADR 0266, ADR 0501 §4). Nothing was done") + } + module, repository, path := args[0], strings.Trim(args[1], "/"), strings.Trim(args[2], "/") + if path == sdkdelivery.RootPath { + path = "" + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return err + } + defer closeKeys() + trunk, err := keys.ModuleAt(ctx, module, "") + if errors.Is(err, builderkeys.ErrNoValue) { + return fmt.Errorf("the indexer holds no trunk value of %s: a module is registered from the builder's key of a "+ + "merged trunk, and %s has none yet. Nothing was registered", module, module) + } + if err != nil { + return err + } + if err := sameModuleSource(module, repository, path, trunk); err != nil { + return err + } + manifest, kept, err := takeInKey(ctx, open.inventory, keys, keyTake{Module: module, Trunk: trunk, + Ask: registrationAsk{Terminal: true}}) + if errors.Is(err, errKeyWaits) { + return fmt.Errorf("%s is not registered yet, and nothing was asked: %v. Run register again once its key is done", + module, err) + } + if err != nil { + return err + } + fmt.Printf("%s %s registered from the builder's key %s (%s at %.8s), as asked at the controller's terminal; "+ + "assigned nowhere\n", manifest.Module, manifest.Version, short(kept.SourceFingerprint), repository, kept.Commit) + return nil +} + +// sameModuleSource refuses a registration whose repository or path is not the one the indexer's trunk value names: the +// operator names exactly what is registered (ADR 0501 §4). +func sameModuleSource(module, repository, path string, trunk sdkdelivery.Module) error { + if !strings.EqualFold(trunk.Repository, repository) || keyPath(trunk) != path { + return fmt.Errorf("the indexer holds %s in %s at %q, not in %s at %q: the registration names exactly what it "+ + "registers. Nothing was registered", module, trunk.Repository, orRoot(keyPath(trunk)), repository, orRoot(path)) + } + return nil +} diff --git a/cmd/mesh-controller/register_test.go b/cmd/mesh-controller/register_test.go new file mode 100644 index 00000000..99b1e45f --- /dev/null +++ b/cmd/mesh-controller/register_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" +) + +// A11b (ADR 0501 §4): the terminal's registration names exactly the module's repository and path the indexer holds, +// and no verb reaches it. +func TestTheTerminalsRegistrationNamesItsSourceExactlyAndNoVerbReachesIt(t *testing.T) { + trunk := sdkdelivery.Module{Repository: "someone/app", Path: sdkdelivery.RootPath, Commit: "c0ffee"} + if err := sameModuleSource("app", "someone/app", "", trunk); err != nil { + t.Fatalf("the source the indexer holds is refused: %v", err) + } + if err := sameModuleSource("app", "someone/other", "", trunk); err == nil || !strings.Contains(err.Error(), "someone/app") { + t.Fatalf("another repository is taken: %v", err) + } + if err := sameModuleSource("app", "someone/app", "modules/app", trunk); err == nil { + t.Fatal("another path is taken") + } + // The generic verb only reads; and register is the terminal's only, whatever reads. + if _, err := argvFor("command", map[string]any{"command": "register app someone/app ."}); err == nil { + t.Fatal("a verb reaches the terminal's registration") + } + var refused *heldAtTheTerminal + if err := terminalOnly([]string{"register", "app", "someone/app", "."}); !errors.As(err, &refused) { + t.Fatalf("register is not the terminal's only: %v", err) + } + t.Setenv(verbVar, "command") + var held *heldAtTheTerminal + if err := registerCommand(context.Background(), []string{"app", "someone/app", "."}); !errors.As(err, &held) { + t.Fatalf("register through a verb: %v", err) + } +} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index c07fe9ba..46485477 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -1784,6 +1784,9 @@ var terminalOnlyCommands = map[string]string{ "broker": "the bus's accounts", "api": "the controller's API keys", "licence": "the licences' secrets", + // The terminal's exception of ADR 0501 §4: a module from a repository the catalogue does not build, registered + // from its done key. + "register": "a module's registration from a repository the catalogue does not build", } // givenAtTheDesk is exactly the line the `give` and `secret-ask` verbs compose, and nothing beside it: `secret ask From 1963d84a7a453436e8ada54c500310ceb2b13899 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:13:17 +0200 Subject: [PATCH 14/38] =?UTF-8?q?Refuse,=20in=20the=20one=20function=20eve?= =?UTF-8?q?ry=20ask=20goes=20through,=20an=20ask=20of=20build-agent=20for?= =?UTF-8?q?=20any=20module=20but=20the=20builder's=20own=20(issue=201851,?= =?UTF-8?q?=20A12,=20ADR=200453=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/builders_own_test.go | 61 ++++++++++++++++++++++++ cmd/mesh-controller/release_plan.go | 49 +++++++++++++++++++ 2 files changed, 110 insertions(+) create mode 100644 cmd/mesh-controller/builders_own_test.go diff --git a/cmd/mesh-controller/builders_own_test.go b/cmd/mesh-controller/builders_own_test.go new file mode 100644 index 00000000..3c2d717b --- /dev/null +++ b/cmd/mesh-controller/builders_own_test.go @@ -0,0 +1,61 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" +) + +// A12 (ADR 0453 §5, ADR 0498 §7; issue 1851): the one function every ask goes through refuses any module but the +// builder's own, naming the builder's key it is taken from instead; the builder's own module passes the refusal. +func TestBuildAgentIsAskedForTheBuildersOwnModuleAlone(t *testing.T) { + was := askedModuleOf + t.Cleanup(func() { askedModuleOf = was }) + askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) { + return strings.TrimPrefix(path, "modules/"), nil + } + _, err := askABuild(t.Context(), buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, "modules/sudo", "main") + if !errors.Is(err, errNotTheBuildersOwn) || !strings.Contains(err.Error(), "sudo is taken from the builder's done key") || + !strings.Contains(err.Error(), "mesh-delivery-builder_builds") { + t.Fatalf("an ask of another module: %v", err) + } + if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/mesh-delivery", Seat: "git"}, + "modules/mesh-delivery-builder"); err != nil { + t.Fatalf("the builder's own module is refused: %v", err) + } + askedModuleOf = func(context.Context, buildSource, string) (string, error) { return "", errors.New("no store") } + if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x"}, ""); !errors.Is(err, errNotTheBuildersOwn) { + t.Fatalf("an ask whose module cannot be read went on: %v", err) + } +} + +// A12: a walk of the builder's own module and one other asks build-agent for the first alone; the other is taken from +// its key. +func TestAWalkAsksBuildAgentForTheBuildersOwnModuleAlone(t *testing.T) { + var asked []string + was := askABuild + askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) { + asked = append(asked, path) + return "build-1", nil + } + t.Cleanup(func() { askABuild = was }) + p, byName := aWalkOfOne("sudo") + p.Tiers = [][]string{{"mesh-delivery-builder", "sudo"}} + byName["mesh-delivery-builder"] = byName["sudo"] + e := byName["mesh-delivery-builder"] + e.Source.Repository, e.Source.Path = "novox/mesh-delivery", "modules/mesh-delivery-builder" + byName["mesh-delivery-builder"] = e + for _, m := range p.Tiers[0] { + askModule(t.Context(), p, m, byName) + } + if len(asked) != 1 || asked[0] != "modules/mesh-delivery-builder" { + t.Fatalf("build-agent was asked for %v, want the builder's own module alone", asked) + } + if s := p.Modules["mesh-delivery-builder"]; s.Key || s.Build != "build-1" { + t.Fatalf("the builder's own module is not asked: %+v", s) + } + if s := p.Modules["sudo"]; !s.Key || s.Build != "" { + t.Fatalf("the other module is not taken from its key: %+v", s) + } +} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index 8cc11981..1238d769 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -6,6 +6,7 @@ import ( "errors" "flag" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "os" "slices" "sort" @@ -410,11 +411,59 @@ var askABuild func(ctx context.Context, source buildSource, path, ref string) (s func init() { askABuild = func(ctx context.Context, source buildSource, path, ref string) (string, error) { + // **Only the builder's own module is asked of build-agent** (novox/hq ADR 0453 §5, ADR 0498 §7; issue 1851, + // A12): refused here, in the one function every ask goes through, whoever calls it. + if err := onlyTheBuildersOwn(ctx, source, path); err != nil { + return "", err + } // Kept by each asker with its own name once the ask is made (novox/hq issue 325). return buildOneAsked(ctx, source, path, ref, 0, false, "") } } +// errNotTheBuildersOwn says an ask of build-agent was refused: from R4e build-agent builds the builder's own module +// alone, and every other is taken from the builder's done key. +var errNotTheBuildersOwn = errors.New("build-agent is asked for the builder's own module alone (ADR 0453 §5)") + +// askedModuleOf is the module the catalogue builds from a source and path, empty when it builds none from it. A variable +// so a test needs no store. +var askedModuleOf = func(ctx context.Context, source buildSource, path string) (string, error) { + open, err := openStores(ctx) + if err != nil { + return "", err + } + defer open.Close() + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + return "", err + } + forms := newSourceForms(ctx, open.inventory) + for _, e := range entries { + if forms.same(e.Source.Repository, e.Source.Seat, source.Repository, source.Seat) && + strings.Trim(e.Source.Path, "/") == strings.Trim(path, "/") { + return e.Manifest.Module, nil + } + } + return "", nil +} + +// onlyTheBuildersOwn refuses an ask of any module but the builder's own, naming the key it is taken from instead. +func onlyTheBuildersOwn(ctx context.Context, source buildSource, path string) error { + module, err := askedModuleOf(ctx, source, path) + if err != nil { + return fmt.Errorf("%w: which module %s at %q is cannot be read, so nothing is asked: %v", errNotTheBuildersOwn, + source, orRoot(path), err) + } + if isBuildersOwn(module) { + return nil + } + if module == "" { + module = fmt.Sprintf("%s at %q", source, orRoot(path)) + } + return fmt.Errorf("%w: %s is taken from the builder's done key of its fingerprint in %s (ADR 0501), and no build "+ + "of it is asked", errNotTheBuildersOwn, module, sdkdelivery.BuilderBuildsBucket) +} + // pullsOfWalk are the pull requests of the merges a walk answers, each once: what its end lets go of (forgetChecksOfWalk) // and what a check of one was (same_program.go). func pullsOfWalk(p *inventory.Plan) []link.PullRef { From a8387ebad8a001b58424e00772f5cd14797166d9 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:15:13 +0200 Subject: [PATCH 15/38] Name, before the switch, which modules it moves: the held build's digests against the builder's done key, images and packages said as what cannot be compared (issue 1851, A13) --- .../key_takein_integration_test.go | 27 +++ cmd/mesh-controller/main.go | 2 + cmd/mesh-controller/seatverbs.go | 2 + cmd/mesh-controller/switch_list.go | 204 ++++++++++++++++++ cmd/mesh-controller/switch_list_test.go | 46 ++++ 5 files changed, 281 insertions(+) create mode 100644 cmd/mesh-controller/switch_list.go create mode 100644 cmd/mesh-controller/switch_list_test.go diff --git a/cmd/mesh-controller/key_takein_integration_test.go b/cmd/mesh-controller/key_takein_integration_test.go index 1b68c58d..0c5d0e70 100644 --- a/cmd/mesh-controller/key_takein_integration_test.go +++ b/cmd/mesh-controller/key_takein_integration_test.go @@ -222,3 +222,30 @@ func TestAKeyWithTheHeldBuildsDigestsIsNoMove(t *testing.T) { t.Fatalf("A9: sudo is registered as %q, not with the held build's artifacts", shelf["sudo"].Version) } } + +// A13 against the store: the switch's list reads the build the mesh holds of each module and the builder's key of its +// trunk fingerprint, and names each module unmoved, moved or not known. +func TestTheSwitchsListReadsTheHeldBuildsAndTheKeys(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + keys, trunk := aKeyFor("sudo", "1", fpOf('b'), "novox/mesh-catalog", "modules/sudo", "c0ffee0123456789") + k := keys.keys["sudo"] + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-sudo-held", Module: "sudo", Repository: "novox/mesh-catalog", + Commit: "c0ffee0123456789", Made: []inventory.Artifact{{Name: "a", Kind: "archive", + Reference: k.Build.Artifacts[0].Reference}}}); err != nil { + t.Fatal(err) + } + reads := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@": trunk}, keys: map[string]builderkeys.Key{"sudo": k}} + lines, err := switchList(ctx, inv, reads) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, l := range lines { + got[l.Module] = l.Verdict + } + if got["sudo"] != switchUnmoved || got["mesh-host"] != switchUnknown { + t.Fatalf("A13: the switch's list is %+v", lines) + } +} diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index e56bcac1..9de2b094 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -73,6 +73,8 @@ func run() error { } switch args[0] { + case "switch": + return switchCommand(ctx, args[1:]) case "register": return registerCommand(ctx, args[1:]) case "build", "rebuild": diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 46485477..3b86f5e3 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -1714,6 +1714,8 @@ var commandReadForms = map[string]func(rest []string) bool{ "status": flagsOnly, "version": flagsOnly, "help": flagsOnly, "seats": flagsOnly, "healers": flagsOnly, "hand-acts": flagsOnly, "durations": flagsOnly, "collection": flagsOnly, "images": flagsOnly, "artifacts": flagsOnly, "data": flagsOnly, "builds": flagsOnly, "queue": flagsOnly, + // The switch's list reads the builder's keys and the builds held, and moves nothing (issue 1851, A13). + "switch": flagsOnly, // `plan ` previews a node's declaration; it sends nothing. "plan": func([]string) bool { return true }, // `plans` lists and `plans ` shows one; `plans stop|close|go` acts. diff --git a/cmd/mesh-controller/switch_list.go b/cmd/mesh-controller/switch_list.go new file mode 100644 index 00000000..b3ac0ac5 --- /dev/null +++ b/cmd/mesh-controller/switch_list.go @@ -0,0 +1,204 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The switch's list (novox/hq ADR 0453 §6, ADR 0499; issue 1851, A13): which modules the switch to the builder's keys +// moves, read before R4e is proposed and while it is delivered. For each module the catalogue builds, the held build's +// artifacts are compared by digest with the builder's done key of the fingerprint the indexer's trunk value names: +// every digest the same is no move (the standing build, A9); one that differs moves. An image and a package are named +// with why they cannot be compared before the trunk builds them: an image of build-agent and one of mesh-build-image +// are not one byte-for-byte build, and a candidate's image is built again for the trunk (ADR 0498 §6); a package is +// published by its version, not its digest. It moves nothing. + +// The verdicts of the switch's list. +const ( + switchUnmoved = "unmoved" + switchMoves = "moves" + switchUncompared = "cannot be compared" + switchUnknown = "not known" +) + +// switchLine is one module's line of the switch's list. +type switchLine struct { + Module string `json:"module"` + Verdict string `json:"verdict"` + Why string `json:"why"` +} + +// switchReads is what the switch's list reads of the builder and the indexer. +type switchReads interface { + ModuleAt(ctx context.Context, module, commit string) (sdkdelivery.Module, error) + Build(ctx context.Context, module, fp string) (builderkeys.Key, error) +} + +// switchLineOf judges one module: its held build's artifacts against its key, as each artifact's kind allows. +func switchLineOf(module string, held []inventory.Artifact, trunk sdkdelivery.Module, k builderkeys.Key) switchLine { + line := switchLine{Module: module} + switch { + case trunk.FP2 == "": + line.Verdict, line.Why = switchUnknown, "the indexer wrote no fingerprint of the new form: "+orWords(trunk.None2, "none said") + return line + case !k.Done(): + line.Verdict, line.Why = switchUnknown, k.Says() + return line + case len(held) == 0: + line.Verdict, line.Why = switchMoves, "the mesh holds no build of it to compare" + return line + } + digests := map[string]string{} + for _, a := range held { + digests[a.Name] = digestOfReference(a.Reference) + } + var moved, uncompared []string + for _, a := range k.Build.Artifacts { + was, had := digests[a.Name] + delete(digests, a.Name) + switch { + case !had: + moved = append(moved, a.Name+" is new") + case was == a.Digest: + case a.Kind == catalogue.ArtifactImage: + uncompared = append(uncompared, a.Name+" is an image: build-agent's and mesh-build-image's are not one "+ + "byte-for-byte build, and a candidate's image is built again for the trunk (ADR 0498 §6)") + case a.Kind == "package": + uncompared = append(uncompared, a.Name+" is a package: it is published by its version, not its digest") + default: + moved = append(moved, fmt.Sprintf("%s %s -> %s", a.Name, shortDigest(was), shortDigest(a.Digest))) + } + } + for name := range digests { + moved = append(moved, name+" is gone") + } + sort.Strings(moved) + sort.Strings(uncompared) + switch { + case len(moved) > 0: + line.Verdict, line.Why = switchMoves, strings.Join(append(moved, uncompared...), "; ") + case len(uncompared) > 0: + line.Verdict, line.Why = switchUncompared, strings.Join(uncompared, "; ") + default: + line.Verdict, line.Why = switchUnmoved, "every artifact's digest is the held build's: registered with it, no move" + } + return line +} + +// digestOfReference is the `sha256:` a reference names; empty when it names none. +func digestOfReference(reference string) string { + for _, marker := range []string{"@sha256:", "/sha256:"} { + if _, after, ok := strings.Cut(reference, marker); ok { + return "sha256:" + after + } + } + return "" +} + +func shortDigest(d string) string { + d = strings.TrimPrefix(d, "sha256:") + if len(d) > 12 { + return d[:12] + } + if d == "" { + return "none" + } + return d +} + +// switchCommand is `switch [--json]`: the switch's list, read-only. +func switchCommand(ctx context.Context, args []string) error { + set := flag.NewFlagSet("switch", flag.ContinueOnError) + asJSON := set.Bool("json", false, "answer the list as JSON") + if err := set.Parse(args); err != nil { + return err + } + open, err := openStores(ctx) + if err != nil { + return err + } + defer open.Close() + keys, closeKeys, err := openBuilderKeys() + if err != nil { + return err + } + defer closeKeys() + lines, err := switchList(ctx, open.inventory, keys) + if err != nil { + return err + } + if *asJSON { + return json.NewEncoder(os.Stdout).Encode(lines) + } + counts := map[string]int{} + for _, l := range lines { + counts[l.Verdict]++ + fmt.Printf("%-28s %-18s %s\n", l.Module, l.Verdict, l.Why) + } + fmt.Printf("\n%d module(s): %d unmoved, %d move, %d cannot be compared, %d not known\n", len(lines), + counts[switchUnmoved], counts[switchMoves], counts[switchUncompared], counts[switchUnknown]) + return nil +} + +// switchList is the switch's line of every module the catalogue builds but the builder's own, by name. +func switchList(ctx context.Context, inv *inventory.Inventory, keys switchReads) ([]switchLine, error) { + entries, err := inv.Catalogued(ctx) + if err != nil { + return nil, err + } + var out []switchLine + for _, e := range entries { + m := e.Manifest.Module + if e.Provided || e.Source.Repository == "" || isBuildersOwn(m) { + continue + } + trunk, err := keys.ModuleAt(ctx, m, "") + if errors.Is(err, builderkeys.ErrNoValue) { + out = append(out, switchLine{Module: m, Verdict: switchUnknown, Why: "the indexer holds no trunk value of it"}) + continue + } + if err != nil { + return nil, err + } + var k builderkeys.Key + if trunk.FP2 != "" { + if k, err = keys.Build(ctx, m, trunk.FP2); err != nil { + return nil, err + } + } + held, err := heldArtifacts(ctx, inv, m, e.Source.BuiltFrom) + if err != nil { + return nil, err + } + out = append(out, switchLineOf(m, held, trunk, k)) + } + sort.Slice(out, func(i, j int) bool { return out[i].Module < out[j].Module }) + return out, nil +} + +// heldArtifacts are the artifacts of the build the mesh holds of a module: the newest that worked at the commit it is +// registered at. +func heldArtifacts(ctx context.Context, inv *inventory.Inventory, module, builtFrom string) ([]inventory.Artifact, error) { + builds, err := inv.Builds(ctx, module, 20) + if err != nil { + return nil, err + } + for _, b := range builds { + if b.Worked() && sameCommitID(b.Commit, builtFrom) { + return b.Made, nil + } + } + // empty-on-error: not an error; the mesh holds no build of that commit on record, which the line says + return nil, nil +} diff --git a/cmd/mesh-controller/switch_list_test.go b/cmd/mesh-controller/switch_list_test.go new file mode 100644 index 00000000..1055d032 --- /dev/null +++ b/cmd/mesh-controller/switch_list_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "strings" + "testing" + + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + "github.com/novox/mesh-controller/internal/builderkeys" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A13 (ADR 0453 §6; issue 1851): the switch's list names every module moved or unmoved and why: the held build's +// digests against the done key's; an image or a package as what cannot be compared; a key not done as not known. +func TestTheSwitchsListNamesWhatMovesAndWhy(t *testing.T) { + const d1, d2 = "sha256:" + "1111111111111111111111111111111111111111111111111111111111111111", + "sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" + trunk := sdkdelivery.Module{FP2: "src2:" + strings.Repeat("a", 64)} + done := func(arts ...sdkdelivery.BuiltArtifact) builderkeys.Key { + return builderkeys.Key{Found: true, Fingerprint: trunk.FP2, Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildDone, + Artifacts: arts}} + } + held := []inventory.Artifact{{Name: "tools", Kind: "bundle", Reference: "artifact-store://x/tools/blobs/" + d1}} + for _, c := range []struct { + what string + held []inventory.Artifact + trunk sdkdelivery.Module + key builderkeys.Key + verdict string + says string + }{ + {"the same digest", held, trunk, done(sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: d1}), switchUnmoved, "no move"}, + {"another digest", held, trunk, done(sdkdelivery.BuiltArtifact{Name: "tools", Kind: "bundle", Digest: d2}), switchMoves, "tools 111111111111 -> 222222222222"}, + {"an image", []inventory.Artifact{{Name: "img", Kind: "image", Reference: "artifact-store://x/img@" + d1}}, trunk, + done(sdkdelivery.BuiltArtifact{Name: "img", Kind: "image", Digest: d2}), switchUncompared, "img is an image"}, + {"a package", []inventory.Artifact{{Name: "lib", Kind: "package", Reference: "@novox/lib@1.2.0"}}, trunk, + done(sdkdelivery.BuiltArtifact{Name: "lib", Kind: "package", Digest: d2}), switchUncompared, "published by its version"}, + {"a key building", held, trunk, builderkeys.Key{Module: "x", Fingerprint: trunk.FP2, Found: true, + Build: sdkdelivery.BuilderBuild{State: sdkdelivery.BuildBuilding}}, switchUnknown, "is building"}, + {"no fingerprint", held, sdkdelivery.Module{None2: "no lock"}, builderkeys.Key{}, switchUnknown, "no lock"}, + } { + l := switchLineOf("x", c.held, c.trunk, c.key) + if l.Verdict != c.verdict || !strings.Contains(l.Why, c.says) { + t.Errorf("%s: %+v, want %s saying %q", c.what, l, c.verdict, c.says) + } + } +} From e7e23c57ccb99017392ae8b1fe29dab3d994bf87 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:16:50 +0200 Subject: [PATCH 16/38] =?UTF-8?q?Read=20a=20module=20whose=20every=20artif?= =?UTF-8?q?act=20names=20its=20paths=20from=20those=20paths=20and=20its=20?= =?UTF-8?q?manifest=20alone,=20so=20a=20change=20elsewhere=20in=20its=20di?= =?UTF-8?q?rectory=20moves=20nothing=20(issue=201758,=20B1,=20ADR=200499?= =?UTF-8?q?=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/key_takein.go | 40 ++++++++++++++++++++++-- cmd/mesh-controller/paths_source_test.go | 40 ++++++++++++++++++++++++ 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 cmd/mesh-controller/paths_source_test.go diff --git a/cmd/mesh-controller/key_takein.go b/cmd/mesh-controller/key_takein.go index b418c049..67423633 100644 --- a/cmd/mesh-controller/key_takein.go +++ b/cmd/mesh-controller/key_takein.go @@ -289,8 +289,14 @@ func buildOfKey(ctx context.Context, keys keyReads, t keyTake, kc keyContext) ( Commit: t.Trunk.Commit, On: k.Build.By, Path: keyPath(t.Trunk), Manifest: raw, SourceFingerprint: k.Fingerprint, At: time.Now().UTC()} // **Read from the manifest, fresh at every key** (the same rule build-agent says, internal/buildreads). Sources are - // left unsaid: the planner then reads the module's whole directory, over-planning and never under-planning, and a - // fingerprint that did not change moves nothing (R4e A9). + // left unsaid but for a module whose every artifact names its paths: the planner then reads the module's whole + // directory, over-planning and never under-planning, and a fingerprint that did not change moves nothing (R4e A9). + // **Its build source where its every artifact names its paths** (novox/hq ADR 0499 §1, §5; issue 1758, B1): the + // planner reads a merge's changed files against those paths and the manifest alone, so a change elsewhere in the + // module's directory touches nothing it builds from. + if own := pathsBuildSource(manifest, keyPath(t.Trunk)); own != nil { + kept.Sources = []inventory.BuildSource{{Paths: own}} + } for _, c := range buildreads.Of(manifest) { kept.Read = append(kept.Read, inventory.ReadRepository{Repository: c.Repository, Ref: c.Ref}) } @@ -422,3 +428,33 @@ func openBuilderKeys() (builderkeys.Reader, func(), error) { } return builderkeys.Reader{JS: api}, js.Close, nil } + +// pathsBuildSource is the build source of a module whose every artifact names the paths it is built from (ADR 0499 +// §1): its manifest and each path, relative to its repository's root, a path read as a file or everything below a +// directory of that name (`/**` holds both). Nil when an artifact names none: the module reads its directory. +func pathsBuildSource(m catalogue.Manifest, dir string) []string { + if m.Build == nil || len(m.Build.Artifacts) == 0 { + return nil + } + in := func(p string) string { + if dir == "" { + return p + } + return dir + "/" + p + } + seen := map[string]bool{} + out := []string{in("module.json")} + for _, a := range m.Build.Artifacts { + if len(a.Paths) == 0 { + return nil + } + for _, p := range a.Paths { + entry := in(strings.Trim(path.Clean(p), "/")) + "/**" + if !seen[entry] { + seen[entry] = true + out = append(out, entry) + } + } + } + return out +} diff --git a/cmd/mesh-controller/paths_source_test.go b/cmd/mesh-controller/paths_source_test.go new file mode 100644 index 00000000..9f667375 --- /dev/null +++ b/cmd/mesh-controller/paths_source_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// B1 (ADR 0499 §1, §5; issue 1758): a module whose every artifact names the paths it is built from is read from those +// paths and its manifest alone. mesh-sdk's TypeScript package at its repository's root: a change to its go/ touches no +// module, one to its src/ or its package.json touches it. A module with an artifact naming no paths reads its directory. +func TestAModuleNamingItsPathsIsReadFromThemAlone(t *testing.T) { + m, err := catalogue.ParseManifest([]byte(`{"module":"mesh-sdk","version":"1","build":{"artifacts":[{"name":"lib", + "kind":"package","language":"typescript","paths":["src","package.json","package-lock.json","tsconfig.json"]}]}, + "resources":[]}`)) + if err != nil { + t.Fatal(err) + } + own := pathsBuildSource(m, "") + read := inventory.WithBuildSources(nil, []inventory.BuildSource{{Paths: own}}) + e := inventory.Entry{Manifest: m, Source: inventory.Source{Repository: "novox/mesh-sdk", Path: ""}} + for file, reads := range map[string]bool{ + "go/delivery/builder.go": false, "go/go.mod": false, "README.md": false, + "src/index.ts": true, "src/deep/x.ts": true, "package.json": true, "module.json": true, + } { + if got := readsFile(e, read, file); got != reads { + t.Errorf("%s: read %v, want %v", file, got, reads) + } + } + // In a module's directory, the paths are under it. + if got := pathsBuildSource(m, "modules/sdk"); got[0] != "modules/sdk/module.json" || got[1] != "modules/sdk/src/**" { + t.Errorf("paths under a directory: %v", got) + } + // An artifact naming no paths: the whole directory, as before. + m.Build.Artifacts = append(m.Build.Artifacts, catalogue.Artifact{Name: "files", Kind: "archive", From: "files"}) + if got := pathsBuildSource(m, ""); got != nil { + t.Errorf("a module with an artifact naming no paths is narrowed to %v", got) + } +} From 323e1270cc934f239ef1b431bd4fa4b57cebec3d Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:20:37 +0200 Subject: [PATCH 17/38] =?UTF-8?q?Ask=20no=20pull=20request=20check=20of=20?= =?UTF-8?q?build-agent=20but=20the=20own=20merge-check.sh=20of=20a=20repos?= =?UTF-8?q?itory=20whose=20trunk=20requires=20mesh/repo-check=20alone,=20a?= =?UTF-8?q?nd=20say=20every=20verdict=20with=20no=20gate=20layer=20(issue?= =?UTF-8?q?=201299,=20C1,=20ADR=200503=20=C2=A73,=20=C2=A75)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/build_source.go | 33 +-- cmd/mesh-controller/check_siblings.go | 165 ------------ cmd/mesh-controller/check_siblings_test.go | 248 ------------------ cmd/mesh-controller/checks.go | 151 ++++++----- cmd/mesh-controller/checks_queue_test.go | 97 ------- .../checks_stop_integration_test.go | 94 +++++++ .../composed_member_check_test.go | 68 ----- cmd/mesh-controller/composed_verdict_test.go | 36 +-- .../delivery_check_answer_test.go | 2 +- cmd/mesh-controller/replay867_test.go | 2 - cmd/mesh-controller/try_test.go | 4 +- 11 files changed, 201 insertions(+), 699 deletions(-) delete mode 100644 cmd/mesh-controller/check_siblings.go delete mode 100644 cmd/mesh-controller/check_siblings_test.go delete mode 100644 cmd/mesh-controller/checks_queue_test.go create mode 100644 cmd/mesh-controller/checks_stop_integration_test.go delete mode 100644 cmd/mesh-controller/composed_member_check_test.go diff --git a/cmd/mesh-controller/build_source.go b/cmd/mesh-controller/build_source.go index 3520c8fd..82f29a7b 100644 --- a/cmd/mesh-controller/build_source.go +++ b/cmd/mesh-controller/build_source.go @@ -373,22 +373,7 @@ type protectionRule struct { // branch, else the first glob rule, in the forge's order, that covers it (gitea's first matching rule). Never a // later rule that happens to be stronger: the forge does not read it. func judgedRule(rules []protectionRule, branch string) forgeFacts { - var applied *protectionRule - for i := range rules { - if rules[i].Rule == branch { - applied = &rules[i] - break - } - } - if applied == nil { - for i := range rules { - if ruleCovers(rules[i].Rule, branch) { - applied = &rules[i] - break - } - } - } - switch r := applied; { + switch r := appliedRule(rules, branch); { case r == nil: return forgeFacts{Why: fmt.Sprintf("no protection rule covers %s", branch)} case r.Push: @@ -401,6 +386,22 @@ func judgedRule(rules []protectionRule, branch string) forgeFacts { return forgeFacts{Guarded: true} } +// appliedRule is the one rule the forge applies to a branch: the rule named for it, else the first glob rule, in the +// forge's order, that covers it; nil when none does. +func appliedRule(rules []protectionRule, branch string) *protectionRule { + for i := range rules { + if rules[i].Rule == branch { + return &rules[i] + } + } + for i := range rules { + if ruleCovers(rules[i].Rule, branch) { + return &rules[i] + } + } + return nil +} + // ruleCovers says a protection rule's name — a branch, or a glob of them — covers a branch, as the forge reads it. func ruleCovers(rule, branch string) bool { if rule == branch { diff --git a/cmd/mesh-controller/check_siblings.go b/cmd/mesh-controller/check_siblings.go deleted file mode 100644 index df08f3f5..00000000 --- a/cmd/mesh-controller/check_siblings.go +++ /dev/null @@ -1,165 +0,0 @@ -package main - -import ( - "context" - "fmt" - "regexp" - "sort" - "strings" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382, issue 867). -// -// A mesh-host change and a mesh-controller change that need each other could not both pass their own merge checks: -// each was read beside the other's repository at the commit the mesh runs, which holds neither change. So the -// controller asks mesh-delivery's `groups` once for every pull request's check it asks (§1), and names, in the ask, -// the group's other open pull requests into a base of the same name, in a repository the build seat reads beside the -// check — mesh-lab never — each with its whole head. The build seat reads those repositories at the sibling's merge (§2). -// A pull request for which mesh-delivery answered and named no sibling is asked exactly as before (§7). - -// wholeCommit is a commit as the forge says a head: forty hexadecimal characters. -var wholeCommit = regexp.MustCompile(`^[0-9a-f]{40}$`) - -// besideRepositories are the repositories (lower case, owner/repo) a check reads beside its own, by the directory each -// is found under — the core's, as besideOf clones them, and mesh-sdk beside the controller; never mesh-lab, which is -// always read at its trunk. -func besideRepositories(entries []inventory.Entry) map[string]string { - read := map[string]string{} - for _, e := range entries { - dir, core := coreModules[e.Manifest.Module] - if !core || e.Provided || e.Source.Repository == "" { - continue - } - read[nameOf(e.Source.Repository)] = dir - if dir == "mesh-controller" && e.Source.Seat != "" { - read[nameOf(siblingOf(e.Source.Repository, "mesh-sdk"))] = "mesh-sdk" - } - } - delete(read, "") - return read -} - -// nameOf is a recorded repository as owner/repo in lower case, empty when it names no owner. -func nameOf(repository string) string { - owner, repo := sourceOwner(repository), sourceName(repository) - if owner == "" || repo == "" { - return "" - } - return strings.ToLower(owner + "/" + repo) -} - -// siblingsFor is the pull request's siblings, as mesh-delivery's groups say them (ADR 0382 §1): the other members of -// the group its head branch names that are open, unmerged and into a base of its own base's name, in a repository the -// check reads beside it (beside: the directories the ask already names). The second answer is why they could not be -// learned, empty when they could: the group's members carry no head or base — a mesh-delivery from before the rule. -func siblingsFor(p link.PullUpdated, entries []inventory.Entry, beside map[string]link.CheckedOut, - lines []groupsLine) ([]link.CheckSibling, string) { - own := strings.ToLower(p.Owner + "/" + p.Repo) - var group *groupsLine - if p.Head == "" { - // A recheck (mesh-delivery's `recheck` through `delivery check`) carries no head branch name: the group is the one - // whose members hold this very pull request, by repository and number. - learnable := false - for i := range lines { - for _, m := range lines[i].Details { - learnable = true - if p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own { - group = &lines[i] - } - } - } - if group == nil { - for _, l := range lines { - if len(l.Details) == 0 && len(l.Members) > 0 && !learnable { - return nil, "mesh-delivery named the groups' members without their pull requests (a mesh-delivery from before novox/hq ADR 0382): " + - "the group of a pull request announced without its head branch cannot be found" - } - } - return nil, "" // a lone pull request - } - } else { - for i := range lines { - if lines[i].ID == p.Head { - group = &lines[i] - } - } - if group == nil { - return nil, "" - } - } - if len(group.Details) == 0 && len(group.Members) > 0 { - return nil, "mesh-delivery named the group's members without their heads and bases (a mesh-delivery from before novox/hq ADR 0382)" - } - read := besideRepositories(entries) - var siblings []link.CheckSibling - for _, m := range group.Details { - repository := strings.ToLower(m.Repository) - if repository == own { - continue - } - if !m.Open { - continue - } - if m.Head == "" || m.Base == "" { - return nil, "mesh-delivery named " + m.ID + " without its head or base (a mesh-delivery from before novox/hq ADR 0382)" - } - dir, readBeside := read[repository] - if m.Base != p.Base || !readBeside || dir == "mesh-lab" { - continue - } - if _, there := beside[dir]; !there { - continue - } - head := strings.ToLower(m.Head) - if !wholeCommit.MatchString(head) { - return nil, fmt.Sprintf("mesh-delivery named %s with a head that is not a whole commit (%q)", m.ID, m.Head) - } - siblings = append(siblings, link.CheckSibling{Repository: m.Repository, Number: m.Number, Head: head, Base: m.Base, Dir: dir}) - } - sort.Slice(siblings, func(i, j int) bool { return siblings[i].Dir < siblings[j].Dir }) - return siblings, "" -} - -// withSiblings asks mesh-delivery's `groups` once and says in the request the pull request's siblings, or why they -// could not be learned (ADR 0382 §1): a check never waits on mesh-delivery past groupAskWithin and never fails for it, -// so a fix to mesh-delivery is still checked without it. -func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry) { - lines, err := groupsAsked(ctx, deliverySeatHeld(entries)) - siblingsFrom(request, p, entries, lines, err) -} - -// siblingsFrom is withSiblings from mesh-delivery's answer already had. -func siblingsFrom(request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry, lines []groupsLine, err error) { - if request.Check == nil { - return - } - if err != nil { - request.Check.SiblingsUnknown = fmt.Sprintf("mesh-delivery did not say the group's pull requests (%v): "+ - "the check reads no siblings", err) - return - } - request.Check.Siblings, request.Check.SiblingsUnknown = siblingsFor(p, entries, request.Check.Beside, lines) -} - -// composedMember says the pull request is a member of a delivery group whose gate is the composed check's (novox/hq -// ADR 0403 clauses 1 and 7): the group has two members or more and mesh-delivery says so on its line. The group is -// found as siblingsFor finds it, by its head branch name, or, for a recheck that carries none, by the pull request's -// repository and number — not by siblings, which name only what the check reads beside it. -func composedMember(p link.PullUpdated, lines []groupsLine) bool { - own := strings.ToLower(p.Owner + "/" + p.Repo) - for _, l := range lines { - found := p.Head != "" && l.ID == p.Head - for _, m := range l.Details { - if p.Head == "" && p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own { - found = true - } - } - if found { - return l.ComposedGate && max(len(l.Details), len(l.Members)) >= 2 - } - } - return false -} diff --git a/cmd/mesh-controller/check_siblings_test.go b/cmd/mesh-controller/check_siblings_test.go deleted file mode 100644 index 379ecd98..00000000 --- a/cmd/mesh-controller/check_siblings_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A delivery group's member is checked beside the other members' heads** (novox/hq ADR 0382 §1, §7, issue 867): the -// controller asks mesh-delivery's `groups` once for every check it asks, and names in the ask the group's other open -// pull requests into a base of the same name, in a repository the build seat reads beside it — mesh-lab never. - -const ( - headHost = "1111111111111111111111111111111111111111" - headController = "2222222222222222222222222222222222222222" - headSDK = "3333333333333333333333333333333333333333" - headLab = "4444444444444444444444444444444444444444" - headHq = "5555555555555555555555555555555555555555" -) - -func theCoreOnTheGitSeat() []inventory.Entry { - onSeat := func(module, repository string) inventory.Entry { - e := fromRepo(module, repository, "") - e.Source.Seat = "git" - return e - } - nats := onSeat("nats", "novox/mesh-catalog") - nats.Source.Path = "modules/nats" - held := catalogue.Manifest{Module: "mesh-delivery", Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}} - return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"), - onSeat("node-tools", "novox/mesh-tools"), nats, {Manifest: held, On: []string{"anchor"}}} -} - -func besideTheCore() map[string]link.CheckedOut { - return map[string]link.CheckedOut{"mesh-controller": {Repository: "c"}, "mesh-host": {Repository: "h"}, - "mesh-tools": {Repository: "t"}, "mesh-catalog": {Repository: "k"}, "mesh-sdk": {Repository: "s"}, "mesh-lab": {Repository: "l"}} -} - -func member(repository string, number int, head, base string, open bool) memberLine { - return memberLine{ID: repository + "@" + head[:12], Repository: repository, State: "checked", Number: number, Head: head, - Base: base, Open: open} -} - -func group(id string, members ...memberLine) groupsLine { - line := groupsLine{ID: id, Details: members} - for _, m := range members { - line.Members = append(line.Members, m.ID+" ("+m.State+")") - } - return line -} - -func TestASiblingIsTheGroupsOtherOpenPullRequestIntoABaseOfTheSameName(t *testing.T) { - entries := theCoreOnTheGitSeat() - pull := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost} - for _, c := range []struct { - what string - lines []groupsLine - want []string - unknown string - }{ - {"the other member, open, same base", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))}, - []string{"mesh-controller=novox/mesh-controller#346@" + headController + " onto main"}, ""}, - {"a member into a base of another name is not one", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "release", true))}, - nil, ""}, - {"a member not open (merged or closed) is not one", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", false))}, - nil, ""}, - {"mesh-lab is never one", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-lab", 5, headLab, "main", true))}, - nil, ""}, - {"a repository the check reads nothing of is not one", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 9, headHq, "main", true))}, - nil, ""}, - {"mesh-sdk, read beside the controller, is one", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-sdk", 7, headSDK, "main", true))}, - []string{"mesh-sdk=novox/mesh-sdk#7@" + headSDK + " onto main"}, ""}, - {"another group is none of its business", []groupsLine{group("feat/y", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true))}, - nil, ""}, - {"a group of its own alone has none", []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true))}, - nil, ""}, - {"an older mesh-delivery names no heads", []groupsLine{{ID: "feat/x", - Members: []string{"novox/mesh-host@111111111111 (checked)", "novox/mesh-controller@222222222222 (checked)"}}}, - nil, "without their heads and bases"}, - {"a member without its base", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "", true))}, - nil, "without its head or base"}, - {"a head that is no commit", []groupsLine{group("feat/x", - member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController[:12]+"zzzzzzzzzzzzzzzzzzzzzzzzzzzz", "main", true))}, - nil, "not a whole commit"}, - } { - got, unknown := siblingsFor(pull, entries, besideTheCore(), c.lines) - var said []string - for _, s := range got { - said = append(said, s.Dir+"="+s.Repository+"#"+itoa(s.Number)+"@"+s.Head+" onto "+s.Base) - } - if strings.Join(said, ";") != strings.Join(c.want, ";") { - t.Errorf("%s: siblings %v, wanted %v", c.what, said, c.want) - } - if (c.unknown == "") != (unknown == "") || !strings.Contains(unknown, c.unknown) { - t.Errorf("%s: could not learn %q, wanted %q", c.what, unknown, c.unknown) - } - } - // A pull request into a release branch has siblings into the release branch of the same name (ADR 0358 §1). - release := pull - release.Base = "feat/57" - lines := []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "feat/57", true), - member("novox/mesh-controller", 346, headController, "feat/57", true), member("novox/mesh-tools", 8, headLab, "main", true))} - if got, _ := siblingsFor(release, entries, besideTheCore(), lines); len(got) != 1 || got[0].Dir != "mesh-controller" { - t.Errorf("into a release branch: %+v", got) - } - // A recheck announces the pull request without its head branch: its group is the one holding it (repository, number). - pull.Head = "" - got, unknown := siblingsFor(pull, entries, besideTheCore(), lines) - if len(got) != 1 || got[0].Dir != "mesh-tools" || unknown != "" { - t.Errorf("a recheck of a group member: %+v %q", got, unknown) - } - // A pull request in no group, rechecked, is alone: no sibling, and nothing said. - pull.Number = 999 - if got, unknown := siblingsFor(pull, entries, besideTheCore(), lines); got != nil || unknown != "" { - t.Errorf("a lone pull request: %+v %q", got, unknown) - } - if got, unknown := siblingsFor(pull, entries, besideTheCore(), nil); got != nil || unknown != "" { - t.Errorf("no groups at all: %+v %q", got, unknown) - } - // A mesh-delivery that names members without their pull requests cannot find the group of a recheck: it says so. - older := []groupsLine{{ID: "feat/x", Members: []string{"novox/mesh-host@111111111111 (checked)"}}} - pull.Number = 114 - if got, unknown := siblingsFor(pull, entries, besideTheCore(), older); got != nil || unknown == "" { - t.Errorf("an older mesh-delivery: %+v %q", got, unknown) - } -} - -func itoa(n int) string { b, _ := json.Marshal(n); return string(b) } - -// Every check asks mesh-delivery's `groups` once; the ask names the sibling's repository, head and base. -func TestEveryCheckAsksMeshDeliveryOnceAndNamesItsSiblings(t *testing.T) { - entries := theCoreOnTheGitSeat() - asked := 0 - was := groupsAsked - t.Cleanup(func() { groupsAsked = was }) - groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) { - asked++ - if !held { - t.Error("the seat is held, yet it was asked as not held") - } - return []groupsLine{group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), - member("novox/mesh-controller", 346, headController, "main", true))}, nil - } - request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}} - withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries) - if asked != 1 { - t.Fatalf("mesh-delivery was asked %d times for one check", asked) - } - want := link.CheckSibling{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", Dir: "mesh-controller"} - if got := request.Check.Siblings; len(got) != 1 || got[0] != want || request.Check.SiblingsUnknown != "" { - t.Fatalf("the ask names siblings %+v, unknown %q", got, request.Check.SiblingsUnknown) - } - body, _ := json.Marshal(request.Check) - if !strings.Contains(string(body), `"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"`+headController+ - `","base":"main","dir":"mesh-controller"}]`) { - t.Fatalf("on the wire: %s", body) - } -} - -// When mesh-delivery does not answer in time (or is not held, or answered without the fields), the check reads no -// sibling and its request says it could not learn them, so a fix to mesh-delivery is still checked without it. -func TestWhenMeshDeliveryDoesNotAnswerTheCheckReadsNoSiblingAndSaysSo(t *testing.T) { - was := groupsAsked - t.Cleanup(func() { groupsAsked = was }) - for _, c := range []struct { - what string - err error - held bool - }{{"it did not answer", context.DeadlineExceeded, true}, {"nothing serves it", errors.New("nothing serves mesh-delivery.groups"), true}, - {"no holder on record", errors.New("no holder of mesh-delivery is on record to say it"), false}} { - entries := theCoreOnTheGitSeat() - if !c.held { - entries = entries[:4] - } - groupsAsked = func(_ context.Context, held bool) ([]groupsLine, error) { - if held != c.held { - t.Errorf("%s: held %v", c.what, held) - } - return nil, c.err - } - request := link.BuildRequest{Check: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Base: "main", Beside: besideTheCore()}} - withSiblings(t.Context(), &request, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries) - if len(request.Check.Siblings) != 0 || !strings.Contains(request.Check.SiblingsUnknown, c.err.Error()) || - !strings.Contains(request.Check.SiblingsUnknown, "reads no siblings") { - t.Errorf("%s: siblings %+v, unknown %q", c.what, request.Check.Siblings, request.Check.SiblingsUnknown) - } - } -} - -// **A pull request for which mesh-delivery answered and named no sibling is asked exactly as before** (ADR 0382 §7): -// byte for byte the request it was without the rule. -func TestAPullRequestWithNoSiblingIsAskedByteForByteAsBefore(t *testing.T) { - entries := theCoreOnTheGitSeat() - was := groupsAsked - t.Cleanup(func() { groupsAsked = was }) - for _, answer := range [][]groupsLine{nil, {group("feat/other", member("novox/mesh-controller", 346, headController, "main", true))}, - {group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/hq", 3, headHq, "main", true))}} { - groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, nil } - build := func() link.BuildRequest { - return link.BuildRequest{ID: "build-1", Repository: "r", Ref: headHost, Check: &link.CheckRequest{Owner: "novox", - Repo: "mesh-host", Number: 114, Base: "main", Beside: besideTheCore(), Modules: []string{"mesh-host"}, Judge: "validator"}} - } - before, _ := json.Marshal(build()) - after := build() - withSiblings(t.Context(), &after, link.PullUpdated{Owner: "novox", Repo: "mesh-host", Base: "main", Head: "feat/x"}, entries) - got, _ := json.Marshal(after) - if string(before) != string(got) { - t.Fatalf("the request changed:\nbefore %s\nafter %s", before, got) - } - } -} - -// What the build seat's verdict says of its siblings reaches the forge's holder in the controller's `checked`. -func TestTheVerdictCarriesTheSiblingsAndWhereTheJudgeCameFrom(t *testing.T) { - read := []link.CheckSibling{{Repository: "novox/mesh-controller", Number: 346, Head: headController, Base: "main", BaseCommit: headHost}} - c := checkedOf(link.BuildResult{ID: "b", Ref: headHost, Checked: &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114}, - Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok", Siblings: read, JudgeFrom: "sibling novox/mesh-controller#346 merge abc", - Validator: "the change's own validator"}}) - if len(c.Siblings) != 1 || c.Siblings[0] != read[0] || c.JudgeFrom == "" || c.Validator == "" { - t.Fatalf("the verdict says %+v", c) - } - body, _ := json.Marshal(c) - for _, want := range []string{`"siblings":[{"repository":"novox/mesh-controller","number":346,"head":"` + headController + - `","base":"main","base_commit":"` + headHost + `"}]`, `"judge_from":"sibling novox/mesh-controller#346 merge abc"`} { - if !strings.Contains(string(body), want) { - t.Fatalf("the verdict lacks %s:\n%s", want, body) - } - } - unknown := checkedOf(link.BuildResult{ID: "b", Check: &link.CheckOutcome{Verdict: "pass", SiblingsUnknown: "mesh-delivery did not answer"}}) - if body, _ := json.Marshal(unknown); !strings.Contains(string(body), `"siblings_unknown":"mesh-delivery did not answer"`) || - strings.Contains(string(body), `"siblings":`) { - t.Fatalf("the verdict that could not learn them says %s", body) - } -} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index a408e842..1cca29e7 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "io" "os" "path" @@ -161,39 +162,11 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri return "", err } scope := pullScope(p, entries, read, edges) - // **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull - // request's as a group's composed check is: no order exists for what it changes. - alone := orderMember{ID: p.Owner + "/" + p.Repo + "@" + twelve(p.Commit), Repository: p.Owner + "/" + p.Repo, Base: p.Base, - Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA} - if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone}, - map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil { - return "", err - } else if cyclic { - failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA) - fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary) - sayChecked(ctx, w, failing) - return "", nil - } - // **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403 - // clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is - // checked alone, gate included (clause 6). - lines, groupsErr := groupsAsked(ctx, deliverySeatHeld(entries)) - composed := groupsErr == nil && composedMember(p, lines) - // The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted - // with the verdict whatever the verdict is. - plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) { - u, err := inv.UpgradeOf(ctx, module) - return u, err == nil - }) - fmt.Fprint(w, planText(plan)) - direct := directVerdict(p, plan) - if composed { - scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil - composedGate(&direct) - } - // **Mesh-config is told what the head declares at every announced head of a pull request in a repository that holds - // modules** (novox/hq ADR 0502 clauses 2 and 6, issue 1826), also when the head touches no module: a head that reverts - // the change must drop the value held for it. Asked of nothing else here. + // **The merge gate's checks stop** (novox/hq ADR 0503 §3, §5; issue 1299): the checker judges every repository whose + // trunk requires `mesh/check`, and the controller asks nothing of it and says nothing. Only a repository whose trunk + // requires `mesh/repo-check` alone (hq) has its own merge-check.sh asked of build-agent, and said with that layer + // alone; what the forge's protection requires is read, never listed (lead decision L11). + // Mesh-config is still told what the head declares (ADR 0502), whatever judges it. this := pullAtOf(p) if this.Truncated { this.Paths, this.Truncated = truncatedPaths(entries, p.Owner, p.Repo, scope.Modules), false @@ -201,29 +174,54 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri if scope.gated() || holdsModules(entries, p.Owner, p.Repo) { tellPullDeclares(ctx, w, []pullAt{this}) } + required, err := requiredStatusesOf(ctx, p.Owner, p.Repo, p.Base) switch { - case !scope.gated() && !scope.Mesh: - fmt.Fprintf(w, "%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n", - p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched) + case err != nil: + fmt.Fprintf(w, "%s/%s#%d (%.8s): what %s's protection requires cannot be read, so nothing is asked and nothing is "+ + "said: %v\n", p.Owner, p.Repo, p.Number, p.Commit, p.Base, err) + return "", nil + case slices.Contains(required, sdkdelivery.StatusContextCheck): + fmt.Fprintf(w, "%s/%s#%d (%.8s): %s requires %s: the checker judges it; nothing is asked\n", p.Owner, p.Repo, + p.Number, p.Commit, p.Base, sdkdelivery.StatusContextCheck) + return "", nil + case !slices.Contains(required, sdkdelivery.StatusContextRepoCheck): + fmt.Fprintf(w, "%s/%s#%d (%.8s): %s requires neither %s nor %s: nothing is asked and nothing is said\n", p.Owner, + p.Repo, p.Number, p.Commit, p.Base, sdkdelivery.StatusContextCheck, sdkdelivery.StatusContextRepoCheck) + return "", nil + } + return f.repoCheckAlone(ctx, w, p, scope, entries) +} + +// repoCheckAlone asks build-agent for a repository's own merge-check.sh alone, for a pull request of a repository whose +// trunk requires `mesh/repo-check` and not `mesh/check` (ADR 0503 §5): no gate, no module built, its verdict said with the +// repo-check layer alone. +func (f following) repoCheckAlone(ctx context.Context, w io.Writer, p link.PullUpdated, scope checkScope, + entries []inventory.Entry) (string, error) { + inv := f.open.inventory + plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) { + u, err := inv.UpgradeOf(ctx, module) + return u, err == nil + }) + fmt.Fprint(w, planText(plan)) + direct := directVerdict(p, plan) + switch { + case !scope.Mesh: + direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: "the repository is not the mesh's: nothing run"} + fmt.Fprintf(w, "%s/%s#%d (%.8s): the repository is not the mesh's: said, nothing run\n", p.Owner, p.Repo, p.Number, + p.Commit) sayChecked(ctx, w, direct) return "", nil - case !scope.gated() && p.MergeCheckSaid && !p.MergeCheck: + case p.MergeCheckSaid && !p.MergeCheck: direct.RepoCheck = &link.CheckLayer{Verdict: "warning", Summary: noMergeCheck} - fmt.Fprintf(w, "%s/%s#%d (%.8s): %s; %s\n", p.Owner, p.Repo, p.Number, p.Commit, noModuleTouched, noMergeCheck) + fmt.Fprintf(w, "%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, noMergeCheck) sayChecked(ctx, w, direct) return "", nil } - request, err := checkRequestFor(ctx, f.open, p, scope, entries) + request, err := checkRequestFor(ctx, f.open, p, checkScope{Mesh: true, From: scope.From}, entries) if err != nil { return "", err } request.Check.Plan = &plan - request.Check.Composed = composed - if !composed { - carryHeld(ctx, w, &request, []pullAt{this}) - } - // **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1). - siblingsFrom(&request, p, entries, lines, groupsErr) seat := buildSeatHeld(ctx) ask, err := askOverOn(seat) if err != nil { @@ -233,23 +231,42 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri if err := ask.Ask(ctx, request); err != nil { return "", err } - // **A newer request replaces the one still waiting, and a newer key stops the one running** (novox/hq ADR 0406 §3, §4, - // issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked - // of the forge here (issue 726). - what := "its own merge-check.sh alone: " + noModuleTouched - if composed { - what = "its own merge-check.sh alone: the gate is its group's composed check's" - } - if scope.gated() { - what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))", - strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "), - scope.Width, scope.Tiers) - } - fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to check it before it merges — %s — as %s\n", p.Owner, p.Repo, p.Number, - p.Commit, seat, what, request.ID) + fmt.Fprintf(w, "%s/%s#%d (%.8s): asked %s to run its own merge-check.sh alone, as %s\n", p.Owner, p.Repo, p.Number, + p.Commit, seat, request.ID) return request.ID, nil } +// requiredStatusesOf is what the forge's protection of a repository's branch requires before a merge, as the forge +// picks the rule it applies (the one named for the branch, else the first glob covering it). A variable so a test needs +// no forge. +var requiredStatusesOf = func(ctx context.Context, owner, repo, branch string) ([]string, error) { + var rules struct { + Rules []protectionRule `json:"rules"` + } + js, err := aBus() + if err != nil { + return nil, err + } + defer js.Close() + raw, _ := json.Marshal(map[string]any{"owner": owner, "repo": repo}) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_branch_protection_get", raw, 30*time.Second) + if err != nil { + return nil, err + } + if answer.Error != "" { + return nil, fmt.Errorf("gitea.gitea_branch_protection_get: %s", answer.Error) + } + if err := json.Unmarshal(answer.Result, &rules); err != nil { + return nil, err + } + r := appliedRule(rules.Rules, branch) + if r == nil { + // empty-on-error: not an error; no rule covers the branch, so it requires nothing, which the caller says + return nil, nil + } + return r.RequiredStatuses, nil +} + // composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1). const composedWords = "judged by its group's composed check" @@ -378,15 +395,6 @@ func trunkOf(s inventory.Source) string { return "main" } -// sourceName is the name of a recorded repository, a path on the git seat or a URL: novox/mesh-host → mesh-host. -func sourceName(repository string) string { - parts := strings.Split(strings.Trim(strings.TrimSuffix(repository, ".git"), "/"), "/") - if len(parts) < 2 { - return "" - } - return parts[len(parts)-1] -} - // namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the // check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key // still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be @@ -676,6 +684,9 @@ func prefixedAll(prefix string, items []string) []string { // sayChecked says a merge check's verdict on the bus, where the forge's holder hears it. func sayChecked(ctx context.Context, w io.Writer, c link.Checked) { + // **No merge-gate layer from R4e, for any repository** (ADR 0503 §3, §5; issue 1299): the forge's holder writes + // `mesh/repo-check` from the repo-check layer alone, and the checker owes `mesh/check`. + c.Gate = nil repo := "none" if c.RepoCheck != nil { repo = strings.ToUpper(c.RepoCheck.Verdict) + " — " + c.RepoCheck.Summary @@ -684,8 +695,8 @@ func sayChecked(ctx context.Context, w io.Writer, c link.Checked) { if c.Trunk != "" { judged = " merged into trunk " + c.Trunk } - fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: gate %s — %s; repository %s\n", c.ID, c.Owner, c.Repo, c.Number, - c.Commit, judged, orSomewhere(c.On), strings.ToUpper(c.Verdict), c.Summary, repo) + fmt.Fprintf(w, "%s: %s/%s#%d at %.8s%s checked on %s: repository %s\n", c.ID, c.Owner, c.Repo, c.Number, + c.Commit, judged, orSomewhere(c.On), repo) if checkEvents == nil { return } diff --git a/cmd/mesh-controller/checks_queue_test.go b/cmd/mesh-controller/checks_queue_test.go deleted file mode 100644 index b9fb48f2..00000000 --- a/cmd/mesh-controller/checks_queue_test.go +++ /dev/null @@ -1,97 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "encoding/json" - "fmt" - "testing" - - "github.com/nats-io/nats.go" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// Acceptance 5 of milestone 2 (novox/hq ADR 0406 §3, issue 974), live, with the store and a real bus: two merges into one -// repository within a minute, with two open pull requests, leave each pull request with exactly one queued check, by the -// stream's rule. Each merge moves the trunk, and mesh-delivery asks the controller to check both open pull requests again -// against it (`delivery check`, which is the controller's own path for a head announced); nothing moves a request, nothing -// asks the forge, and the queue is read back whole. -func TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - js, _ := aBuildQueue(t) - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(nil) }) - holdTheDeliverySeat(t, open) - was := askDeliveryOwner - t.Cleanup(func() { askDeliveryOwner = was }) - askDeliveryOwner = func(_ context.Context, _ *nats.Conn, verb string, _ map[string]any) (json.RawMessage, error) { - if verb != "groups" { - t.Fatalf("asked the owner %s", verb) - } - return json.RawMessage(`[]`), nil - } - // The repository the two pull requests are into, as a module the mesh builds from it. - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - - heads := map[int]string{7: "7777777777777777777777777777777777777777", 8: "8888888888888888888888888888888888888888"} - check := func(number int, trunk string) { - t.Helper() - p := orderMember{ID: fmt.Sprintf("novox/mesh-controller@%.12s", heads[number]), Repository: "novox/mesh-controller", - Base: "main", Head: heads[number], Number: number, Paths: []string{"main.go"}, ModuleDirsSaid: true, - BaseSHA: trunk, Ready: number == 8}.pull() - p.MergeCheck, p.MergeCheckSaid = true, true - if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil { - t.Fatal(err) - } - } - // Announced against the trunk as it stands, then asked again after each of two merges. - for _, trunk := range []string{"1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", - "3333333333333333333333333333333333333333"} { - check(7, trunk) - check(8, trunk) - } - - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - seen := map[int]string{} - for _, a := range q.Asks { - c := a.Request.Check - if c == nil { - t.Fatalf("%s is no check", a.ID) - } - if _, twice := seen[c.Number]; twice { - t.Errorf("pull request %d has two queued checks", c.Number) - } - seen[c.Number] = c.Trunk - if a.State != link.AskWaiting { - t.Errorf("%s is %s", a.ID, a.State) - } - if want := "3333333333333333333333333333333333333333"; c.Trunk != want { - t.Errorf("pull request %d is queued against %.8s; want the newest trunk %.8s", c.Number, c.Trunk, want) - } - } - if len(seen) != 2 { - t.Fatalf("queued checks for %v; want one for each of the two open pull requests", seen) - } - // By the streams' own count: the ready one on its stream, the other on its. - for stream, want := range map[string]uint64{broker.ChecksReadyStream: 1, broker.ChecksOtherStream: 1} { - info, err := js.Context().StreamInfo(stream) - if err != nil { - t.Fatal(err) - } - if info.State.Msgs != want { - t.Errorf("%s holds %d request(s); want %d", stream, info.State.Msgs, want) - } - } -} diff --git a/cmd/mesh-controller/checks_stop_integration_test.go b/cmd/mesh-controller/checks_stop_integration_test.go new file mode 100644 index 00000000..f37ac855 --- /dev/null +++ b/cmd/mesh-controller/checks_stop_integration_test.go @@ -0,0 +1,94 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// C1 (ADR 0503 §3, §5; issue 1299, lead decision L11), with the store and a real build queue: a pull request of a +// repository whose trunk requires mesh/check asks build-agent nothing and says nothing; one whose trunk requires +// mesh/repo-check alone (hq) has its own merge-check.sh asked, and every verdict said names no gate layer; a protection +// that cannot be read asks and says nothing. +func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + js, _ := aBuildQueue(t) + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + said := &saidOnTheBus{} + checkEvents = said + t.Cleanup(func() { checkEvents = nil }) + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, + inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + required := map[string][]string{"novox/mesh-controller": {"mesh/check"}, "novox/hq": {"mesh/repo-check"}} + was := requiredStatusesOf + requiredStatusesOf = func(_ context.Context, owner, repo, _ string) ([]string, error) { + if r, ok := required[owner+"/"+repo]; ok { + return r, nil + } + return nil, errors.New("the forge did not answer") + } + t.Cleanup(func() { requiredStatusesOf = was }) + queued := func() int { + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + return len(q.Asks) + } + pull := func(repo string, number int, mergeCheck bool) link.PullUpdated { + p := link.PullUpdated{Owner: "novox", Repo: repo, Number: number, Base: "main", Commit: "abcdefabcdefabcdefabcdefabcdefabcdefabcd", + BaseSHA: "1111111111111111111111111111111111111111", Paths: []string{"main.go"}, ModuleDirsSaid: true} + p.MergeCheck, p.MergeCheckSaid = mergeCheck, true + return p + } + f := following{open: open, oneShot: true} + + if err := f.PullUpdated(ctx, pull("mesh-controller", 7, true)); err != nil { + t.Fatal(err) + } + if n := queued(); n != 0 || len(said.events) != 0 { + t.Fatalf("0503 §3: a repository the checker judges asked %d check(s) and said %v", n, said.events) + } + if err := f.PullUpdated(ctx, pull("unread", 3, true)); err != nil { + t.Fatal(err) + } + if n := queued(); n != 0 || len(said.events) != 0 { + t.Fatalf("L11: a protection that cannot be read asked %d check(s) and said %v", n, said.events) + } + if err := f.PullUpdated(ctx, pull("hq", 9, true)); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].Request.Check == nil || len(q.Asks[0].Request.Check.Modules) != 0 || + q.Asks[0].Request.Check.Repo != "hq" { + t.Fatalf("0503 §5: hq's repository check is not the one ask: %+v", q.Asks) + } + // A head with no merge-check.sh is said at once, with the repo-check layer alone. + if err := f.PullUpdated(ctx, pull("hq", 10, false)); err != nil { + t.Fatal(err) + } + if len(said.events) != 1 { + t.Fatalf("hq's head with no merge-check.sh was said %d time(s)", len(said.events)) + } + var c link.Checked + if err := json.Unmarshal(said.bodies[0], &c); err != nil { + t.Fatal(err) + } + if c.Gate != nil || c.RepoCheck == nil || c.RepoCheck.Verdict != "warning" { + t.Fatalf("0503 §5: the verdict said names a gate layer, or no repo-check layer: %+v", c) + } +} diff --git a/cmd/mesh-controller/composed_member_check_test.go b/cmd/mesh-controller/composed_member_check_test.go deleted file mode 100644 index 7943412f..00000000 --- a/cmd/mesh-controller/composed_member_check_test.go +++ /dev/null @@ -1,68 +0,0 @@ -//go:build integration - -package main - -import ( - "context" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// A member of a group whose gate is the composed check's is asked of the build seat with no module: its repository's -// check alone (novox/hq ADR 0403 clause 1). With no word from mesh-delivery, or one that does not say the gate is -// composed, it is checked alone with its gate (clause 6). - -func askedForMember(t *testing.T, answer []groupsLine, answerErr error) link.CheckRequest { - t.Helper() - open := aMesh(t) - ctx := t.Context() - js, _ := aBuildQueue(t) - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(nil) }) - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - was := groupsAsked - groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, answerErr } - t.Cleanup(func() { groupsAsked = was }) - p := link.PullUpdated{Owner: "novox", Repo: "mesh-controller", Number: 346, Base: "main", Head: "feat/x", Commit: headController, - Paths: []string{"main.go"}, ModuleDirsSaid: true, MergeCheck: true, MergeCheckSaid: true} - if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil || len(q.Asks) != 1 { - t.Fatalf("asks %d: %v", len(q.Asks), err) - } - return *q.Asks[0].Request.Check -} - -func TestAComposedMembersCheckIsItsRepositoryChecksAlone(t *testing.T) { - line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true)) - line.ComposedGate = true - c := askedForMember(t, []groupsLine{line}, nil) - if len(c.Modules) != 0 || len(c.New) != 0 || len(c.Dependents) != 0 || !c.Composed { - t.Fatalf("the member's check is asked as %+v; want no module and composed", c) - } -} - -func TestAMemberOfAGroupOfAnOlderMeshDeliveryKeepsItsOwnGate(t *testing.T) { - line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true)) - c := askedForMember(t, []groupsLine{line}, nil) - if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed { - t.Fatalf("an older mesh-delivery's member is asked as %+v; want its own gate", c) - } -} - -func TestAMemberWhoseGroupCannotBeLearnedIsCheckedAloneWithItsGate(t *testing.T) { - c := askedForMember(t, nil, context.DeadlineExceeded) - if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed || - !strings.Contains(c.SiblingsUnknown, "reads no siblings") { - t.Fatalf("the member is asked as %+v; want its gate and why its siblings are unknown", c) - } -} diff --git a/cmd/mesh-controller/composed_verdict_test.go b/cmd/mesh-controller/composed_verdict_test.go index f96bc00c..5fcd49f7 100644 --- a/cmd/mesh-controller/composed_verdict_test.go +++ b/cmd/mesh-controller/composed_verdict_test.go @@ -14,37 +14,11 @@ import ( // A delivery group's gate is the composed check's, and its members' own checks run their repository's check alone // (novox/hq ADR 0403 clauses 1 and 7). Pure: no store, no bus. -// A member of a group whose line says composed_gate is a composed member; one whose group says nothing of it (an older -// mesh-delivery), one of a group of a single pull request and a lone pull request are not. -func TestAMemberOfAComposedGroupIsKnownByTheGroupsLine(t *testing.T) { - two := func(composed bool) groupsLine { - l := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true)) - l.ComposedGate = composed - return l - } - p := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost} - recheck := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Commit: headHost} - for name, c := range map[string]struct { - p link.PullUpdated - lines []groupsLine - want bool - }{ - "composed": {p, []groupsLine{two(true)}, true}, - "a recheck, by its number": {recheck, []groupsLine{two(true)}, true}, - "an older mesh-delivery": {p, []groupsLine{two(false)}, false}, - "another group": {p, []groupsLine{group("feat/y", member("novox/mesh-host", 1, headHost, "main", true))}, false}, - "no group": {p, nil, false}, - } { - if got := composedMember(c.p, c.lines); got != c.want { - t.Errorf("%s: composed %v, want %v", name, got, c.want) - } - } - alone := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true)) - alone.ComposedGate = true - if composedMember(p, []groupsLine{alone}) { - t.Error("a group of a single pull request is composed") - } -} +// The heads of a group's members, by repository. +const ( + headHost = "1111111111111111111111111111111111111111" + headController = "2222222222222222222222222222222222222222" +) // A composed member's verdict never says its gate passed: the gate layer, and the verdict that is the gate's, read // "composed"; a failure or an error stays what it was. diff --git a/cmd/mesh-controller/delivery_check_answer_test.go b/cmd/mesh-controller/delivery_check_answer_test.go index 21eee54d..c9609ea7 100644 --- a/cmd/mesh-controller/delivery_check_answer_test.go +++ b/cmd/mesh-controller/delivery_check_answer_test.go @@ -41,7 +41,7 @@ func TestSayCheckedWritesTheSameLineToItsWriter(t *testing.T) { var w bytes.Buffer sayChecked(context.Background(), &w, link.Checked{ID: "c1", Owner: "novox", Repo: "x", Number: 7, Commit: "0123456789abcdef", Verdict: "pass", Summary: "fine"}) - want := "c1: novox/x#7 at 01234567 checked on a machine that did not say which: gate PASS — fine; repository none\n" + want := "c1: novox/x#7 at 01234567 checked on a machine that did not say which: repository none\n" if w.String() != want { t.Fatalf("sayChecked said %q; want %q", w.String(), want) } diff --git a/cmd/mesh-controller/replay867_test.go b/cmd/mesh-controller/replay867_test.go index 879713e3..a8316cee 100644 --- a/cmd/mesh-controller/replay867_test.go +++ b/cmd/mesh-controller/replay867_test.go @@ -36,11 +36,9 @@ import ( // It runs the whole path: the forge announces each pull request (following.PullUpdated), the controller asks // mesh-delivery which pull requests the group holds, the ask is queued for the build seat, and the build seat's check // (builder.Check, with a container runtime that runs each container's command on this machine) judges it. -func TestReplay867(t *testing.T) { replay867(t, false) } // TestReplay867Recheck is the same pair re-checked through mesh-delivery (`delivery check` with one member): the forge's // announcement is not repeated, so the pull request reaches the controller without its head branch name. -func TestReplay867Recheck(t *testing.T) { replay867(t, true) } func replay867(t *testing.T, recheck bool) { if _, err := exec.LookPath("go"); err != nil { diff --git a/cmd/mesh-controller/try_test.go b/cmd/mesh-controller/try_test.go index e696945b..32fb8aa6 100644 --- a/cmd/mesh-controller/try_test.go +++ b/cmd/mesh-controller/try_test.go @@ -22,10 +22,12 @@ import ( type saidOnTheBus struct { link.Bus events []string + bodies [][]byte } -func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, _ []byte) error { +func (s *saidOnTheBus) PublishSeatEvent(_ context.Context, seat, event string, body []byte) error { s.events = append(s.events, seat+" "+event) + s.bodies = append(s.bodies, body) return nil } From 25fdf9920e58ef0cad1e6da2c84052aefc6fd393 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:21:47 +0200 Subject: [PATCH 18/38] Read a commit as the same as itself however short, and fit the pending assignments' tests to a mesh that asks no build (issue 1851, A9, A10) --- .../pending_integration_test.go | 59 +------------------ cmd/mesh-controller/plan_keys.go | 3 + cmd/mesh-controller/replay325_test.go | 9 +-- cmd/mesh-controller/same_commit_test.go | 17 ++++++ 4 files changed, 24 insertions(+), 64 deletions(-) create mode 100644 cmd/mesh-controller/same_commit_test.go diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go index e45d5207..3e1b4780 100644 --- a/cmd/mesh-controller/pending_integration_test.go +++ b/cmd/mesh-controller/pending_integration_test.go @@ -3,7 +3,6 @@ package main import ( - "context" "encoding/json" "errors" "fmt" @@ -178,7 +177,7 @@ func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { t.Fatalf("a module whose build failed: %v", err) } for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", - "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { + "failed: go build: undefined: sensorsRead", "registered from the builder's done key", "novox/mesh-catalog (path modules/sensors)"} { if !strings.Contains(err.Error(), want) { t.Fatalf("the refusal does not say %q:\n%v", want, err) } @@ -308,7 +307,7 @@ func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { } for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", - "asks for nothing here"} { + "asks for nothing: the controller asks no build"} { if !strings.Contains(err.Error(), want) { t.Fatalf("the refusal does not say %q:\n%v", want, err) } @@ -478,60 +477,6 @@ func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { } } -func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asked := 0 - asksOfEveryModule(t) // the one ask left is the builder's own (ADR 0453 §5): this test is of what a walk does around it - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { - asked++ - return fmt.Sprintf("b-%d", asked), nil - } - t.Cleanup(func() { askABuild = was }) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-1") - settlePending(ctx, open, time.Now()) - first := pendingOf(t, open)[0] - if !conditionOpen(t, first) { - t.Fatal("the first assignment not made raised nothing") - } - - if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-2") - settlePending(ctx, open, time.Now()) - - var second inventory.PendingAssignment - for _, p := range pendingOf(t, open) { - if p.Build == "b-2" { - second = p - } - } - if second.State != inventory.PendingExpired { - t.Fatalf("the second: %+v", second) - } - if !conditionOpen(t, second) { - t.Fatal("the second assignment's condition was cleared in the tick that raised it") - } - if !conditionOpen(t, rowOf(t, open, first.ID)) { - t.Fatal("the first was judged answered by a second that itself was not made") - } - // unassign takes both back, and the next tick clears both. - if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") || - !strings.Contains(said, "b-2") { - t.Fatalf("unassign: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, first) || conditionOpen(t, second) { - t.Fatal("a condition stayed open after both were taken back") - } -} - func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { open := aCatalogueMesh(t) ctx := t.Context() diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go index e3c48291..16cecbf1 100644 --- a/cmd/mesh-controller/plan_keys.go +++ b/cmd/mesh-controller/plan_keys.go @@ -209,6 +209,9 @@ func sameCommitID(a, b string) bool { if a == "" || b == "" { return false } + if a == b { + return true + } n := min(len(a), len(b)) return n >= 7 && a[:n] == b[:n] } diff --git a/cmd/mesh-controller/replay325_test.go b/cmd/mesh-controller/replay325_test.go index 2ca214b8..b4dacb4f 100644 --- a/cmd/mesh-controller/replay325_test.go +++ b/cmd/mesh-controller/replay325_test.go @@ -27,13 +27,8 @@ func TestReplay325(t *testing.T) { BuiltFrom: "c0", Head: "c0"}); err != nil { t.Fatal(err) } - asksWithPaths(t) - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "5e450125aa", - Paths: []string{"modules/sensors/module.json", "modules/sensors/cmd/sensors/main.go"}, - ModuleDirs: []string{"modules/sensors"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } + // The build the merge asked before R4e (nothing asks one since, ADR 0501 §6), kept as it was. + mergeAdding(t, open, "modules/sensors", "5e450125aa") said, err := assign(ctx, open, "laptop", "sensors") if err != nil { diff --git a/cmd/mesh-controller/same_commit_test.go b/cmd/mesh-controller/same_commit_test.go new file mode 100644 index 00000000..2e33b719 --- /dev/null +++ b/cmd/mesh-controller/same_commit_test.go @@ -0,0 +1,17 @@ +package main + +import "testing" + +// A held build's commit as recorded and as registered name one commit however each is abbreviated, a short name +// included (the fault the try of R4e's controller found: "c0" was never the same as "c0"). +func TestACommitIsTheSameAsItselfHoweverShort(t *testing.T) { + for _, c := range []struct { + a, b string + same bool + }{{"c0", "c0", true}, {"c0ffee0123456789", "c0ffee0", true}, {"c0ffee0123", "c0ffee0999", false}, {"", "", false}, + {"c0", "c1", false}} { + if got := sameCommitID(c.a, c.b); got != c.same { + t.Errorf("%q %q: %v", c.a, c.b, got) + } + } +} From 3a1937914e7099d7e363c52f81e305dd1f5087cc Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:23:23 +0200 Subject: [PATCH 19/38] =?UTF-8?q?Ask=20build-agent=20no=20composed=20check?= =?UTF-8?q?=20of=20a=20delivery=20group,=20and=20answer=20that=20the=20che?= =?UTF-8?q?cker=20judges=20it=20(issue=201299,=20C2,=20ADR=200503=20=C2=A7?= =?UTF-8?q?3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../checks_stop_integration_test.go | 47 ++++++ cmd/mesh-controller/composed_verdict_test.go | 49 ------ cmd/mesh-controller/delivery.go | 143 +----------------- cmd/mesh-controller/delivery_edges_test.go | 29 ---- cmd/mesh-controller/pull_declares.go | 66 -------- cmd/mesh-controller/pull_declares_test.go | 47 +----- 6 files changed, 53 insertions(+), 328 deletions(-) diff --git a/cmd/mesh-controller/checks_stop_integration_test.go b/cmd/mesh-controller/checks_stop_integration_test.go index f37ac855..d6f32d1d 100644 --- a/cmd/mesh-controller/checks_stop_integration_test.go +++ b/cmd/mesh-controller/checks_stop_integration_test.go @@ -6,6 +6,9 @@ import ( "context" "encoding/json" "errors" + "io" + "os" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -92,3 +95,47 @@ func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t t.Fatalf("0503 §5: the verdict said names a gate layer, or no repo-check layer: %+v", c) } } + +// C2 (ADR 0503 §3; issue 1299): a delivery group's composed check asks build-agent nothing and answers that the checker +// judges the group. +func TestTheGroupCheckAsksNothingAndNamesTheChecker(t *testing.T) { + aMesh(t) + ctx := t.Context() + js, _ := aBuildQueue(t) + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + members := `[{"id":"novox/a@111111111111","repository":"novox/a","base":"main","head":"1111111111111111111111111111111111111111","number":1},` + + `{"id":"novox/b@222222222222","repository":"novox/b","base":"main","head":"2222222222222222222222222222222222222222","number":2}]` + out := capturedStdout(t, func() error { + return deliveryCommand(ctx, []string{"check", "--group", "feat/x", "--members", members}) + }) + if !strings.Contains(out, `"judged-by": "mesh-delivery-checker"`) || !strings.Contains(out, `"asked": ""`) { + t.Fatalf("0503 §3: the group check answered %s", out) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 0 { + t.Fatalf("0503 §3: the group check asked %+v", q.Asks) + } +} + +// capturedStdout is what run writes to standard output. +func capturedStdout(t *testing.T, run func() error) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + was := os.Stdout + os.Stdout = w + runErr := run() + os.Stdout = was + w.Close() + out, _ := io.ReadAll(r) + if runErr != nil { + t.Fatal(runErr) + } + return string(out) +} diff --git a/cmd/mesh-controller/composed_verdict_test.go b/cmd/mesh-controller/composed_verdict_test.go index 5fcd49f7..58e4ee36 100644 --- a/cmd/mesh-controller/composed_verdict_test.go +++ b/cmd/mesh-controller/composed_verdict_test.go @@ -1,13 +1,10 @@ package main import ( - "context" "encoding/json" "strings" "testing" - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -62,49 +59,3 @@ func TestTheComposedVerdictCarriesEveryMember(t *testing.T) { } } } - -// Members whose manifests order their modules in a cycle fail the composed check, naming every module and edge, and no -// ask is made of the build seat. -func TestAGroupInACycleFailsItsComposedCheckWithoutABuild(t *testing.T) { - members := []orderMember{ - {ID: "novox/x@" + headHost[:12], Repository: "novox/x", Base: "main", Head: headHost, Number: 7, Paths: []string{"module.json"}}, - {ID: "novox/y@" + headController[:12], Repository: "novox/y", Base: "main", Head: headController, Number: 9, Paths: []string{"module.json"}}, - } - reach := map[string]orderReach{members[0].ID: {Moved: []string{"mx"}}, members[1].ID: {Moved: []string{"my"}}} - was, wasParent := manifestAt, parentOf - t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) - parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } - manifestAt = func(_ context.Context, _, repo, _, ref string) (catalogue.Manifest, bool, error) { - if ref == "main" { - return catalogue.Manifest{}, false, nil // what the pull requests merge into holds no such manifest - } - if repo == "x" { - return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}}, - Invokes: []string{"seat:s.b"}}, true, nil - } - return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}}, - Invokes: []string{"seat:s.a"}}, true, nil - } - c, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, []inventory.Edge(nil), nil) - if err != nil || !cyclic { - t.Fatalf("no cycle found: %v", err) - } - if c.Verdict != "fail" || c.Group != "feat/cycle" || len(c.Members) != 2 || c.Gate == nil || c.Gate.Verdict != "fail" { - t.Fatalf("the verdict is %+v", c) - } - for _, w := range []string{"mx", "my", "s.a", "s.b"} { - if !strings.Contains(c.Summary, w) { - t.Errorf("the cycle does not name %s: %s", w, c.Summary) - } - } - // Members with no edge between them are no cycle. - manifestsReadMu.Lock() - clear(manifestsRead) - manifestsReadMu.Unlock() - manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) { - return catalogue.Manifest{}, false, nil - } - if _, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, nil, nil); err != nil || cyclic { - t.Fatalf("a cycle where there is none: %v %v", cyclic, err) - } -} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index e8af7c27..4e102431 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -914,11 +914,10 @@ func deliveryCommand(ctx context.Context, args []string) error { if *group == "" || len(members) < 2 { return errors.New("delivery check --group --members (two heads or more), or --members ") } - id, err := askGroupCheck(ctx, open, *group, members) - if err != nil { - return err - } - return answer(map[string]any{"asked": id, "group": *group}) + // **The group's composed check is the checker's** (novox/hq ADR 0503 §3; issue 1299, C2): nothing is asked of + // build-agent, and the answer says who judges it. + return answer(map[string]any{"asked": "", "group": *group, "judged-by": checkerModule, + "said": "the checker judges a delivery group's heads merged together (ADR 0503 §3): nothing was asked"}) case "go": if len(positionals) != 1 { return errors.New("delivery go [--by ] [--why ]") @@ -1093,38 +1092,6 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, return entries, read, edges, nil } -// cycleVerdict is the failing verdict of a group whose modules are ordered in a cycle, and whether there is one: the -// group's composed check, said without a build. The manifests' change is read from the forge; an answer it cannot give -// is an error, never read as no cycle. -func cycleVerdict(ctx context.Context, group string, members []orderMember, reach map[string]orderReach, - edges []inventory.Edge, entries []inventory.Entry) (link.Checked, bool, error) { - changes, err := changesOf(ctx, changeSourcesOf(members, entries, reach)) - if err != nil { - return link.Checked{}, false, fmt.Errorf("what the group's members say of their manifests cannot be read: %w", err) - } - var moved []string - for _, m := range members { - moved = appendNew(moved, reach[m.ID].Moved...) - } - said := orderEdges(moved, changes, edges).CycleSaid() - if said == "" { - return link.Checked{}, false, nil - } - first := members[0].pull() - c := link.Checked{ID: link.NewBuildID(time.Now()), Owner: first.Owner, Repo: first.Repo, Number: first.Number, - Commit: first.Commit, Verdict: "fail", Summary: said, Report: said, Group: group, - Gate: &link.CheckLayer{Verdict: "fail", Summary: said}} - for _, m := range members { - if group == "" { - break // a lone pull request's verdict names no group and no member - } - p := m.pull() - c.Members = append(c.Members, link.CheckedMember{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit, - Trunk: twelve(p.BaseSHA)}) - } - return c, true, nil -} - // twelve is a commit as twelve characters. func twelve(c string) string { if len(c) > 12 { @@ -1133,99 +1100,6 @@ func twelve(c string) string { return c } -// groupPrimary is the head a group's composed check is run from: the one moving the controller, which then -// judges the group by itself; else the one moving the node-engine, whose validator judges; else the first. -func groupPrimary(members []orderMember, reach map[string]orderReach) int { - for _, want := range []string{"mesh-controller", "mesh-host"} { - for i, m := range members { - if slices.Contains(reach[m.ID].Moved, want) { - return i - } - } - } - return 0 -} - -// askGroupCheck asks the build seat for a group's composed check: every head laid over the mesh in turn, -// judged as one future state (novox/hq ADR 0239). The verdict comes back as `checked` with the group's id, -// for mesh-delivery; the forge's holder sets no status from it. -func askGroupCheck(ctx context.Context, open *stores, group string, members []orderMember) (string, error) { - entries, read, edges, err := theGraph(ctx, open.inventory) - if err != nil { - return "", err - } - reach := reachOfMembers(members, entries, read, edges) - // **A cycle of edges fails the group's composed check, naming every module and edge** (novox/hq ADR 0419 clause 6), - // before the build seat is asked: no composition of the heads has an order to be sent in. - if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil { - return "", err - } else if cyclic { - sayChecked(ctx, os.Stderr, failing) - return failing.ID, nil - } - primary := groupPrimary(members, reach) - p := members[primary].pull() - scope := pullScope(p, entries, read, edges) - // The gate runs over what any member moves; a judge from the primary alone. - for _, m := range members { - r := reach[m.ID] - scope.Modules = appendNew(scope.Modules, r.Moved...) - scope.Dependents = appendNew(scope.Dependents, r.Dependents...) - } - request, err := checkRequestFor(ctx, open, p, scope, entries) - if err != nil { - return "", err - } - request.Check.Group = group - // **The values held for the keys the members introduce are told and carried** (novox/hq ADR 0502, issue 1826). - var pulls []pullAt - for i, s := range changeSourcesOf(members, entries, reach) { - pulls = append(pulls, pullAt{changeSource: s, Number: members[i].pull().Number}) - } - withHeldConfig(ctx, os.Stderr, &request, pulls, true) - // A group's check is queued as ready when any member's pull request is marked ready. - request.Check.Ready = slices.ContainsFunc(members, func(m orderMember) bool { return m.Ready }) - world, err := restOfTheMesh(ctx, open, shelfOf(entries), "") - if err != nil { - return "", err - } - for i, m := range members { - if i == primary { - continue - } - mp := m.pull() - // As the mesh clones a module built from it; a repository no module is built from, from the forge. - source := inventory.Source{Seat: gitSeat, Repository: mp.Owner + "/" + mp.Repo} - for _, e := range entries { - if !e.Provided && sameRepository(e.Source.Repository, link.SourceMoved{Owner: mp.Owner, Repo: mp.Repo}) { - source = e.Source - break - } - } - url := source.Repository - if source.Seat != "" { - url, err = clonedFromSeat(world, source.Seat, source.Repository) - } - if err != nil { - return "", fmt.Errorf("%s cannot be cloned for the group's check: %w", m.ID, err) - } - request.Check.Members = append(request.Check.Members, link.GroupMember{Owner: mp.Owner, Repo: mp.Repo, - Number: mp.Number, Repository: url, Ref: mp.Commit, Paths: mp.Paths, Base: mp.Base, Trunk: mp.BaseSHA}) - } - seat := buildSeatHeld(ctx) - ask, err := askOverOn(seat) - if err != nil { - return "", err - } - defer ask.Close() - if err := ask.Ask(ctx, request); err != nil { - return "", err - } - fmt.Fprintf(os.Stderr, "group %s: asked %s to check %d head(s) composed together, as %s\n", group, seat, - len(members), request.ID) - return request.ID, nil -} - // appendNew appends what is not there yet. func appendNew(to []string, items ...string) []string { for _, i := range items { @@ -1236,15 +1110,6 @@ func appendNew(to []string, items ...string) []string { return to } -// shelfOf is the catalogue's manifests by name. -func shelfOf(entries []inventory.Entry) map[string]catalogue.Manifest { - shelf := map[string]catalogue.Manifest{} - for _, e := range entries { - shelf[e.Manifest.Module] = e.Manifest - } - return shelf -} - // sayPlanMoved says a walk kept in a new state as the controller's `plan-moved`, the plan whole: what the // delivery it walks reads its steps from. Never in the way of the walk: said beside it, and a save that could // not be said is logged — the delivery's owner, which also asks for the walks it follows, finds it by diff --git a/cmd/mesh-controller/delivery_edges_test.go b/cmd/mesh-controller/delivery_edges_test.go index b858720c..96da775d 100644 --- a/cmd/mesh-controller/delivery_edges_test.go +++ b/cmd/mesh-controller/delivery_edges_test.go @@ -181,32 +181,3 @@ func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) t.Fatalf("the member's change is read as %+v", src) } } - -// A lone pull request's cycle fails its check, naming every module and edge, with no group and no members. -func TestALonePullRequestInACycleFailsWithNoGroup(t *testing.T) { - was, wasParent := manifestAt, parentOf - t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) - manifestsReadMu.Lock() - clear(manifestsRead) - manifestsReadMu.Unlock() - const head = "3333333333333333333333333333333333333333" - parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } - manifestAt = func(_ context.Context, _, _, file, ref string) (catalogue.Manifest, bool, error) { - if ref == "main" { - return catalogue.Manifest{}, false, nil - } - if file == "modules/x/module.json" { - return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}}, - Invokes: []string{"seat:s.b"}}, true, nil - } - return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}}, - Invokes: []string{"seat:s.a"}}, true, nil - } - m := orderMember{ID: "novox/r@" + head[:12], Repository: "novox/r", Base: "main", Head: head, Number: 5, - Paths: []string{"modules/x/module.json", "modules/y/module.json"}} - c, cyclic, err := cycleVerdict(context.Background(), "", []orderMember{m}, map[string]orderReach{m.ID: {Moved: []string{"mx", "my"}}}, nil, nil) - if err != nil || !cyclic || c.Verdict != "fail" || c.Group != "" || len(c.Members) != 0 || !strings.Contains(c.Summary, "s.a") || - !strings.Contains(c.Summary, "my") { - t.Fatalf("the verdict is %+v (cyclic %v, %v)", c, cyclic, err) - } -} diff --git a/cmd/mesh-controller/pull_declares.go b/cmd/mesh-controller/pull_declares.go index b639a6b3..726be0ae 100644 --- a/cmd/mesh-controller/pull_declares.go +++ b/cmd/mesh-controller/pull_declares.go @@ -2,7 +2,6 @@ package main import ( "context" - "encoding/json" "fmt" "io" "path" @@ -138,71 +137,6 @@ func tellPullDeclares(ctx context.Context, w io.Writer, pulls []pullAt) { } } -// heldFor asks mesh-config for the approved held values of these pull requests at their heads (ADR 0502 clause 4). Only -// those of the pull requests asked about are carried. A failure is said to w and answers none. -func heldFor(ctx context.Context, w io.Writer, pulls []pullAt) []link.HeldConfig { - if len(pulls) == 0 { - return nil - } - type pullHead struct { - Pull string `json:"pull"` - Head string `json:"head"` - } - asked := make([]pullHead, 0, len(pulls)) - for _, p := range pulls { - asked = append(asked, pullHead{p.key(), p.New}) - } - raw, err := meshConfigAsk(ctx, map[string]any{"action": "held", "pulls": asked}) - if err != nil { - fmt.Fprintf(w, "mesh-config cannot say what it holds for these pull requests (%v); the check goes on without held values\n", err) - return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess - } - var answer struct { - Held []link.HeldConfig `json:"held"` - Skipped []struct { - Pull string `json:"pull"` - Why string `json:"why"` - } `json:"skipped"` - } - if len(raw) == 0 { - return nil - } - if err := json.Unmarshal(raw, &answer); err != nil { - fmt.Fprintf(w, "mesh-config's held values cannot be read (%v); the check goes on without them\n", err) - return nil // empty-on-error: said above; a check without held values fails for want of the value and says so, it never passes on a guess - } - for _, s := range answer.Skipped { - fmt.Fprintf(w, "%s: mesh-config holds a value it does not carry: %s\n", s.Pull, s.Why) - } - var out []link.HeldConfig - for _, h := range answer.Held { - for _, p := range pulls { - if h.Pull == p.key() { - out = append(out, h) - break - } - } - } - return out -} - -// withHeldConfig tells mesh-config what these pull requests declare and puts the approved held values of them on the -// check request (ADR 0502 clause 4). With carry false, a member of a group whose composed check composes, only the -// declarations are told. -func withHeldConfig(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt, carry bool) { - tellPullDeclares(ctx, w, pulls) - if carry { - carryHeld(ctx, w, request, pulls) - } -} - -// carryHeld puts the approved held values of these pull requests on the check request. -func carryHeld(ctx context.Context, w io.Writer, request *link.BuildRequest, pulls []pullAt) { - if request.Check != nil { - request.Check.HeldConfig = heldFor(ctx, w, pulls) - } -} - // holdsModules is whether a module of the mesh is built from this repository. func holdsModules(entries []inventory.Entry, owner, repo string) bool { return slices.ContainsFunc(entries, func(e inventory.Entry) bool { diff --git a/cmd/mesh-controller/pull_declares_test.go b/cmd/mesh-controller/pull_declares_test.go index c325e79a..c136842e 100644 --- a/cmd/mesh-controller/pull_declares_test.go +++ b/cmd/mesh-controller/pull_declares_test.go @@ -9,7 +9,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) const ( @@ -153,50 +152,8 @@ func TestMeshConfigNotAnsweringIsSaidAndGoneOn(t *testing.T) { meshConfigAsk = func(context.Context, map[string]any) ([]byte, error) { return nil, io.ErrUnexpectedEOF } var said strings.Builder tellPullDeclares(context.Background(), &said, []pullAt{thePull()}) - got := heldFor(context.Background(), &said, []pullAt{thePull()}) - if len(got) != 0 || !strings.Contains(said.String(), "mesh-config") { - t.Fatalf("held %v, said %q", got, said.String()) - } -} - -// 1826/AC2: the held values mesh-config answers for the check's pull requests are what the check request carries. -func TestTheHeldValuesOfTheChecksPullRequestsAreAsked(t *testing.T) { - calls := fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","head":"` + headSHA + - `","node":"ace","module":"n8n","key":"media-gid","value":2000,"approved":"the operator"}]}`}) - other := pullAt{Number: 7, changeSource: changeSource{Owner: "novox", Repo: "mesh-controller", New: baseSHA}} - got := heldFor(context.Background(), io.Discard, []pullAt{thePull(), other}) - if len(got) != 1 || got[0].Node != "ace" || got[0].Key != "media-gid" || string(got[0].Value) != "2000" { - t.Fatalf("got %+v", got) - } - c := (*calls)[0] - pulls, _ := json.Marshal(c["pulls"]) - if c["action"] != "held" || string(pulls) != `[{"pull":"novox/mesh-catalog#434","head":"`+headSHA+ - `"},{"pull":"novox/mesh-controller#7","head":"`+baseSHA+`"}]` { - t.Fatalf("asked %v %s", c, pulls) - } -} - -// 1826/AC2: a value held for a pull request the check is not of is never carried. -func TestAHeldValueOfAnotherPullRequestIsNotCarried(t *testing.T) { - fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#999","head":"x","node":"ace","module":"n8n","key":"k","value":1}]}`}) - if got := heldFor(context.Background(), io.Discard, []pullAt{thePull()}); len(got) != 0 { - t.Fatalf("carried %+v", got) - } -} - -// 1826/AC2: the check request carries what mesh-config answered; a member of a composed group asks none. -func TestTheCheckRequestCarriesTheHeldValues(t *testing.T) { - fakeManifests(t, map[string]*catalogue.Manifest{baseSHA: manifest("n8n", nil), headSHA: manifest("n8n", nil)}) - fakeConfig(t, map[string]string{"held": `{"held":[{"pull":"novox/mesh-catalog#434","node":"ace","module":"n8n","key":"media-gid","value":2000}]}`}) - request := link.BuildRequest{Check: &link.CheckRequest{}} - withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, true) - if len(request.Check.HeldConfig) != 1 || request.Check.HeldConfig[0].Key != "media-gid" { - t.Fatalf("the request carries %+v", request.Check.HeldConfig) - } - request = link.BuildRequest{Check: &link.CheckRequest{}} - withHeldConfig(context.Background(), io.Discard, &request, []pullAt{thePull()}, false) - if len(request.Check.HeldConfig) != 0 { - t.Fatalf("a composed member carries %+v", request.Check.HeldConfig) + if !strings.Contains(said.String(), "mesh-config") { + t.Fatalf("said %q", said.String()) } } From 45d091778c844b1db4d65a63eb5060554af7719e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:26:37 +0200 Subject: [PATCH 20/38] =?UTF-8?q?Refuse=20delivery-plan=20and=20forget-che?= =?UTF-8?q?cks=20by=20name=20and=20take=20a=20merge's=20warrant=20from=20t?= =?UTF-8?q?he=20teller=20alone,=20no=20longer=20mesh-delivery's=20(issue?= =?UTF-8?q?=201318,=20C3,=20ADR=200450=20=C2=A78)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/catch_up.go | 10 +-- cmd/mesh-controller/catch_up_test.go | 40 ++------- cmd/mesh-controller/check_builds.go | 24 ------ cmd/mesh-controller/delivery.go | 25 ++---- cmd/mesh-controller/delivery_test.go | 2 +- .../first_builds_integration_test.go | 2 +- cmd/mesh-controller/main.go | 2 +- cmd/mesh-controller/moved.go | 25 +++--- cmd/mesh-controller/moved_asker_test.go | 15 ++-- cmd/mesh-controller/moved_bus_test.go | 3 +- cmd/mesh-controller/moved_test.go | 5 +- cmd/mesh-controller/new_module_bus_test.go | 3 +- cmd/mesh-controller/plans_data_test.go | 85 ------------------- cmd/mesh-controller/seatverbs.go | 20 ++--- cmd/mesh-controller/seatverbs_test.go | 11 ++- 15 files changed, 59 insertions(+), 213 deletions(-) diff --git a/cmd/mesh-controller/catch_up.go b/cmd/mesh-controller/catch_up.go index f3f40bb7..f69e4afc 100644 --- a/cmd/mesh-controller/catch_up.go +++ b/cmd/mesh-controller/catch_up.go @@ -241,10 +241,9 @@ func mergeAskPrefix(number int, head string) string { return fmt.Sprintf("merge-%d-%s-", number, head) } -// askersOfMerges are the askers a merge's warrant is read under: the teller, and mesh-delivery for a merge before release 3e -// (ADR 0437 §6). Release 4's controller work stops accepting mesh-delivery (ADR 0450 §8); a merge asked under it earlier is -// still found here until then. -func askersOfMerges() []string { return []string{sdkdelivery.TellerModule, theDelivery} } +// askersOfMerges are the askers a merge's warrant is read under: the teller alone (ADR 0437 §6), since R4e stopped taking +// mesh-delivery's (ADR 0450 §8; issue 1318). +func askersOfMerges() []string { return []string{sdkdelivery.TellerModule} } // caughtUpWarrant judges a chosen merge ask of the router as the warrant of a merge: the ask is the operator's, the warrant // answers that very ask as it was published, and it is the operator's Approve proved by the channel it came through (ADR 0437 @@ -391,9 +390,6 @@ var findMergeWarrant = func(ctx context.Context, inv catalogueReader, owner, rep if asker == sdkdelivery.TellerModule && !strings.HasPrefix(id, sdkdelivery.AskKindMerge+"-") { continue } - if asker == theDelivery && !named { - continue - } ask, w, err := closedRecordOf(ctx, inv, asker, id) if err != nil { return "", false, "", err diff --git a/cmd/mesh-controller/catch_up_test.go b/cmd/mesh-controller/catch_up_test.go index b120a1cd..eb7be1bb 100644 --- a/cmd/mesh-controller/catch_up_test.go +++ b/cmd/mesh-controller/catch_up_test.go @@ -69,12 +69,12 @@ func TestAForgeThatDoesNotAnswerIsAnErrorNotAnEmptyTrunk(t *testing.T) { func TestACaughtUpMergeIsAcceptedOnTheSameWarrantAsAnAnnouncedOne(t *testing.T) { ok := approvedMerge(t) - for _, asker := range []string{sdkdelivery.TellerModule, theDelivery} { - w := ok.warrant - w.Asker = asker - if _, err := caughtUpWarrant(asker, &ok.ask, &w); err != nil { - t.Errorf("the operator's Approve under %s was refused: %v", asker, err) - } + if _, err := caughtUpWarrant(sdkdelivery.TellerModule, &ok.ask, &ok.warrant); err != nil { + t.Errorf("the operator's Approve under the teller was refused: %v", err) + } + // 1318 (ADR 0450 §8): mesh-delivery's warrant of before the gate is no longer taken. + if _, err := caughtUpWarrant(theDelivery, &ok.ask, &ok.warrant); err == nil { + t.Error("a warrant under mesh-delivery was accepted") } if _, err := caughtUpWarrant("keycloak", &ok.ask, &ok.warrant); err == nil { t.Error("a warrant of an asker the controller takes none from was accepted") @@ -229,34 +229,6 @@ func TestAGroupsWarrantIsFoundByTheAsksWordsForThePullRequestAndItsHead(t *testi } } -// Keycloak's merge was asked as mesh-delivery.merge-378-b9104454f9a5-1, before release 3e: the id names the pull request and -// the first twelve digits of its head, and the warrant is found by them under mesh-delivery. -func TestAWarrantAskedUnderMeshDeliveryIsFoundByThePullRequestAndItsHead(t *testing.T) { - ok := approvedMerge(t) - head := "b9104454f9a5e0c4d3b2a1f0e9d8c7b6a5948372" - askID := "merge-378-b9104454f9a5-1" - ask, w := ok.ask, ok.warrant - ask.ID, w.Ask, w.Asker = askID, askID, theDelivery - w.AskDigest = ask.Digest() - withRouter(t, map[string]map[string]aRouterRecord{theDelivery: { - "merge-377-aaaaaaaaaaaa-1": {}, - askID: {ask, w}, - }}) - why, found, reason, err := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 378, head) - if err != nil || !found || !strings.Contains(why, askID) { - t.Fatalf("the warrant was not found: %q %v %q %v", why, found, reason, err) - } - // Another head names another ask, and none is found. - if _, found, _, _ := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 378, "ffffffffffff0000"); found { - t.Fatal("another head found a warrant") - } - // A router that cannot be read is an error, never "no warrant". - routerAskIDs = func(context.Context, catalogueReader, string) ([]string, error) { return nil, errors.New("no answer") } - if _, found, _, err := findMergeWarrant(t.Context(), nil, "novox", "mesh-catalog", 378, head); found || err == nil { - t.Fatalf("an unreadable router read as no warrant: %v %v", found, err) - } -} - // A trunk shorter than a page, or with none, is read without a fault: the pages end when the forge's do. func TestAShortTrunkIsReadWholeAndAnEmptyOneIsNoMerges(t *testing.T) { was := trunkCommitsPage diff --git a/cmd/mesh-controller/check_builds.go b/cmd/mesh-controller/check_builds.go index b1f78938..51b1c0dd 100644 --- a/cmd/mesh-controller/check_builds.go +++ b/cmd/mesh-controller/check_builds.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "os" - "strconv" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/inventory" @@ -62,26 +61,3 @@ func forgetChecksOfWalk(ctx context.Context, inv *inventory.Inventory, p invento fmt.Printf("%s: %s\n", p.ID, said) } } - -// forgetChecksCommand is `forget-checks `: the verb the delivery's owner asks when the forge -// says a pull request closed unmerged (novox/hq ADR 0350 §5). -func forgetChecksCommand(ctx context.Context, args []string) error { - if len(args) != 3 { - return errors.New("forget-checks ") - } - number, err := strconv.Atoi(args[2]) - if err != nil || number <= 0 { - return fmt.Errorf("%q is not a pull request's number", args[2]) - } - open, err := openStores(ctx) - if err != nil { - return err - } - defer open.Close() - said, err := forgetChecks(ctx, open.inventory, builder.PullRef{Owner: args[0], Repo: args[1], Number: number}) - if err != nil { - return err - } - fmt.Println(said) - return nil -} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 4e102431..17a9a16c 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -815,16 +815,14 @@ func deliveryPlanDocument(m orderMember, entries []inventory.Entry, read map[str // delivery walks [-n 50] [--plan ] func deliveryCommand(ctx context.Context, args []string) error { if len(args) == 0 { - return errors.New("delivery plan|order|check|go|stop|walks") + return errors.New("delivery order|check|go|stop|walks") } sub, rest := args[0], args[1:] + if sub == "plan" { + // Removed (ADR 0450 §8; issue 1318): refused by name before any flag is read, naming its replacement. + return removedByName("delivery-plan") + } set := flag.NewFlagSet("delivery "+sub, flag.ContinueOnError) - repository := set.String("repository", "", "owner/repository") - base := set.String("base", "main", "the branch it merges into") - head := set.String("head", "", "the commit at hand") - paths := set.String("paths", "", "the files it changes, comma-separated") - moduleDirs := set.String("module-dirs", "", "the directories holding a module.json at the head, comma-separated") - removed := set.String("removed", "", "the files it deletes, comma-separated") membersJSON := set.String("members", "", "a group's members, as JSON") membersFile := set.String("members-file", "", "a file of a group's members, as JSON: the same as --members, for a list "+ "too long for an argument (novox/hq issue 1272)") @@ -872,19 +870,6 @@ func deliveryCommand(ctx context.Context, args []string) error { return enc.Encode(v) } switch sub { - case "plan": - if *repository == "" || *head == "" && *paths == "" { - return errors.New("delivery plan --repository owner/repo --head --paths a,b") - } - m := changeMember(*repository, *base, *head, splitList(*paths), splitList(*removed), moduleDirsOf(*moduleDirs)) - entries, read, edges, err := theGraph(ctx, inv) - if err != nil { - return err - } - return answer(deliveryPlanDocument(m, entries, read, edges, func(module string) (inventory.Upgrade, bool) { - u, err := inv.UpgradeOf(ctx, module) - return u, err == nil - })) case "order": if len(members) == 0 { return errors.New("delivery order --members : a group has members") diff --git a/cmd/mesh-controller/delivery_test.go b/cmd/mesh-controller/delivery_test.go index a3127130..e21c319e 100644 --- a/cmd/mesh-controller/delivery_test.go +++ b/cmd/mesh-controller/delivery_test.go @@ -139,7 +139,7 @@ func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { {"delivery-check", map[string]any{"group": "feat/x", "members": "[]"}, []string{"delivery", "check", "--group", "feat/x", "--members", "[]"}}, {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"}, - []string{"delivery", "plan", "--repository", "novox/a", "--paths", "x", "--head", "c0"}}, + []string{"delivery", "plan"}}, {"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}}, } { got, err := argvFor(c.verb, c.args) diff --git a/cmd/mesh-controller/first_builds_integration_test.go b/cmd/mesh-controller/first_builds_integration_test.go index bab2559b..4608f763 100644 --- a/cmd/mesh-controller/first_builds_integration_test.go +++ b/cmd/mesh-controller/first_builds_integration_test.go @@ -64,7 +64,7 @@ func TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey(t *tes a := aDeliveryMergeOf(t, head, map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": head}) if c.approved { record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant}) - if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil { + if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil { t.Fatal(err) } } diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 9de2b094..9eab8a5e 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -108,7 +108,7 @@ func run() error { case "kill": return killCommand(ctx, args[1:]) case "forget-checks": - return forgetChecksCommand(ctx, args[1:]) + return removedByName("forget-checks") case "try": return tryCommand(ctx, args[1:]) case "pause", "resume": diff --git a/cmd/mesh-controller/moved.go b/cmd/mesh-controller/moved.go index d1e97efc..6c07c210 100644 --- a/cmd/mesh-controller/moved.go +++ b/cmd/mesh-controller/moved.go @@ -52,23 +52,22 @@ type deliveryMerge struct { Asker string `json:"asker,omitempty"` } -// askerOf is the asker whose warrant the merge is read under: the one the announcement names, mesh-delivery when it -// names none. -func (d deliveryMerge) askerOf() string { - if d.Asker == "" { - return theDelivery - } - return d.Asker -} +// askerOf is the asker whose warrant the merge is read under: the one the announcement names. One that names none is +// from mesh-delivery of before the gate, whose warrant is no longer taken (novox/hq ADR 0450 §8; issue 1318). +func (d deliveryMerge) askerOf() string { return d.Asker } -// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller, and mesh-delivery -// until the gate takes the seat over (novox/hq ADR 0437 §6). Any other registers no move. +// askerAccepted is whether the controller takes a warrant from this asker for a merge: the teller alone, since the gate +// took the delivery seat over (novox/hq ADR 0437 §6, ADR 0450 §8; issue 1318). Any other registers no move. func askerAccepted(asker string) error { - if asker == theDelivery || asker == sdkdelivery.TellerModule { + if asker == sdkdelivery.TellerModule { return nil } - return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s and %s", - asker, sdkdelivery.TellerModule, theDelivery) + if asker == "" { + return fmt.Errorf("the merge names no asker, as mesh-delivery's of before the gate did, and the controller takes "+ + "a warrant only from %s (ADR 0450 §8)", sdkdelivery.TellerModule) + } + return fmt.Errorf("the merge names the asker %q, and the controller takes a warrant only from %s (ADR 0450 §8)", + asker, sdkdelivery.TellerModule) } // parseDeliveryMerge reads an announcement, refusing one that names no pull request, head or ask. diff --git a/cmd/mesh-controller/moved_asker_test.go b/cmd/mesh-controller/moved_asker_test.go index 9c6c3e37..2fd529fb 100644 --- a/cmd/mesh-controller/moved_asker_test.go +++ b/cmd/mesh-controller/moved_asker_test.go @@ -48,18 +48,19 @@ func TestAMergeNamingTheTellerIsMovedOnTheTellersWarrant(t *testing.T) { } } -// 1213/AC2: an announcement naming mesh-delivery, or none (today's module): read under mesh-delivery, registered. -func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) { +// 1318 (ADR 0450 §8): a merge naming no asker, or the asker mesh-delivery of before the gate, registers no move; the +// teller's is taken, and its warrant read under its name. +func TestAMergeOfMeshDeliveryOrNoAskerMovesNothing(t *testing.T) { for _, named := range []string{"", theDelivery} { a := askedBy(t, named, theDelivery) - if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err != nil { - t.Errorf("asker %q: refused: %v", named, err) + if _, err := movedOnApprove(a.merge, &a.ask, &a.warrant); err == nil || !strings.Contains(err.Error(), "ADR 0450 §8") { + t.Errorf("asker %q: %v", named, err) } } was := routerAskOf wasKept := keptDeliveryMerge t.Cleanup(func() { routerAskOf, keptDeliveryMerge = was, wasKept }) - a := askedBy(t, "", theDelivery) + a := askedBy(t, sdkdelivery.TellerModule, sdkdelivery.TellerModule) keptDeliveryMerge = func(context.Context, string, string, int, string) (deliveryMerge, bool, error) { return a.merge, true, nil } @@ -71,8 +72,8 @@ func TestAMergeNamingNoAskerIsMovedOnMeshDeliverysWarrant(t *testing.T) { if _, err := approvedAnnounced(t.Context(), nil, "novox", "mesh-controller", announcedPull(7, a.merge.HeadSHA)); err != nil { t.Fatal(err) } - if readUnder != theDelivery { - t.Errorf("with no asker named the warrant was read under %q, not %q", readUnder, theDelivery) + if readUnder != sdkdelivery.TellerModule { + t.Errorf("the warrant was read under %q, not %q", readUnder, sdkdelivery.TellerModule) } } diff --git a/cmd/mesh-controller/moved_bus_test.go b/cmd/mesh-controller/moved_bus_test.go index e8aebce2..277295bd 100644 --- a/cmd/mesh-controller/moved_bus_test.go +++ b/cmd/mesh-controller/moved_bus_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "testing" "time" @@ -68,7 +69,7 @@ func TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTh a.merge.Owner, a.merge.Repo, a.merge.Number = "novox", "mesh-host", 7 if c.answer { record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant}) - if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil { + if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil { t.Fatal(err) } } diff --git a/cmd/mesh-controller/moved_test.go b/cmd/mesh-controller/moved_test.go index 1a487280..5fa9d85d 100644 --- a/cmd/mesh-controller/moved_test.go +++ b/cmd/mesh-controller/moved_test.go @@ -1,6 +1,7 @@ package main import ( + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "strings" "testing" "time" @@ -21,7 +22,7 @@ func approvedMerge(t *testing.T) anApprovedMerge { heads := map[string]string{"novox/mesh-controller@aaaaaaaaaaaa": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", "novox/mesh-catalog@bbbbbbbbbbbb": "b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2"} m := deliveryMerge{Owner: "novox", Repo: "mesh-controller", Number: 7, HeadSHA: "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1", - Group: "g1", AskID: "merge-g1", Option: optionApprove, Heads: heads} + Group: "g1", AskID: "merge-g1", Option: optionApprove, Heads: heads, Asker: sdkdelivery.TellerModule} bind := func(option string, h map[string]string) string { d, err := asks.ActDigest(mergeAct(m.AskID, option, h)) if err != nil { @@ -35,7 +36,7 @@ func approvedMerge(t *testing.T) anApprovedMerge { Options: []asks.Option{ {ID: optionApprove, Label: "Approve", Does: "merges the group", Level: asks.Approve, Binds: bind(optionApprove, heads)}, {ID: "decline", Label: "Decline", Does: "leaves it", Level: asks.Approve, Binds: bind("decline", heads)}}} - w := asks.Warrant{Ask: m.AskID, Asker: theDelivery, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve", + w := asks.Warrant{Ask: m.AskID, Asker: sdkdelivery.TellerModule, Outcome: asks.OutcomeChosen, Option: optionApprove, Label: "Approve", Level: asks.Approve, By: &asks.Person{Who: asks.Operator, Kind: "telegram", Identity: "1", Verified: "user id"}, At: at, AskDigest: ask.Digest()} return anApprovedMerge{merge: m, ask: ask, warrant: w} diff --git a/cmd/mesh-controller/new_module_bus_test.go b/cmd/mesh-controller/new_module_bus_test.go index 05f65214..03ff6e9a 100644 --- a/cmd/mesh-controller/new_module_bus_test.go +++ b/cmd/mesh-controller/new_module_bus_test.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" "testing" "time" @@ -69,7 +70,7 @@ func TestANewModuleFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheB map[string]string{"novox/mesh-delivery@a1a1a1a1a1a1": "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1"}) if c.answer { record, _ := json.Marshal(map[string]any{"state": "closed", "ask": a.ask, "warrant": a.warrant}) - if _, err := records.Put(ctx, theDelivery+"."+a.merge.AskID, record); err != nil { + if _, err := records.Put(ctx, sdkdelivery.TellerModule+"."+a.merge.AskID, record); err != nil { t.Fatal(err) } } diff --git a/cmd/mesh-controller/plans_data_test.go b/cmd/mesh-controller/plans_data_test.go index 7250c300..f7fc58c0 100644 --- a/cmd/mesh-controller/plans_data_test.go +++ b/cmd/mesh-controller/plans_data_test.go @@ -1,13 +1,10 @@ package main import ( - "encoding/json" - "reflect" "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/inventory" ) // flagsOf reads a command line's flags (and its first positional after --what-if) into a map. @@ -28,74 +25,6 @@ func flagsOf(argv []string) map[string]string { return out } -// **The plan of a change as data from `plans`** (novox/hq ADR 0450 §2, issue 1324 AC1): asked for one change -// both ways, `plans` with data and `delivery-plan` answer the same plan in every field. -func TestPlansWhatIfAsDataAnswersWhatDeliveryPlanAnswers(t *testing.T) { - change := map[string]any{"repository": "novox/mesh-catalog", "paths": "modules/gitea/index.ts,modules/gitea/old.txt", - "head": "abc123", "base": "main", "module-dirs": "modules/gitea", "removed": "modules/gitea/old.txt"} - with := func(extra map[string]any) map[string]any { - out := map[string]any{} - for k, v := range change { - out[k] = v - } - for k, v := range extra { - out[k] = v - } - return out - } - plansArgv, err := argvFor("plans", with(map[string]any{"data": "true"})) - if err != nil { - t.Fatal(err) - } - deliveryArgv, err := argvFor("delivery-plan", change) - if err != nil { - t.Fatal(err) - } - if plansArgv[0] != "plans" || !slicesHas(plansArgv, "--data") || !slicesHas(plansArgv, "--what-if") { - t.Fatalf("plans with data became %q", strings.Join(plansArgv, " ")) - } - p, d := flagsOf(plansArgv), flagsOf(deliveryArgv) - for _, name := range []string{"paths", "head", "base", "module-dirs", "removed"} { - if p[name] == "" || p[name] != d[name] { - t.Errorf("--%s: plans says %q, delivery-plan %q", name, p[name], d[name]) - } - } - if p["what-if"] != d["repository"] { - t.Errorf("repository: plans says %q, delivery-plan %q", p["what-if"], d["repository"]) - } - - // Both commands build their document from the one function, from the same member. - const src = "http://forge.internal:20000/novox/mesh-catalog.git" - gitea := fromRepo("gitea", src, "modules/gitea") - gitea.Source.BuiltFrom = "old" - gitea.On = []string{"anchor", "laptop"} - entries := []inventory.Entry{gitea} - policy := func(string) (inventory.Upgrade, bool) { return inventory.Upgrade{RollOut: true}, true } - member := func(f map[string]string, repository string) orderMember { - return changeMember(repository, f["base"], f["head"], splitList(f["paths"]), splitList(f["removed"]), - moduleDirsOf(f["module-dirs"])) - } - a := deliveryPlanDocument(member(p, p["what-if"]), entries, nil, nil, policy) - b := deliveryPlanDocument(member(d, d["repository"]), entries, nil, nil, policy) - ja, _ := json.Marshal(a) - jb, _ := json.Marshal(b) - if !reflect.DeepEqual(ja, jb) { - t.Errorf("the two plans differ:\n%s\n%s", ja, jb) - } - plan := a["plan"].(interface{}) - raw, _ := json.Marshal(plan) - var fields map[string]any - _ = json.Unmarshal(raw, &fields) - for _, want := range []string{"moved", "tiers", "machines", "summary", "class", "head", "base", "repository"} { - if _, ok := fields[want]; !ok { - t.Errorf("the plan has no %q: %s", want, raw) - } - } - if fields["head"] != "abc123" || fields["base"] != "main" { - t.Errorf("head and base are %v and %v", fields["head"], fields["base"]) - } -} - // Without the data form the text answer is unchanged (AC2): no data flag, and the data-only arguments are not passed. func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) { argv, err := argvFor("plans", map[string]any{"repository": "novox/x", "paths": "a,b"}) @@ -114,20 +43,6 @@ func TestPlansWhatIfWithoutDataIsUnchanged(t *testing.T) { } } -// delivery-plan stays and answers as before (AC3, ADR 0450 §8). -func TestDeliveryPlanStaysAsItWas(t *testing.T) { - argv, err := argvFor("delivery-plan", map[string]any{"repository": "novox/x", "paths": "a", "head": "h"}) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(argv, " "); got != "delivery plan --repository novox/x --paths a --head h" { - t.Errorf("delivery-plan reads %q", got) - } - if _, ok := catalogueVerb("delivery-plan"); !ok { - t.Error("delivery-plan is gone") - } -} - func catalogueVerb(name string) (catalogue.Verb, bool) { return controllerVerb(name) } func slicesHas(s []string, v string) bool { diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 3b86f5e3..2f73d897 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -412,16 +412,9 @@ func (a *verbArguments) commandLine() ([]string, error) { return argv, nil // The delivery's owner's verbs (novox/hq ADR 0239). case "delivery-plan": - if err := need("repository", "paths"); err != nil { - return nil, err - } - argv := []string{"delivery", "plan", "--repository", str("repository"), "--paths", str("paths")} - for _, flag := range []string{"head", "base", "module-dirs", "removed"} { - if v := str(flag); v != "" { - argv = append(argv, "--"+flag, v) - } - } - return argv, nil + // Removed (ADR 0450 §8; issue 1318): the command refuses it by name, whatever was asked. + a.passOver() + return []string{"delivery", "plan"}, nil case "delivery-order": if err := need("members"); err != nil { return nil, err @@ -493,10 +486,9 @@ func (a *verbArguments) commandLine() ([]string, error) { // As data, the table its output (novox/hq issue 455). return []string{"queue", "--json"}, nil case "forget-checks": - if err := need("owner", "repo", "number"); err != nil { - return nil, err - } - return []string{"forget-checks", str("owner"), str("repo"), str("number")}, nil + // Removed (ADR 0503 §3; issue 1318): the command refuses it by name, whatever was asked. + a.passOver() + return []string{"forget-checks"}, nil case "try": // A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered. if id := str("id"); id != "" { diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 49756a89..3b88d5c5 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -126,18 +126,25 @@ func TestTheRemovedVerbsAreRefusedByName(t *testing.T) { {"build", map[string]any{"on": "mesh-tools"}, "build"}, {"rebuild", map[string]any{"what": "gitea"}, "rebuild"}, {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, "replay build-1 --register"}, + // 1318: delivery-plan and forget-checks go too. + {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x"}, "delivery plan"}, + {"forget-checks", map[string]any{"owner": "novox", "repo": "a", "number": "7"}, "forget-checks"}, } { argv, err := argvFor(c.verb, c.args) if err != nil || strings.Join(argv, " ") != c.want { t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want) } } - for _, removed := range []string{"build", "rebuild"} { + for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery-plan"} { err := removedByName(removed) - if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 0501") { + if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 05") { t.Errorf("%s: %v", removed, err) } } + if err := deliveryCommand(context.Background(), []string{"plan", "--repository", "novox/a"}); err == nil || + !strings.Contains(err.Error(), "`delivery-plan` is removed") { + t.Errorf("delivery plan: %v", err) + } err := replayCommand(context.Background(), []string{"build-1", "--register"}) if err == nil || !strings.Contains(err.Error(), "`replay --register` is removed") { t.Errorf("replay --register: %v", err) From b6ed18197b316d78a2d0fc5cd692c97d9eb5a150 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:34:44 +0200 Subject: [PATCH 21/38] Vendor the R4e mesh-sdk member's head with the states announcement (issue 1790; re-pinned to the final head in E) --- go.mod | 2 +- go.sum | 2 + internal/broker/states_agreement_test.go | 67 ------- testdata/beside/CAPTURED | 2 +- .../novox/mesh-sdk/go/announce/announce.go | 126 +++++++++++++ .../novox/mesh-sdk/go/buildstep/blob.go | 170 +++++++++++++++++ .../novox/mesh-sdk/go/buildstep/isolate.go | 175 ++++++++++++++++++ .../novox/mesh-sdk/go/buildstep/request.go | 115 ++++++++++++ .../novox/mesh-sdk/go/buildstep/trees.go | 58 +++++- .../novox/mesh-sdk/go/catalogue/build.go | 27 +++ .../mesh-sdk/go/catalogue/delivery_seat.go | 3 +- .../go/catalogue/predefined_inputs.go | 41 ++++ .../mesh-sdk/go/catalogue/seats_declared.go | 7 - .../novox/mesh-sdk/go/catalogue/state.go | 43 ----- .../novox/mesh-sdk/go/catalogue/verbs.go | 78 +++----- .../novox/mesh-sdk/go/delivery/builder.go | 117 +++++++++++- .../novox/mesh-sdk/go/delivery/gate.go | 4 +- .../novox/mesh-sdk/go/delivery/namedtree.go | 47 +++++ .../novox/mesh-sdk/go/delivery/preview.go | 8 +- .../novox/mesh-sdk/go/delivery/source.go | 97 ++++++++-- .../git.novox.be/novox/mesh-sdk/go/keeps.go | 80 ++++++++ .../novox/mesh-sdk/go/merge-check.sh | 9 +- .../novox/mesh-sdk/go/process/conn.go | 6 + .../novox/mesh-sdk/go/process/keeps.go | 71 +++++++ .../novox/mesh-sdk/go/process/state.go | 3 + .../git.novox.be/novox/mesh-sdk/go/state.go | 3 + .../git.novox.be/novox/mesh-sdk/go/stdio.go | 3 + vendor/modules.txt | 3 +- 28 files changed, 1166 insertions(+), 201 deletions(-) delete mode 100644 internal/broker/states_agreement_test.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/keeps.go create mode 100644 vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go diff --git a/go.mod b/go.mod index fde4f61c..23a119b3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index c8522d52..de414370 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c h1:YdSbkI42 git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b h1:cfJ2LK/etUiV1eL58pnaEjmSi34+EVmO4ud9J7pXKOs= git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/internal/broker/states_agreement_test.go b/internal/broker/states_agreement_test.go deleted file mode 100644 index 87ad8f70..00000000 --- a/internal/broker/states_agreement_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package broker_test - -import ( - "slices" - "testing" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - -// The facts the control plane states are named twice — in the grant that permits them and in the code -// that states them — because `link` imports `broker` and the dependency cannot go the other way. So a -// test keeps them agreeing: a subject the grant omits is refused at the moment the mesh has something -// to say, and one the grant adds that nothing states is authority nobody uses. -// -// An external test package, because it may import both while neither imports the other. -func TestTheFactsTheGrantPermitsAreTheFactsTheMeshStates(t *testing.T) { - if broker.ControllerSeat != link.MeshControllerSeat { - t.Fatalf("the grant is written for the %q seat and the mesh states its facts under %q", - broker.ControllerSeat, link.MeshControllerSeat) - } - // And what is wrong, as it changes, and the self-check's heartbeat (novox/hq to-be 45 §2, §4). - states := append([]string{link.KeyApplied, link.KeyRefused, link.KeyBuiltBefore}, conditions.Events...) - states = append(states, conditions.HeartbeatEvent) - // And a value given by hand, replaced after its module's first good start (novox/hq ADR 0228). - states = append(states, link.KeySecretReplaced) - // And every act a healer takes (novox/hq to-be 45 §7). - states = append(states, link.KeyHealerActed) - // And a build put back after its gate failed (novox/hq ADR 0236). - states = append(states, link.KeyRolledBack) - // And a pull request's merge check, judged (novox/hq to-be 45 §9). - states = append(states, link.KeyChecked) - // And a walk kept in a new state, for the delivery it walks (novox/hq ADR 0239). - states = append(states, link.KeyPlanMoved) - // And a tier sent to a machine, once the send is kept (novox/hq ADR 0406 §5, issue 974). - states = append(states, link.KeySent) - // And a key of a record mirror written or removed (novox/hq ADR 0289, design 54 §3; design 57 F3). - states = append(states, link.KeyAssignmentChanged, link.KeyDeclaredConfigChanged, link.KeyPredefinedChanged) - for _, event := range states { - if !slices.Contains(broker.ControllerStates, event) { - t.Errorf("the mesh states %q and its account may not publish it", event) - } - } - if len(broker.ControllerStates) != len(states) { - t.Errorf("the grant permits %v, which is more than the mesh states", broker.ControllerStates) - } - // The record mirrors a module may read are the ones the bus holds (design 54 §5, design 57 F3). - if !slices.Equal(catalogue.ControllerMirrors, broker.ControllerMirrors()) { - t.Errorf("the catalogue lets a module read the controller's mirrors %v and the bus holds %v", - catalogue.ControllerMirrors, broker.ControllerMirrors()) - } - // **And the seat says it.** A seat carries the protocol of its role (novox/hq ADR 0129), so the - // facts the control plane states are the seat's `emits` — which is what lets anything else declare - // that it consumes them, and what the subject-agreement check reads to know they have an owner. - var declared []string - for _, seat := range catalogue.SeatsWithAProtocol() { - if seat.Name == broker.ControllerSeat { - declared = seat.Emits - } - } - if !slices.Equal(declared, broker.ControllerStates) { - t.Errorf("the %s seat emits %v and the grant permits %v", broker.ControllerSeat, - declared, broker.ControllerStates) - } -} diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index d14a5a7b..e8914254 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk dfc2c858d94ba877caa96b5e0b84200590c1391f conformance/events/module-event.json +mesh-sdk e76a5a31a07bedb0cfa1e1244e54b391a33f9b75 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go b/vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go new file mode 100644 index 00000000..37dfd21d --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/announce/announce.go @@ -0,0 +1,126 @@ +// Package announce is how a module says on the bus which states it keeps and which of them others may read (novox/hq +// ADR 0495 clauses 2 and 3; issue 1790): the subject it says it on, the value it says, and where the controller keeps +// the last one said by each module. What a module announced is the truth of what it keeps; a reader is granted a read +// from it, never from a list or from the owner's manifest. +// +// A module announces from its code (process.Conn.Keeps, stdio.Keeps, the TypeScript SDK's keeps), never from its +// manifest, which only the core reads (clause 1). A module that keeps no state on the bus announces nothing. +package announce + +import ( + "encoding/json" + "fmt" + "regexp" + "sort" +) + +// Version is the version of the value this package writes. +const Version = 1 + +// Event is the event type of an announcement, under the module's own namespace. +const Event = "states" + +// Subject is where module announces what it keeps: its own event subject, mesh.mod..event.states, which its +// account may already publish to and the events stream keeps. +func Subject(module string) string { return "mesh.mod." + module + ".event." + Event } + +// SubjectAll is every module's announcement, what the controller consumes. +const SubjectAll = "mesh.mod.*.event." + Event + +// KeptBucket is the controller's bucket of the last announcement of each module, by Key. +const KeptBucket = "mesh-controller_states-announced" + +var name = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// Key is module's key in KeptBucket: the module itself, which must be one plain name, since a module's name is part of +// its buckets' names (catalogue.StateProblems). +func Key(module string) (string, error) { + if !name.MatchString(module) { + return "", fmt.Errorf("announce: %q is not a module that keeps state: lower-case letters, digits and hyphens", module) + } + return module, nil +} + +// State is one state a module keeps, by its local name, and whether others may read it. Others never write it: a +// state another module writes is the owner's own code's business. +type State struct { + Name string `json:"name"` + Readable bool `json:"readable,omitempty"` +} + +// States is an announcement: the module, the machine whose instance said it, and every state the module keeps, sorted +// by name. An empty list says the module keeps nothing. +type States struct { + V int `json:"v"` + Module string `json:"module"` + Node string `json:"node,omitempty"` + States []State `json:"states"` +} + +// Keeps says whether the module announced keeping the state. +func (s States) Keeps(state string) bool { + for _, k := range s.States { + if k.Name == state { + return true + } + } + return false +} + +// Readable says whether the module announced the state readable by others. +func (s States) Readable(state string) bool { + for _, k := range s.States { + if k.Name == state { + return k.Readable + } + } + return false +} + +func (s *States) check() error { + if s.Module == "" { + return fmt.Errorf("announce: an announcement names no module") + } + if _, err := Key(s.Module); err != nil { + return err + } + seen := map[string]bool{} + for _, k := range s.States { + if !name.MatchString(k.Name) { + return fmt.Errorf("announce: %s announces the state %q, and a state is named locally: lower-case letters, digits and hyphens", s.Module, k.Name) + } + if seen[k.Name] { + return fmt.Errorf("announce: %s announces the state %q twice", s.Module, k.Name) + } + seen[k.Name] = true + } + return nil +} + +// Encode is s as JSON at Version, its states sorted; it refuses a module or a state no bucket could be named by and a +// state announced twice, naming it. +func Encode(s States) ([]byte, error) { + if err := s.check(); err != nil { + return nil, err + } + s.V = Version + s.States = append([]State{}, s.States...) + sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name }) + return json.Marshal(s) +} + +// Decode reads an announcement; it refuses another version and what Encode refuses. +func Decode(data []byte) (States, error) { + var s States + if err := json.Unmarshal(data, &s); err != nil { + return States{}, fmt.Errorf("announce: the announcement is not JSON: %w", err) + } + if s.V != Version { + return States{}, fmt.Errorf("announce: the announcement is version %d, and this reads version %d", s.V, Version) + } + if err := s.check(); err != nil { + return States{}, err + } + sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name }) + return s, nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go new file mode 100644 index 00000000..127c6ace --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/blob.go @@ -0,0 +1,170 @@ +package buildstep + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "syscall" + "time" +) + +// RegistryWait is how long a push waits for an artifact store that refuses connections, longer than its nightly +// collection holds it still (about a minute and a half, issue 457); RegistryFirstPause is the first pause, doubling +// to RegistryMostPause. Variables so a test shortens them. +var ( + RegistryWait = 5 * time.Minute + RegistryFirstPause = time.Second + RegistryMostPause = 10 * time.Second +) + +// BlobReference is where a pushed bundle or archive is, as a done key names it and the controller delivers from +// (novox/hq ADR 0498 clause 1): artifact-store:////blobs/, the store's address left out +// (issue 102). +func BlobReference(module, artifact, digest string) string { + return "artifact-store://" + module + "/" + artifact + "/blobs/" + digest +} + +// PushBlob puts body into the artifact store at registry (host:port) as a blob of repository (/) by +// its digest (`sha256:` and hex), from the build module's own process, after the build step ended (novox/hq ADR 0498 +// clauses 1 and 2): a blob the store already holds is done without an upload; otherwise it is uploaded (POST +// blobs/uploads/, then PUT with the digest). A store refusing connections is waited out up to RegistryWait; any other +// refusal fails, naming the store's status and its own words, for the caller's `not stored:` error. Bytes that are not +// their digest are refused before anything is sent. +func PushBlob(ctx context.Context, client *http.Client, registry, repository, digest string, body []byte) error { + sum := sha256.Sum256(body) + if got := "sha256:" + hex.EncodeToString(sum[:]); got != digest { + return fmt.Errorf("the blob is %s, not the digest %s it was to be pushed as", got, digest) + } + if client == nil { + client = http.DefaultClient + } + base := "http://" + registry + "/v2/" + repository + "/blobs/" + var present bool + if err := waitForStore(ctx, registry, func() error { + var err error + present, err = blobPresent(ctx, client, base+digest) + return err + }); err != nil { + return err + } + if present { + return nil + } + var location string + if err := waitForStore(ctx, registry, func() error { + var err error + location, err = startUpload(ctx, client, base+"uploads/") + return err + }); err != nil { + return err + } + target, err := url.Parse(base) + if err != nil { + return err + } + loc, err := target.Parse(location) + if err != nil { + return fmt.Errorf("the store answered the upload's location %q, which is not a URL: %w", location, err) + } + q := loc.Query() + q.Set("digest", digest) + loc.RawQuery = q.Encode() + req, err := http.NewRequestWithContext(ctx, http.MethodPut, loc.String(), bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/octet-stream") + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("putting %s of %s: %w", digest, repository, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusCreated { + return storeSaid(resp, "putting "+digest+" of "+repository) + } + return nil +} + +func blobPresent(ctx context.Context, client *http.Client, at string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodHead, at, nil) + if err != nil { + return false, err + } + resp, err := client.Do(req) + if err != nil { + return false, err + } + resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusNotFound: + return false, nil + } + return false, fmt.Errorf("asking the store for %s: it answered %s", at, resp.Status) +} + +func startUpload(ctx context.Context, client *http.Client, at string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodPost, at, nil) + if err != nil { + return "", err + } + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusAccepted { + return "", storeSaid(resp, "starting an upload at "+at) + } + location := resp.Header.Get("Location") + if location == "" { + return "", fmt.Errorf("starting an upload at %s: the store named no location", at) + } + return location, nil +} + +// storeSaid is the store's refusal: its status and the first of its own words. +func storeSaid(resp *http.Response, what string) error { + words, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return fmt.Errorf("%s: the store answered %s: %s", what, resp.Status, strings.TrimSpace(string(words))) +} + +// refused is whether an error is a connection refused. +func refused(err error) bool { + return err != nil && (errors.Is(err, syscall.ECONNREFUSED) || strings.Contains(err.Error(), "connection refused")) +} + +// waitForStore runs try, and while it fails because the store refuses connections, tries again with a growing pause +// until RegistryWait has passed. A refusal is the one error waited for: nothing was sent, so trying again does nothing +// twice. +func waitForStore(ctx context.Context, registry string, try func() error) error { + err := try() + if !refused(err) { + return err + } + started, pause := time.Now(), RegistryFirstPause + for { + left := RegistryWait - time.Since(started) + if left <= 0 { + return fmt.Errorf("the artifact store at %s refused every connection for %s, longer than its nightly collection holds it still, so it is down, not paused: %w", + registry, time.Since(started).Round(time.Millisecond), err) + } + select { + case <-ctx.Done(): + return fmt.Errorf("stopped while waiting for the artifact store at %s: %w (last: %v)", registry, ctx.Err(), err) + case <-time.After(min(pause, left)): + } + pause = min(pause*2, RegistryMostPause) + if err = try(); !refused(err) { + return err + } + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go new file mode 100644 index 00000000..2efa739e --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/isolate.go @@ -0,0 +1,175 @@ +package buildstep + +import ( + "context" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "git.novox.be/novox/mesh-sdk/go/forgekey" +) + +// Bubblewrap is the program a run step runs in (novox/hq ADR 0448): a build module declares its package, which needs +// unprivileged user namespaces (issue 1308). +const Bubblewrap = "bwrap" + +// Isolation is how a run step's command is shut off (novox/hq ADR 0448; issue 1309, the lead's decision L2 of R4e: one +// runner in the SDK for every build module). The fetch step holds the forge key and the network and runs none of the +// change's code; the run step runs the change's build and tests in bubblewrap with: +// +// - no network: its own network namespace, with only a loopback, so a test may start a bus or a store of its own but +// resolves no name and reaches no socket of the machine; +// - none of the account: not its home, not the forge key, not a bus login, not the runtime directory (/run), since +// none of them is bound; HOME is a directory in the step's own /tmp unless Env names one; +// - an environment of exactly Env and the runner's env: nothing of the build module's own passes; +// - the system's programs and libraries read-only, and of /etc only what a program needs to run (users, libraries, +// certificates, time zone), with a hosts file naming only the loopback; +// - the Workspace and Writable directories read-write, the ReadOnly ones (the toolchain, the modules the fetch step +// downloaded) read-only; +// - the shared warm Caches read-only under a throwaway writable layer (S18), or read-write for a trunk build alone +// (WriteCaches), so a candidate's code never changes what a later build reads. +type Isolation struct { + Workspace string + Writable []string + ReadOnly []string + Caches []string + WriteCaches bool + Env []string + // Hosts is the file bound as /etc/hosts; Isolated writes one naming only the loopback. + Hosts string +} + +// etcNeeded is what of /etc a program in the step may read: who the users are, where the libraries are, the +// certificates, the time zone. Not resolv.conf, not hosts: the step resolves no name. +var etcNeeded = []string{"/etc/passwd", "/etc/group", "/etc/nsswitch.conf", "/etc/ld.so.cache", "/etc/ld.so.conf", + "/etc/ld.so.conf.d", "/etc/ssl", "/etc/ca-certificates", "/etc/pki", "/etc/localtime", "/etc/os-release", "/etc/alternatives"} + +// systemArgs binds the system's programs and libraries read-only: /usr, and /bin, /sbin, /lib, /lib64 as the symlinks +// they are on a merged-usr system, or bound when they are directories. A variable so a unit test fixes it. +var systemArgs = func() []string { + args := []string{"--ro-bind", "/usr", "/usr"} + for _, d := range []string{"/bin", "/sbin", "/lib", "/lib64"} { + info, err := os.Lstat(d) + switch { + case err != nil: + case info.Mode()&os.ModeSymlink != 0: + if target, err := os.Readlink(d); err == nil { + args = append(args, "--symlink", target, d) + } + case info.IsDir(): + args = append(args, "--ro-bind", d, d) + } + } + for _, e := range etcNeeded { + args = append(args, "--ro-bind-try", e, e) + } + return args +} + +// stepHome is the HOME of a step that is given none: inside its own /tmp, gone with it. +const stepHome = "/tmp/home" + +func checkDir(what, dir string) error { + if dir == "" || !strings.HasPrefix(dir, "/") || path.Clean(dir) != dir || dir == "/" { + return fmt.Errorf("buildstep: the %s %q is not an absolute, clean directory other than the root", what, dir) + } + return nil +} + +// Command is the bubblewrap command line that runs name with args shut off as iso says, env added to iso.Env. It +// refuses a directory that is not absolute and clean, or is the root, and an environment word without "=". +func (iso Isolation) Command(name string, args ...string) ([]string, error) { + return iso.command(nil, name, args...) +} + +func (iso Isolation) command(env []string, name string, args ...string) ([]string, error) { + if err := checkDir("workspace", iso.Workspace); err != nil { + return nil, err + } + for _, group := range []struct { + what string + dirs []string + }{{"writable directory", iso.Writable}, {"read-only directory", iso.ReadOnly}, {"cache", iso.Caches}} { + for _, d := range group.dirs { + if err := checkDir(group.what, d); err != nil { + return nil, err + } + } + } + words := append(append([]string{}, iso.Env...), env...) + home := false + for _, w := range words { + k, _, ok := strings.Cut(w, "=") + if !ok || k == "" { + return nil, fmt.Errorf("buildstep: the environment word %q is not NAME=value", w) + } + home = home || k == "HOME" + } + argv := []string{"--unshare-all", "--die-with-parent", "--new-session", "--clearenv", + "--proc", "/proc", "--dev", "/dev", "--tmpfs", "/tmp", "--dir", stepHome} + argv = append(argv, systemArgs()...) + for _, d := range iso.ReadOnly { + argv = append(argv, "--ro-bind", d, d) + } + argv = append(argv, cacheArgs(iso.Caches, iso.WriteCaches)...) + argv = append(argv, "--bind", iso.Workspace, iso.Workspace) + for _, d := range iso.Writable { + argv = append(argv, "--bind", d, d) + } + if iso.Hosts != "" { + argv = append(argv, "--ro-bind", iso.Hosts, "/etc/hosts") + } + for _, w := range words { + k, v, _ := strings.Cut(w, "=") + argv = append(argv, "--setenv", k, v) + } + if !home { + argv = append(argv, "--setenv", "HOME", stepHome) + } + argv = append(argv, "--chdir", iso.Workspace, "--", name) + return append(argv, args...), nil +} + +// cacheArgs binds the shared warm caches (S18): for a trunk build read-write, so the next build is warm; for any other +// read-only beneath an overlay whose writes go to a tmpfs of the step's own, so the step writes as a build does and +// nothing it wrote outlives it (bubblewrap's --tmp-overlay, an overlay mount in the step's user namespace). +func cacheArgs(caches []string, write bool) []string { + var argv []string + for _, c := range caches { + if write { + argv = append(argv, "--bind", c, c) + continue + } + argv = append(argv, "--overlay-src", c, "--tmp-overlay", c) + } + return argv +} + +// loopbackHosts is the hosts file of a step: the loopback and nothing else. +const loopbackHosts = "127.0.0.1 localhost\n::1 localhost\n" + +// Isolated is run with every command shut off as iso says (Command), run's env going inside the step: the build modules' +// run steps call their test, build and digest commands through it, never run itself. Each command gets a hosts file of +// its own, naming only the loopback, removed when it ends. +func Isolated(run forgekey.Runner, iso Isolation) forgekey.Runner { + return func(ctx context.Context, env []string, name string, args ...string) forgekey.Ran { + dir, err := os.MkdirTemp("", "step-hosts-") + if err != nil { + return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} + } + defer os.RemoveAll(dir) + hosts := filepath.Join(dir, "hosts") + if err := os.WriteFile(hosts, []byte(loopbackHosts), 0o644); err != nil { + return forgekey.Ran{Status: 1, Err: "the step's hosts file: " + err.Error()} + } + iso := iso + iso.Hosts = hosts + argv, err := iso.command(env, name, args...) + if err != nil { + return forgekey.Ran{Status: 1, Err: err.Error()} + } + return run(ctx, nil, Bubblewrap, argv...) + } +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go new file mode 100644 index 00000000..c0189dcc --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/request.go @@ -0,0 +1,115 @@ +package buildstep + +import ( + "fmt" + "path" + "regexp" + "strings" +) + +var ( + requestCommitOK = regexp.MustCompile(`^[0-9a-f]{40}$`) + requestRepositoryOK = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$`) +) + +// FetchRequest is what a build module's fetch tool is asked: the repository and the full commit, the pull request's +// ref when only its head carries the commit, the trunk commit to merge onto for a merge preview, and, for a build of a +// module whose every artifact names its paths, that module's directory and its named paths (novox/hq ADR 0499 clause 4). +// +// **The paths come from the builder, not from a manifest** (ADR 0495 clause 1; the lead's decision L8 of R4e): the +// builder fills Dir and Paths from the indexer's Module value (Path, and KeptInputs2.Paths' keys), and the build module +// gives the build and test steps a workspace holding those paths, the manifest and its lock files. A request without +// them builds from the module's directory, as before. +type FetchRequest struct { + Repository string `json:"repository"` + Commit string `json:"commit"` + Ref string `json:"ref,omitempty"` + Base string `json:"base,omitempty"` + Dir string `json:"dir,omitempty"` + Paths []string `json:"paths,omitempty"` +} + +// FetchRequestProperties are the JSON schema properties of FetchRequest's dir and paths, for a fetch tool's Input. +func FetchRequestProperties() map[string]any { + return map[string]any{ + "dir": map[string]any{"type": "string", "description": "the module's directory in the repository (\".\" for its root), with paths"}, + "paths": map[string]any{"type": "array", "items": map[string]any{"type": "string"}, + "description": "the paths the module names, relative to dir, from the indexer's kept inputs: the build sees only these, the manifest and its lock files"}, + } +} + +// Args is r as a tool call's arguments. +func (r FetchRequest) Args() map[string]any { + args := map[string]any{"repository": r.Repository, "commit": r.Commit} + for k, v := range map[string]string{"ref": r.Ref, "base": r.Base, "dir": r.Dir} { + if v != "" { + args[k] = v + } + } + if len(r.Paths) > 0 { + args["paths"] = append([]string(nil), r.Paths...) + } + return args +} + +// ParseFetchRequest reads a fetch tool's arguments; arguments it does not know (a build module's own) are left to the +// caller. It refuses a repository that is not /, a commit or a base that is not 40 hex digits, paths with no +// module directory or a directory with no path, a directory leaving the repository and a path leaving the directory, +// naming each. +func ParseFetchRequest(args map[string]any) (FetchRequest, error) { + var r FetchRequest + r.Repository, _ = args["repository"].(string) + r.Commit, _ = args["commit"].(string) + r.Ref, _ = args["ref"].(string) + r.Base, _ = args["base"].(string) + r.Dir, _ = args["dir"].(string) + switch v := args["paths"].(type) { + case nil: + case []string: + r.Paths = append([]string(nil), v...) + case []any: + for _, p := range v { + s, ok := p.(string) + if !ok { + return FetchRequest{}, fmt.Errorf("paths is a list of strings") + } + r.Paths = append(r.Paths, s) + } + default: + return FetchRequest{}, fmt.Errorf("paths is a list of strings") + } + return r, r.Check() +} + +// Check is what ParseFetchRequest refuses. +func (r FetchRequest) Check() error { + if !requestRepositoryOK.MatchString(r.Repository) || strings.Contains(r.Repository, "..") { + return fmt.Errorf("%q is not a repository, /", r.Repository) + } + if !requestCommitOK.MatchString(r.Commit) { + return fmt.Errorf("%q is not a full commit (40 hex digits)", r.Commit) + } + if r.Base != "" && !requestCommitOK.MatchString(r.Base) { + return fmt.Errorf("%q is not a trunk commit (40 hex digits)", r.Base) + } + switch { + case len(r.Paths) > 0 && r.Dir == "": + return fmt.Errorf("the request names paths and names no module directory (dir) they are relative to") + case len(r.Paths) == 0 && r.Dir != "": + return fmt.Errorf("the request names a module directory and no path") + } + if r.Dir != "" && (outside(r.Dir) || strings.HasPrefix(r.Dir, "/")) { + return fmt.Errorf("the module directory %q leaves the repository", r.Dir) + } + for _, p := range r.Paths { + if p == "" || strings.HasPrefix(p, "/") || outside(p) || path.Clean(p) == "." { + return fmt.Errorf("the path %q leaves the module's directory or is not a path in it", p) + } + } + return nil +} + +func outside(p string) bool { + c := path.Clean(p) + return c == ".." || strings.HasPrefix(c, "../") +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go index 58b2ccd8..fc31c62d 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go @@ -6,9 +6,12 @@ import ( "encoding/hex" "fmt" "os" + "path" "path/filepath" "strings" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "git.novox.be/novox/mesh-sdk/go/delivery" "git.novox.be/novox/mesh-sdk/go/forgekey" ) @@ -19,12 +22,16 @@ type ModuleTree struct { Tree string `json:"tree"` Manifest string `json:"manifest"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the commit; Tree + // is then their delivery.NamedTree with Manifest (novox/hq ADR 0499 clause 2). Absent otherwise. + Paths map[string]string `json:"paths,omitempty"` } var lockFiles = []string{"go.sum", "package-lock.json"} // ModuleTrees are the trees, in commit, of every directory of the workspace dir that holds a module.json, by directory -// (the repository's root is "."). The workspace is at commit, so the manifests and the lock files are read from it. env is +// (the repository's root is "."); for a module whose every artifact names its paths, the named tree of those paths and +// their objects (novox/hq ADR 0499 clause 2). The workspace is at commit, so the manifests and the lock files are read from it. env is // the environment git runs with (nil for none). Both build modules' fetch steps answer it for a merge. func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, commit string) (map[string]ModuleTree, error) { r := run(ctx, env, "git", "-C", dir, "ls-files", "--", "module.json", "*/module.json") @@ -51,6 +58,9 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co return nil, err } mt.Manifest = sha256Hex(raw) + if err := narrow(ctx, run, env, dir, commit, mdir, raw, &mt); err != nil { + return nil, err + } for _, lock := range lockFiles { if raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(mdir), lock)); err == nil { if mt.Locks == nil { @@ -64,4 +74,50 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co return trees, nil } +// narrow sets mt's Paths and its named Tree when the module's manifest names paths on every artifact (novox/hq ADR 0499 +// clause 2): one `git ls-tree -z` gives each named path's object at commit. The merge preview reads the manifest here, as +// the indexer does, and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse +// is not narrowed: the indexer gives it none. A named path absent at commit fails, naming the module and the path. +func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit, mdir string, raw []byte, mt *ModuleTree) error { + m, err := catalogue.ParseManifest(raw) + if err != nil { + return nil + } + paths, every := m.NamedPaths() + if !every { + return nil + } + full := make([]string, len(paths)) + for i, p := range paths { + full[i] = path.Join(mdir, p) + } + r := run(ctx, env, "git", append([]string{"-C", dir, "ls-tree", "-z", commit, "--"}, full...)...) + if r.Status != 0 || r.Err != "" { + return fmt.Errorf("git ls-tree %s of the module %s: %s", commit, mdir, firstLine(r.Err+" "+r.Stderr)) + } + objects := map[string]string{} + for _, l := range strings.Split(r.Stdout, "\x00") { + meta, name, ok := strings.Cut(l, "\t") + fields := strings.Fields(meta) + if !ok || len(fields) != 3 { + continue + } + objects[name] = fields[2] + } + mt.Paths = map[string]string{} + for i, p := range paths { + object := objects[full[i]] + if object == "" { + return fmt.Errorf("the module %s names the path %q, which is not in %s", mdir, p, commit) + } + mt.Paths[p] = object + } + named, err := delivery.NamedTree(mt.Manifest, mt.Paths) + if err != nil { + return fmt.Errorf("the module %s: %w", mdir, err) + } + mt.Tree = named + return nil +} + func sha256Hex(b []byte) string { h := sha256.Sum256(b); return hex.EncodeToString(h[:]) } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go index f4fcd25b..91e34601 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/build.go @@ -399,6 +399,33 @@ func (b *Build) problems(module string) []string { return problems } +// NamedPaths are the paths a module's fingerprint takes its tree from (novox/hq ADR 0499 clause 2): the union of the +// paths its artifacts name, cleaned and sorted, and every true only when the module builds something and every artifact +// names paths; otherwise none, and the module keeps its directory's tree. +// +// **Read by the core alone** (ADR 0495 clause 1, the lead's decision L8 of R4e): the indexer and its merge preview +// (buildstep.ModuleTrees) call it. A build module's fetch step is handed the paths by the builder, from the indexer's +// kept inputs (delivery.KeptInputs2.Paths), and reads no manifest for them. +func (m Manifest) NamedPaths() (paths []string, every bool) { + if m.Build == nil || len(m.Build.Artifacts) == 0 { + return nil, false + } + seen := map[string]bool{} + for _, a := range m.Build.Artifacts { + if len(a.Paths) == 0 { + return nil, false + } + for _, p := range a.Paths { + seen[path.Clean(p)] = true + } + } + for p := range seen { + paths = append(paths, p) + } + sort.Strings(paths) + return paths, true +} + // pathsProblems is what is wrong with an artifact's `paths` (novox/hq ADR 0499 clause 1): named on a kind whose build // source is derived or named otherwise, or a path that leaves the module's directory, is its whole directory, or is a // pattern. diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go index dbc80faf..7df4d868 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/delivery_seat.go @@ -42,7 +42,8 @@ func deliveryVerbs() []Verb { "paths": "the files it changes, comma-separated, from the repository's root", "base": "the branch it merges into (default main)"}, []string{"repository", "paths"})}, {Name: "checks", Description: "What the mesh's checks said of a pull request's head or of one commit: each " + - "of the commit's mesh statuses (mesh/merge-gate, mesh/repo-check, mesh/delivery, …) with its state, " + + "of the commit's mesh statuses (mesh/check, the one a main requires from R4e, hq's mesh/repo-check, " + + "mesh/delivery, …) with its state, " + "description and when it was set; the merge check's full verdict as the controller said it — each " + "layer's summary, the machine that ran it, when, its build id and its report; and whether the branch's " + "protection would let it merge, every required status being success.", diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go new file mode 100644 index 00000000..1552cf93 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/predefined_inputs.go @@ -0,0 +1,41 @@ +package catalogue + +import "encoding/json" + +// ParseManifestForInputs reads a manifest for the inputs of its build and its fingerprint: the indexer's and the +// builder's read (novox/hq issue 1751). It is never a judge of config values. A value falling back to a predefined key +// (ADR 0348 §7) is judged against the list the mesh published when this process holds one (PublishedPredefined); +// where it holds none, as in the indexer and the builder, each key a value names is taken as the value declares it (its +// type, trusted when it is), since the list is read where a manifest is judged: the merge gate from the tree's root, +// the controller from what it published. No input of a fingerprint or a build comes from the list. Everything else +// ParseManifest refuses is refused here too. +func ParseManifestForInputs(raw []byte) (Manifest, error) { + list := PublishedPredefined() + if list == nil { + list = fallbacksOf(raw) + } + return parseManifest(raw, list) +} + +// fallbacksOf is the list of the predefined keys a manifest's config values fall back to, each as the value declares +// it. A manifest that is not JSON gives an empty list, and the parser says what is wrong with it. +func fallbacksOf(raw []byte) PredefinedList { + var m struct { + Config map[string]struct { + Type string `json:"type"` + Description string `json:"description"` + Predefined string `json:"predefined"` + Trusted bool `json:"trusted"` + } `json:"config"` + } + list := PredefinedList{} + if json.Unmarshal(raw, &m) != nil { + return list + } + for _, d := range m.Config { + if d.Predefined != "" { + list[d.Predefined] = PredefinedKey{Type: d.Type, Description: d.Description, Trusted: d.Trusted} + } + } + return list +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go index 2a9ad3fe..457f87e6 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/seats_declared.go @@ -423,13 +423,6 @@ func CatalogueProblems(shelf Shelf) []string { "operator's account, which every agent runs as (novox/hq ADR 0259)", module, why)) } } - // A read of a module's state that module does not keep (novox/hq ADR 0201) — said only where the - // owner is on the shelf, as a consumer may be installed before its emitter. - var manifests []Manifest - for _, module := range shelfOrder(shelf) { - manifests = append(manifests, shelf[module]) - } - problems = append(problems, StateReadsNothingDeclares(manifests)...) sort.Strings(problems) return problems } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go index 87e402f1..9229d3d3 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/state.go @@ -126,46 +126,3 @@ func StateProblems(m Manifest) []string { } return problems } - -// ControllerMirrors are the controller's record mirrors a module may read, each `mesh-controller.` -// (novox/hq ADR 0289, design 54 §5, design 57 F3). **The controller keeps them, not its manifest's `state`**: -// they are asserted with the controller's own buckets, with options of their own, and written by the -// controller's account alone (the writers table); declared as a module's state, they would be asserted a -// second way and granted to the module's own principal as well. Named here so a read of one is a read of -// kept state. The bus's list is broker.ControllerMirrors; a test keeps the two agreeing. -// plans-building is the plans in their build phase, read by the checker to hold checks while a delivery builds (ADR 0483 -// clause 2, issue 1687). -var ControllerMirrors = []string{"assignments", "declared-config", "predefined", "builds", "rollouts", "checks-running", "plans-building"} - -// StateReadsNothingDeclares is every read across a catalogue whose owner is present and declares no -// such state. An absent owner says nothing — a module may be installed long before the one whose -// state it reads, as a consumer may before its emitter (design 32 §1). -func StateReadsNothingDeclares(manifests []Manifest) []string { - declared := map[string]map[string]bool{} - for _, m := range manifests { - own := map[string]bool{} - for _, s := range m.State { - own[s.Name] = true - } - declared[m.Module] = own - } - if own, present := declared[ControllerSeatName]; present { - for _, name := range ControllerMirrors { - own[name] = true - } - } - var problems []string - for _, m := range manifests { - for _, r := range m.Reads { - module, local, err := ReadState(r) - if err != nil { - continue - } - if own, present := declared[module]; present && !own[local] { - problems = append(problems, fmt.Sprintf( - "%s reads %q, and %s keeps no state called %q", m.Module, r, module, local)) - } - } - } - return problems -} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go index 17e417a0..266bac6f 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/catalogue/verbs.go @@ -147,24 +147,16 @@ var ControllerVerbs = []Verb{ }, nil, "data")}, // The verbs the delivery's owner asks with (novox/hq ADR 0239): the planner's answer, a group's order and // composed check, and the start and end of a walk. Callable by a person too; mesh-delivery is who needs them. - {Name: "delivery-plan", Description: "The delivery plan of a diffset, as the planner computes it and keeping " + - "nothing (novox/hq ADR 0239): what it moves and builds after them in tiers, what each machine receives and " + - "what waits there, the steps that are not an ordinary send, and whether the gate would run.", - Input: schema(map[string]string{"repository": "owner/repository", "head": "the commit at hand", - "base": "the branch it merges into (default main)", - "paths": "the files it changes, comma-separated, from the repository's root", - "module-dirs": "the directories holding a module.json at the head, comma-separated, as the forge says them", - "removed": "the files among paths it deletes, comma-separated"}, []string{"repository", "paths"})}, + {Name: "delivery-plan", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, {Name: "delivery-order", Description: "A delivery group's order (novox/hq ADR 0419): its members in the order " + "they are delivered, every pair and why — built by, verb before caller, version skew, engine before " + "controller — read from the module graph and the manifests' change alone; no `after:` line is read. The " + "cycle, with every module and edge of it, is the one refusal.", Input: schema(map[string]string{"members": "the members, as JSON: [{id, repository, base, head, number, paths, " + "module_dirs, module_dirs_said, removed}]"}, []string{"members"})}, - {Name: "delivery-check", Description: "Ask the build seat for a delivery group's composed check (novox/hq ADR " + - "0239): every member's head laid over the mesh in turn and judged as one future state. Answers the ask's id; " + - "the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " + - "again as its pull request is.", + {Name: "delivery-check", Description: "Answers that the checker (mesh-delivery-checker) judges a group (novox/hq " + + "ADR 0503 §3); asks nothing. Its inputs stay, since hq's repository check still names it (ADR 0513 §4).", Input: schema(map[string]string{"group": "the group's id (none for one head checked again)", "members": "the members, as JSON, as delivery-order takes them", "ready": "\"true\" when the pull request is marked ready: its check is queued before every other check (novox/hq ADR 0406 §3)"}, @@ -378,45 +370,33 @@ var ControllerVerbs = []Verb{ {Name: "clear", Description: "Cancel every waiting ask in the build queue — and with dead, every dead one too — each " + "recorded failed, cancelled by hand. Never touches one in flight.", Input: schema(map[string]string{"dead": "\"true\" to cancel the dead asks as well"}, nil, "dead")}, - {Name: "rebuild", Description: "Ask a module's current source again under a new id — the branch it follows — or, given a " + - "build's id, that build's repository, path and ref. A module a plan holds unbuilt or failed joins that plan. Answers the new id.", - Input: schema(map[string]string{"what": "a module's name, or a build's id"}, []string{"what"})}, - {Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id. " + - "A dry run unless register: nothing recorded or registered. Registering is refused when a newer build of the module " + - "is registered — it would roll the older commit out (novox/hq issue 207) — unless older says so.", - Input: schema(map[string]string{ - "id": "the build's id", - "register": "\"true\" to register what it builds", - "older": "\"true\", with register: even though a newer build of the module is registered", - }, []string{"id"}, "register", "older")}, + {Name: "rebuild", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, + {Name: "replay", Description: "Ask a recorded build's repository and path again at the commit it built, under a new id: " + + "a dry run only, nothing recorded or registered (novox/hq ADR 0429 §18: a fingerprint's key is built again by the " + + "builder, never registered by hand).", + Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})}, // novox/hq issue 686: a builder ran a branch's store tests for the first time in its pull request's merge check. - {Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the build " + - "seat merges it into the repository's trunk and runs the repository's own merge-check.sh on the merge, in the " + - "toolchain containers and against the throwaway store and bus a pull request's merge check gets. Answers at " + + {Name: "try", Description: "Try a pushed branch or commit before its pull request (novox/hq issue 686): the " + + "checker (mesh-delivery-checker) merges the head onto the repository's trunk and runs its levels on the merge, " + + "as it judges a pull request (ADR 0503 §3); no merge-check.sh is run and build-agent is not asked. Answers at " + "once with an id; asked again with that id it waits up to 20 s and answers pass, fail or error, how long it " + "took and the lines naming what failed — or that it is still waiting or running, to ask again. Sets no " + - "status on the forge, registers and builds nothing, runs no merge gate, counts as a check in the build queue " + - "(after deliveries and after every pull request's check, however long it waits) and is bounded as one. One " + - "try of a commit at a time — a second ask answers the first's id — and at most four waiting or running in " + - "all: a fifth is refused. Only for a repository of the mesh; the ref must be on the forge.", + "status on the forge, registers and builds nothing for delivery. One try of a commit at a time — a second " + + "ask answers the first's id. Only for a repository of the mesh; the ref must be on the forge.", Input: schema(map[string]string{ "repository": "the repository on the forge, owner/name; with ref", "ref": "the branch or commit to try, as pushed to the forge", - "base": "the branch it is merged into (the repository's default branch when none is named). The " + - "merge-check.sh that runs is that branch's, never the ref's own: the same trust as a pull request's base", + "base": "the branch it is merged into (the repository's default branch when none is named)", "tests": "only the tests this `go test -run` pattern names, where the repository's merge-check.sh reads " + "MESH_CHECK_TESTS; a script that does not runs whole", - "store-only": "\"true\": only the repository's store tests, where its merge-check.sh reads " + - "MESH_CHECK_STORE_ONLY", - "id": "instead of a repository: the id a try answered, to read what came of it", + "store-only": "\"true\": only the integration level", + "id": "instead of a repository: the id a try answered, to read what came of it", }, nil, "store-only"), Replaces: []string{"sh merge-check.sh", "docker run postgres"}}, // novox/hq ADR 0350 §5: asked by the delivery's owner when the forge says a pull request closed unmerged. - {Name: "forget-checks", Description: "Let go of every build a pull request's merge checks made for its merge in " + - "the artifact store's scratch namespace (novox/hq ADR 0350 §5): asked when it closed unmerged; a merged one's go " + - "when the walk answering its merge ends, and any check build goes after a week.", - Input: schema(map[string]string{"owner": "the repository's owner", "repo": "the repository", - "number": "the pull request's number"}, []string{"owner", "repo", "number"})}, + {Name: "forget-checks", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, {Name: "kill", Description: "End a build where it runs: the machine that took it stops its commands and containers and " + "announces it failed, killed by hand — settled, never handed to another machine.", Input: schema(map[string]string{"id": "the build's id"}, []string{"id"})}, @@ -595,22 +575,8 @@ var ControllerVerbs = []Verb{ "machine": "one machine (optional)", "retired": "\"true\": only what is retired", }, nil, "retired")}, - // Three shapes, as the command has them: one repository, every module standing on a base (on), or - // every module behind its source (behind). One per call; a second is refused as passed over. - {Name: "build", Description: "Have the build machine build a repository — or, instead, every module that " + - "stands on a base (on), or every module behind its source (behind). Answers at once with each build's id: " + - "`builds` with an id follows it line by line, and a module is registered when its outcome comes.", - Input: schema(map[string]string{ - "repository": "the repository's URL, or its path on the forge holding the git seat (owner/name); " + - "needed unless on or behind is given", - "path": "with repository: the module's directory inside it (optional)", - "ref": "with repository: the branch, tag or commit to build (optional)", - "on": "instead of a repository: a module's name. Every module the mesh holds that was built against " + - "its artifacts is built again from its own source, bases among them asked first — the rebuild a " + - "changed base needs, which behind does not see because their sources did not move", - "behind": "\"true\", instead of a repository: build every module the mesh holds whose source has moved " + - "past the commit it was built from, bases asked first", - }, nil, "behind")}, + {Name: "build", Description: "Removed in R4e (ADR 0501 §5 / ADR 0503 §3): refused by name, naming its replacement", + Input: schema(nil, nil)}, // What the records say the registries may keep, asked on demand (novox/hq ADR 0251, to-be 51). {Name: "artifacts", Description: "Every artifact the mesh recorded making that lives in its artifact store, " + "each with its repository, digest, kind (image or archive) and state: kept — and why: a definition names " + diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go index 205e5d56..47bcab76 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/builder.go @@ -63,8 +63,18 @@ func ParseBuilderBuildKey(key string) (module, fingerprint string, err error) { return module, fingerprint, nil } -// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest and, for an -// image, the reference it was pushed to. +// BuiltArtifact is one artifact of a done build: its name in the manifest, its kind, its content digest (`sha256:` and +// 64 hex digits) and the reference that names where it is (novox/hq ADR 0498 clauses 1, 5 and 6; the lead's decision +// L1 of R4e), by kind: +// +// bundle, archive artifact-store:////blobs/ the blob pushed to the artifact store +// image, upstream //@ the image registry's reference, which +// artifact-store:///@ catalogue.Recorded keeps without its address +// package @ the version published from the trunk +// +// is Name and is Digest. A reference is empty for a candidate's image or package, which is never +// pushed or published (ADR 0498 clause 6), and in a key done before R4e; the controller delivers only from a key whose +// every artifact names one (Referenced, clause 8). A kind left empty (a key written before kinds) is not checked. type BuiltArtifact struct { Name string `json:"name"` Kind string `json:"kind,omitempty"` @@ -72,6 +82,83 @@ type BuiltArtifact struct { Reference string `json:"reference,omitempty"` } +const artifactStore = "artifact-store://" + +// referenceProblem says what is wrong with a's reference for its kind, or "". +func (a BuiltArtifact) referenceProblem() string { + r := a.Reference + if r == "" || a.Kind == "" { + return "" + } + switch a.Kind { + case "bundle", "archive": + path, ok := strings.CutPrefix(r, artifactStore) + module, rest, _ := strings.Cut(path, "/") + if !ok || module == "" || rest != a.Name+"/blobs/"+a.Digest { + return "is not artifact-store:///" + a.Name + "/blobs/" + a.Digest + } + case "image", "upstream": + path, digest, ok := strings.Cut(r, "@") + if !ok || digest != a.Digest { + return "does not name its digest " + a.Digest + } + var host, module, name string + if rest, kept := strings.CutPrefix(path, artifactStore); kept { + module, name, _ = strings.Cut(rest, "/") + host = "kept" + } else { + parts := strings.Split(path, "/") + if len(parts) == 3 { + host, module, name = parts[0], parts[1], parts[2] + } + } + if host == "" || !(host == "kept" || host == "localhost" || strings.ContainsAny(host, ".:")) || module == "" || name != a.Name { + return "is not //" + a.Name + "@" + a.Digest + } + case "package": + at := strings.LastIndex(r, "@") + if at <= 0 || at == len(r)-1 || r[:at] == "@" { + return "is not @" + } + } + return "" +} + +// Referenced says whether b is done and every artifact names where it is, which the controller delivers from (novox/hq +// ADR 0498 clause 8), and names the artifacts that name no reference. A done key with no artifact (a module that builds +// none, issue 1613) is referenced. +func (b BuilderBuild) Referenced() (ok bool, missing []string) { + for _, a := range b.Artifacts { + if a.Reference == "" { + missing = append(missing, a.Name) + } + } + return b.State == BuildDone && len(missing) == 0, missing +} + +// The prefixes of a failed key's error that a build module answers (novox/hq ADR 0498 clauses 2, 4 and 5; ADR 0478 +// clause 2), each followed by the artifact and the words of what refused it. +const ( + // NotStored: the push of a bundle, an archive or an upstream copy to the artifact store failed; built again. + NotStored = "not stored: " + // NotPinned: an upstream artifact names no digest. + NotPinned = "not pinned: " + // VersionTaken: the package registry holds the package's version with another packed file. + VersionTaken = "version taken: " + // RefusedKind: no build module builds this kind of artifact in its language. + RefusedKind = "refused kind: " + // WaitsFor: a fetch found the module's lock naming a package version the registry does not hold yet; the builder + // writes the key queued with Waits (novox/hq ADR 0500 clause 6), and it is built once the package is published. + WaitsFor = "waits for: " +) + +// FailedUntilChanged says whether a failed key's error is one that is not built again until the module's fingerprint +// changes (not pinned, version taken, refused kind: the same source fails the same way); every other failed key, a +// push that was not stored included, is built again as a new revision (ADR 0429 clause 18). +func FailedUntilChanged(err string) bool { + return strings.HasPrefix(err, NotPinned) || strings.HasPrefix(err, VersionTaken) || strings.HasPrefix(err, RefusedKind) +} + // ToolchainUsed is what a build used for one language (ADR 0405 clauses 2 and 7): the installed toolchain's version and // the build module's recipe version. type ToolchainUsed struct { @@ -82,7 +169,8 @@ type ToolchainUsed struct { // BuilderBuild is the value of a builder build key: its State (queued, building, done or failed), By the builder // instance that holds it, Step what it is doing or did last, Manifest the digest of the module's manifest (the key of // the indexer's manifests bucket), Artifacts when done, Toolchains by language when it built or is building (a done -// build names them, so two builders that differ are seen: ADR 0405 clause 2), and Error when failed. +// build names them, so two builders that differ are seen: ADR 0405 clause 2), Error when failed, and Waits when queued for a +// package version. type BuilderBuild struct { V int `json:"v"` State string `json:"state"` @@ -92,6 +180,10 @@ type BuilderBuild struct { Artifacts []BuiltArtifact `json:"artifacts,omitempty"` Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` Error string `json:"error,omitempty"` + // Waits is, on a queued key only, the package version `@` the module's lock names and the registry + // does not yet hold: the key is not claimed until that package's trunk key is done and published (novox/hq ADR 0500 + // clause 6). A reader that does not know the field reads the key as queued. + Waits string `json:"waits,omitempty"` } func (b BuilderBuild) check() error { @@ -120,6 +212,17 @@ func (b BuilderBuild) check() error { if a.Digest == "" { return fmt.Errorf("delivery: the artifact %q of the build has no digest", a.Name) } + if p := a.referenceProblem(); p != "" { + return fmt.Errorf("delivery: the reference %q of the %s %q %s", a.Reference, a.Kind, a.Name, p) + } + } + if b.Waits != "" { + if b.State != BuildQueued { + return fmt.Errorf("delivery: a %s build waits for %q; only a queued key waits for a package", b.State, b.Waits) + } + if p := (BuiltArtifact{Kind: "package", Reference: b.Waits}).referenceProblem(); p != "" { + return fmt.Errorf("delivery: the package a key waits for, %q, %s", b.Waits, p) + } } switch b.State { case BuildQueued, BuildBuilding: @@ -127,9 +230,7 @@ func (b BuilderBuild) check() error { return fmt.Errorf("delivery: a %s build carries no artifact and no error", b.State) } case BuildDone: - if len(b.Artifacts) == 0 { - return fmt.Errorf("delivery: a done build names no artifact") - } + // A module that builds no artifact gets a done key too, and is delivered from it (issue 1613). if b.Manifest == "" { return fmt.Errorf("delivery: a done build names no manifest digest") } @@ -144,8 +245,8 @@ func (b BuilderBuild) check() error { return nil } -// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without artifacts or a -// manifest digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a +// EncodeBuilderBuild is b as JSON at Version; it refuses a build its state does not allow (done without a manifest +// digest, failed without an error, an artifact without a name or digest, a toolchain without a version or a // recipe version). func EncodeBuilderBuild(b BuilderBuild) ([]byte, error) { if err := b.check(); err != nil { diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go index 18444c57..5fbe9fb9 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/gate.go @@ -29,7 +29,9 @@ const ( ) // The contexts of the statuses the mesh owes the forge: the gate's, the delivery's view of a head and of a group -// (design 47), and the checker's, which is not required until R4f (ADR 0484 clause 2, ADR 0453 clause 4). +// (design 47), and the checker's. From R4e the checker's mesh/check is the one status a repository's main requires, +// in place of mesh/merge-gate and mesh/repo-check (ADR 0503 clauses 3 and 5); a main that requires only +// mesh/repo-check (hq) keeps it, and the controller writes it for a repository the checker writes no status for. const ( StatusContextMergeGate = "mesh/merge-gate" StatusContextDelivery = "mesh/delivery" diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go new file mode 100644 index 00000000..cc2cbad1 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/namedtree.go @@ -0,0 +1,47 @@ +package delivery + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "path" + "sort" + "strings" +) + +// NamedTreePrefix opens the tree input of a module whose every artifact names its paths, so such a value is never read +// as a git tree id and a reader of a kept Tree can tell the two apart. +const NamedTreePrefix = "named:" + +// NamedTree is the tree input of the new-form fingerprint of a module whose every artifact names the paths it is built +// from (novox/hq ADR 0499 clause 2): `named:` and the hex sha256 of the lines "manifest " and, sorted, +// "path ", joined by a newline. manifest is the module's ManifestDigest (the sha256 of its module.json's +// exact bytes, as Module.Manifest and the merge preview carry it); paths are the named paths, each clean and relative to +// the module's directory as the manifest names them (catalogue.Manifest.NamedPaths), with the git object id each has at +// the commit (a blob for a file, a tree for a directory). +// +// **One function computes it** (clause 2): the indexer, its merge preview (buildstep.ModuleTrees) and the build modules +// call this one, and no second computation exists. It refuses no path, a named path with no object (absent at the +// commit), a path that is not clean, and a manifest that is not a digest, naming them: the indexer then gives the +// module none, with that reason (clause 3). +func NamedTree(manifest string, paths map[string]string) (string, error) { + if _, err := ManifestsKey(manifest); err != nil { + return "", err + } + if len(paths) == 0 { + return "", fmt.Errorf("delivery: the module names no path, so it has no named tree") + } + lines := make([]string, 0, len(paths)) + for p, object := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return "", fmt.Errorf("delivery: the named path %q is not a clean path inside the module's directory", p) + } + if object == "" { + return "", fmt.Errorf("delivery: the named path %q has no git object at the commit", p) + } + lines = append(lines, "path "+p+" "+object) + } + sort.Strings(lines) + sum := sha256.Sum256([]byte("manifest " + manifest + "\n" + strings.Join(lines, "\n"))) + return NamedTreePrefix + hex.EncodeToString(sum[:]), nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go index dd90cb76..36791460 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/preview.go @@ -98,12 +98,15 @@ func ParsePreviewKey(key string) (repository, head, trunk string, err error) { return repository, parts[1], parts[2], nil } -// PreviewModule is what the preview holds of one module directory: the tree sha of the directory, the digest of its +// PreviewModule is what the preview holds of one module directory: the tree sha of the directory (or its named tree), the digest of its // manifest (the key in the manifests bucket) and the hex sha256 of its lock files by path relative to it. type PreviewModule struct { Tree string `json:"tree"` Manifest string `json:"manifest"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge; Tree + // is then their NamedTree (novox/hq ADR 0499 clause 2). Absent otherwise. + Paths map[string]string `json:"paths,omitempty"` } // Preview is the value of a preview key: the repository, the head and the trunk commit it was merged onto, the merge @@ -152,6 +155,9 @@ func (p Preview) check() error { return fmt.Errorf("delivery: the lock %q of the preview's module %q has an empty path or hash", path, dir) } } + if err := checkNamed(m.Tree, m.Manifest, m.Paths, fmt.Sprintf("the preview's module %q", dir)); err != nil { + return err + } } return nil } diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go index 8d85b64a..145b4e25 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/delivery/source.go @@ -5,6 +5,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "path" "reflect" "sort" "strings" @@ -276,6 +277,9 @@ type CandidateModule struct { FP2 string `json:"fp2,omitempty"` None2 string `json:"none2,omitempty"` Manifest string `json:"manifest,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the preview + // (novox/hq ADR 0499 clause 2), so the builder's fetch request for the candidate names them. Absent otherwise. + Paths map[string]string `json:"paths,omitempty"` } func (m CandidateModule) check() error { @@ -285,6 +289,9 @@ func (m CandidateModule) check() error { case m.FP2 == "" && m.None2 == "": return fmt.Errorf("delivery: a module has neither fp2 nor none2") } + if err := checkPathObjects(m.Paths); err != nil { + return err + } return checkNewForm(m.FP2, m.Manifest) } @@ -356,7 +363,7 @@ func (c Candidate) check() error { if err := m.check(); err != nil { return fmt.Errorf("delivery: the candidate's module %q: %w", module, err) } - if before, ok := seen[module]; ok && before != m { + if before, ok := seen[module]; ok && !reflect.DeepEqual(before, m) { return fmt.Errorf("delivery: two pull requests of the candidate give the module %q different fingerprints", module) } seen[module] = m @@ -370,12 +377,13 @@ func (c Candidate) check() error { // Module is the value of a modules key: the module's source fingerprint, or none and why (a build that resolves // packages when built gets none, said here), and where it was computed: Repository and Commit are the trunk merge, -// Src the source revision then (0 before the gate writes source.main). Exactly one of FP and None is set. +// Src the source revision then (0 before the gate writes source.main). At most one of FP and None is set. // // From R4c the indexer writes the new form beside it (novox/hq ADR 0453 clause 6): FP2, the fingerprint of the new // form (`src2:` and 64 hex digits), or None2, why it has none; one of them whenever either is, never both; and // Manifest, the digest of the module's manifest, the key of the manifests bucket. A value written before has none of -// the three, and a reader of today's shape ignores them. +// the three, and a reader of today's shape ignores them. From R4e a value may carry the new form alone (ADR 0453 clause +// 6; issue 1851): one of FP, None, FP2 and None2 is set, and the old form stays readable. type Module struct { V int `json:"v"` FP string `json:"fp,omitempty"` @@ -398,6 +406,11 @@ type Module struct { // builds the fingerprint only with these. Empty when FP2 is empty or the module has no toolchain; a value written // before has none (issue 1615). Toolchains map[string]ToolchainUsed `json:"toolchains,omitempty"` + // ArtifactPaths are, of each artifact that names the paths it is built from, those paths as its manifest names + // them, by artifact (novox/hq ADR 0499 clause 1): the indexer fills them from each catalogue.Artifact.Paths it reads, + // so the checker tells whether a change touches an artifact's build source without reading a manifest (the lead's + // decision L8 of R4e). Absent for an artifact that names none, and in a value written before. + ArtifactPaths map[string][]string `json:"paths,omitempty"` } // RootPath is Module.Path of a module whose manifest is its repository's root. @@ -411,6 +424,39 @@ type KeptInputs2 struct { Builder string `json:"builder,omitempty"` Contexts map[string]string `json:"contexts,omitempty"` Locks map[string]string `json:"locks,omitempty"` + // Paths are, of a module whose every artifact names its paths, each named path's git object id at the merge, by + // path relative to the module's directory; Tree is then their NamedTree with the module's manifest digest, so a + // computation from kept inputs gives the merge's value (novox/hq ADR 0499 clause 2). Absent for every other module, + // whose Tree is its directory's tree, and in a value written before. + Paths map[string]string `json:"paths,omitempty"` +} + +// checkPathObjects refuses a named path that is not clean and relative to the module's directory, or names no object. +func checkPathObjects(paths map[string]string) error { + for p, object := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return fmt.Errorf("delivery: the named path %q is not a clean path inside the module's directory", p) + } + if object == "" { + return fmt.Errorf("delivery: the named path %q names no git object", p) + } + } + return nil +} + +// checkNamed refuses kept named paths that do not compute to the kept tree with the module's manifest digest. +func checkNamed(tree, manifest string, paths map[string]string, of string) error { + if len(paths) == 0 { + return nil + } + named, err := NamedTree(manifest, paths) + if err != nil { + return fmt.Errorf("delivery: %s: %w", of, err) + } + if named != tree { + return fmt.Errorf("delivery: %s keeps the tree %q, and its named paths give the named tree %q", of, tree, named) + } + return nil } // With is the full inputs of module m: the kept ones and the given toolchains. @@ -423,8 +469,8 @@ func (m Module) check() error { switch { case m.FP != "" && m.None != "": return fmt.Errorf("delivery: a module has both a fingerprint and a reason for none") - case m.FP == "" && m.None == "": - return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none") + case m.FP == "" && m.None == "" && m.FP2 == "" && m.None2 == "": + return fmt.Errorf("delivery: a module has neither a fingerprint nor a reason for none, of either form") case m.FP != "" && !strings.HasPrefix(m.FP, SourcePrefix): return fmt.Errorf("delivery: the module's fingerprint %q does not start with %s", m.FP, SourcePrefix) case m.FP2 != "" && m.None2 != "": @@ -438,6 +484,21 @@ func (m Module) check() error { if m.Inputs2 != nil && m.Manifest == "" { return fmt.Errorf("delivery: a module keeps new-form inputs but has no manifest digest") } + for artifact, paths := range m.ArtifactPaths { + if artifact == "" || len(paths) == 0 { + return fmt.Errorf("delivery: the module's named paths of the artifact %q are empty or name no artifact", artifact) + } + for _, p := range paths { + if p == "" || p != path.Clean(p) || strings.HasPrefix(p, "/") || p == ".." || strings.HasPrefix(p, "../") { + return fmt.Errorf("delivery: the artifact %q names the path %q, not a clean path inside the module's directory", artifact, p) + } + } + } + if m.Inputs2 != nil { + if err := checkNamed(m.Inputs2.Tree, m.Manifest, m.Inputs2.Paths, "the module's kept inputs"); err != nil { + return err + } + } return checkNewForm(m.FP2, m.Manifest) } @@ -465,9 +526,13 @@ type Forge struct { // relative to it (novox/hq ADR 0465 clause 1). Older writers send neither. Manifests map[string]string `json:"manifests,omitempty"` Locks map[string]map[string]string `json:"locks,omitempty"` - Act string `json:"act,omitempty"` - Words string `json:"words,omitempty"` - Open []OpenPull `json:"open,omitempty"` + // Paths are, of each module directory whose manifest names paths on every artifact, each named path's git object + // id at the same commit as its manifest, by path relative to the directory (novox/hq ADR 0499 clause 3). The + // indexer names the paths; a message without them gives such a module none. Older writers send none. + Paths map[string]map[string]string `json:"paths,omitempty"` + Act string `json:"act,omitempty"` + Words string `json:"words,omitempty"` + Open []OpenPull `json:"open,omitempty"` } func (f Forge) check() error { @@ -537,8 +602,8 @@ func (f Forge) check() error { if len(f.Dirs) > 0 || f.DirsKnown { return fmt.Errorf("delivery: a refused message carries no module directories") } - if len(f.Manifests) > 0 || len(f.Locks) > 0 { - return fmt.Errorf("delivery: a refused message carries no manifests or lock hashes") + if len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 { + return fmt.Errorf("delivery: a refused message carries no manifests, lock hashes or named paths") } if f.Act == "" { return fmt.Errorf("delivery: a refused message needs the act it answers") @@ -571,6 +636,16 @@ func (f Forge) checkInputs(known func(dir string) bool, of string) error { } } } + for dir, paths := range f.Paths { + if !known(dir) { + return fmt.Errorf("delivery: the named paths of the directory %q are not of %s", dir, of) + } + for path, object := range paths { + if path == "" || object == "" { + return fmt.Errorf("delivery: the named path %q of the directory %q has an empty path or object", path, dir) + } + } + } return nil } @@ -578,7 +653,7 @@ func (f Forge) checkInputs(known func(dir string) bool, of string) error { // an open pull request that is not `/` and a number. func (f Forge) checkOpen() error { if f.Repository != "" || f.Branch != "" || f.Number != 0 || f.Head != "" || f.Action != "" || len(f.Dirs) > 0 || f.DirsKnown || - f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { + f.Merge != "" || len(f.Trees) > 0 || len(f.Manifests) > 0 || len(f.Locks) > 0 || len(f.Paths) > 0 || f.Root != "" || f.Act != "" || f.Words != "" { return fmt.Errorf("delivery: an open message carries only the open pull requests it names") } for _, p := range f.Open { diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/keeps.go b/vendor/git.novox.be/novox/mesh-sdk/go/keeps.go new file mode 100644 index 00000000..cffdf9c7 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/keeps.go @@ -0,0 +1,80 @@ +package stdio + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "sync" + + "git.novox.be/novox/mesh-sdk/go/announce" +) + +// What this bundle said it keeps (novox/hq ADR 0495 clause 2; issue 1790). +var ( + keepsMu sync.Mutex + kept map[string]bool // nil until Keeps: no state is refused then + announced []byte // the announcement, published once the bundle serves +) + +// Keeps says on the bus, as this bundle's module, every state it keeps and which of them others may read (novox/hq +// ADR 0495 clause 2): what a reader is granted is what the owner announced (clause 3). Call it before Serve; the +// announcement is published through the runtime once the bundle serves, or at once when it already does. From then +// on a state of the module's own that it did not announce is refused at the call, naming it, so code and announcement +// cannot drift (clause 5). It refuses a state no bucket could be named by and a state given twice. +func Keeps(states ...announce.State) error { + module := os.Getenv("MESH_MODULE") + if module == "" { + module = os.Getenv(ServedAs) + } + data, err := announce.Encode(announce.States{Module: module, Node: os.Getenv("MESH_NODE"), States: states}) + if err != nil { + return err + } + names := map[string]bool{} + for _, s := range states { + names[s.Name] = true + } + keepsMu.Lock() + kept, announced = names, data + keepsMu.Unlock() + currentMu.Lock() + serving := current != nil + currentMu.Unlock() + if serving { + return announce1() + } + return nil +} + +// announce1 publishes the announcement Keeps made, if any. +func announce1() error { + keepsMu.Lock() + data := announced + keepsMu.Unlock() + if data == nil { + return nil + } + return Emit(announce.Event, json.RawMessage(data)) +} + +// refusedUnannounced is the error of a call on a state of the module's own that Keeps did not name, or nil. A read of +// another module's state (`.`) is the grant's business. +func refusedUnannounced(name string) error { + if strings.Contains(name, ".") { + return nil + } + keepsMu.Lock() + defer keepsMu.Unlock() + if kept != nil && !kept[name] { + return fmt.Errorf("the state %q is not among the states this module announced it keeps (stdio.Keeps)", name) + } + return nil +} + +// forgetKeeps is for a test: nothing announced. +func forgetKeeps() { + keepsMu.Lock() + kept, announced = nil, nil + keepsMu.Unlock() +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh b/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh index 92e9d9cd..00587326 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh +++ b/vendor/git.novox.be/novox/mesh-sdk/go/merge-check.sh @@ -10,13 +10,17 @@ # 1. the unit level: `go test ./...` untagged, no bus or other process, timed and held to a bound # (unit_bound below); over it the check fails and names the slowest tests (issue 895); # 2. the integration level: the whole suite with `-tags integration`, each test against a bus of its -# own (internal/testbus, which issue 1171 brings with the first test that needs a bus) so nothing is +# own (testbus, which issue 1171 brings with the first test that needs a bus) so nothing is # serialised. With no tagged test yet, the level runs the unit tests again and says their count. # # **A test that starts a bus or another process carries `//go:build integration`** as the first line of # its file; a file that held both kinds is two files (`_integration_test.go` beside `_test.go`). # Each level says its test count and its time. # +# **A test that needs bubblewrap carries `//go:build integration && isolation`** (issue 1309's proof that a run step +# is shut off): this script vets it but does not run it, since the toolchain it runs in has no bubblewrap; the checker +# runs the `isolation` tag on the build nodes, which have it (R4e). +# # No dependency is committed (no vendor directory: .gitignore; the operator, 2026-10-19; issue 1304). Every dependency # is resolved by the version and checksum go.sum names (novox/hq ADR 0405 §5), from its public source: this binding # requires nothing of the mesh's own forge. `go mod verify` fails by module name when a downloaded dependency does not @@ -98,6 +102,7 @@ fi go vet ./... # The files tagged `integration` are vetted too, or nothing would read them. go vet -tags integration ./... +go vet -tags integration,isolation ./... # NOT RACE-CHECKED here: the SDK's suite is small and carries no store or other shared state between # packages, so the race detector is not required of this check (novox/hq issue 1175); a module that needs @@ -117,7 +122,7 @@ if [ "$status" -ne 0 ]; then fi judge_unit_run "$work/unit.out" "$unit_bound" -# The integration level: the whole suite with the tag, each test against its own bus (internal/testbus). +# The integration level: the whole suite with the tag, each test against its own bus (testbus). integration_start=$(date +%s) status=0 go test -tags integration -count=1 -v ./... >"$work/integration.out" 2>&1 || status=$? diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go b/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go index 58efb40c..ca469221 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/process/conn.go @@ -24,6 +24,12 @@ type Conn struct { mu sync.Mutex next uint64 pending map[string]map[uint64]context.CancelCauseFunc + + // What Keeps announced (keeps.go): the states' names (nil before Keeps), the announcement, and whether a + // reconnect announces it again. + kept map[string]bool + announced []byte + reannounces bool } // errPermissionRefused marks a context cancelled because the bus refused this account's publish of the diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go b/vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go new file mode 100644 index 00000000..22a1df79 --- /dev/null +++ b/vendor/git.novox.be/novox/mesh-sdk/go/process/keeps.go @@ -0,0 +1,71 @@ +package process + +import ( + "encoding/json" + "fmt" + + "github.com/nats-io/nats.go" + + "git.novox.be/novox/mesh-sdk/go/announce" +) + +// Keeps says on the bus, as this process's module, every state it keeps and which of them others may read (novox/hq +// ADR 0495 clause 2; issue 1790), on announce.Subject; what a reader is granted is what the owner announced (clause 3). +// It publishes at once and again after every reconnect, so the controller hears it whenever it missed it. From then on +// a state of the module that it did not announce is refused at the call, naming it, so code and announcement cannot +// drift (clause 5). It refuses a state no bucket could be named by and a state given twice. +func (c *Conn) Keeps(states ...announce.State) error { + if err := c.remember(states); err != nil { + return err + } + c.mu.Lock() + first := !c.reannounces + c.reannounces = true + c.mu.Unlock() + if first { + before := c.nc.Opts.ReconnectedCB + c.nc.SetReconnectHandler(func(nc *nats.Conn) { + if before != nil { + before(nc) + } + _ = c.publishKeeps() + }) + } + return c.publishKeeps() +} + +// remember makes the announcement of states and binds only them from now on. +func (c *Conn) remember(states []announce.State) error { + data, err := announce.Encode(announce.States{Module: c.account.Module, Node: c.account.Node, States: states}) + if err != nil { + return err + } + names := map[string]bool{} + for _, s := range states { + names[s.Name] = true + } + c.mu.Lock() + c.kept, c.announced = names, data + c.mu.Unlock() + return nil +} + +func (c *Conn) publishKeeps() error { + c.mu.Lock() + data := c.announced + c.mu.Unlock() + if err := c.Emit(announce.Event, json.RawMessage(data)); err != nil { + return fmt.Errorf("announcing what %s keeps: %w", c.account.Module, err) + } + return nil +} + +// refusedUnannounced is the error of a bind of a state Keeps did not name, or nil (also before Keeps). +func (c *Conn) refusedUnannounced(name string) error { + c.mu.Lock() + defer c.mu.Unlock() + if c.kept != nil && !c.kept[name] { + return fmt.Errorf("the state %q is not among the states %s announced it keeps (Conn.Keeps)", name, c.account.Module) + } + return nil +} diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go b/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go index 43221fac..8e294e31 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/process/state.go @@ -39,6 +39,9 @@ func (s State) bucket() string { return s.Module + "_" + s.Name } func (c *Conn) State(name string) State { bucket := c.account.Module + "_" + name return State{Module: c.account.Module, Name: name, kv: func(ctx context.Context) (jetstream.KeyValue, error) { + if err := c.refusedUnannounced(name); err != nil { + return nil, err + } js, err := jetstream.New(c.nc) if err != nil { return nil, err diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/state.go b/vendor/git.novox.be/novox/mesh-sdk/go/state.go index 70e1b35c..56490b6e 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/state.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/state.go @@ -50,6 +50,9 @@ func State(name string) KeptState { return KeptState{name: name} } const stateTimeout = 30 * time.Second func (s KeptState) ask(verb string, params map[string]any) (json.RawMessage, error) { + if err := refusedUnannounced(s.name); err != nil { + return nil, err + } c, err := live() if err != nil { return nil, err diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go b/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go index a61f4dec..a072f40d 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/stdio.go @@ -180,6 +180,9 @@ func ServeOn(name string, tools []Tool, in io.Reader, out io.Writer) error { currentMu.Lock() current = c currentMu.Unlock() + // What the bundle said it keeps is announced once it serves: the runtime answers the publish on this channel, so + // it is asked from beside the loop that reads the answer. + go func() { _ = announce1() }() scanner := bufio.NewScanner(in) scanner.Buffer(make([]byte, 1<<20), 1<<24) for scanner.Scan() { diff --git a/vendor/modules.txt b/vendor/modules.txt index 26ad86b3..7ba7a3f9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,6 +1,7 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go +git.novox.be/novox/mesh-sdk/go/announce git.novox.be/novox/mesh-sdk/go/asks git.novox.be/novox/mesh-sdk/go/buildstep git.novox.be/novox/mesh-sdk/go/catalogue From d36e0ea27809463dbf8868b3597d764d39619e59 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:34:44 +0200 Subject: [PATCH 22/38] =?UTF-8?q?Announce=20the=20controller's=20states,?= =?UTF-8?q?=20keep=20every=20module's=20word,=20and=20grant=20a=20read=20o?= =?UTF-8?q?nly=20where=20its=20owner=20announced=20the=20state=20readable;?= =?UTF-8?q?=20the=20copied=20list=20of=20the=20controller's=20mirrors=20go?= =?UTF-8?q?es=20(issue=201790,=20D1-D4,=20ADR=200495,=20ADR=200513=20?= =?UTF-8?q?=C2=A73)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/mesh-controller/build.go | 14 +- cmd/mesh-controller/main.go | 2 + cmd/mesh-controller/push.go | 8 + cmd/mesh-controller/readable.go | 8 +- cmd/mesh-controller/states_announced.go | 225 ++++++++++++++++++ .../states_announced_integration_test.go | 118 +++++++++ cmd/mesh-controller/states_announced_test.go | 56 +++++ cmd/mesh-controller/status.go | 12 + internal/broker/announced.go | 81 +++++++ internal/broker/announced_test.go | 85 +++++++ .../broker/announced_test_helpers_test.go | 13 + internal/broker/checks_running_mirror_test.go | 4 +- internal/broker/controller_buckets.go | 22 +- internal/broker/controller_buckets_test.go | 6 +- internal/broker/delivery_parts_grant_test.go | 4 +- internal/broker/nats.go | 13 + internal/broker/plans_building_mirror_test.go | 4 +- internal/broker/process_grants_test.go | 2 + internal/broker/state.go | 6 + internal/broker/testdata/composed.conf | 2 +- internal/broker/users.go | 7 + internal/broker/writers.go | 24 ++ internal/broker/writers_test.go | 3 + internal/inventory/busrecords.go | 28 +++ internal/recordmirror/readers_test.go | 11 +- 25 files changed, 732 insertions(+), 26 deletions(-) create mode 100644 cmd/mesh-controller/states_announced.go create mode 100644 cmd/mesh-controller/states_announced_integration_test.go create mode 100644 cmd/mesh-controller/states_announced_test.go create mode 100644 internal/broker/announced.go create mode 100644 internal/broker/announced_test.go create mode 100644 internal/broker/announced_test_helpers_test.go diff --git a/cmd/mesh-controller/build.go b/cmd/mesh-controller/build.go index a9c640cc..c09a49e0 100644 --- a/cmd/mesh-controller/build.go +++ b/cmd/mesh-controller/build.go @@ -683,11 +683,15 @@ func publishable(result link.BuildResult, follows string) error { // answers is what the three questions came back with, read once. type answers struct { - wrong []inventory.Doing - nodes []inventory.Node - quiet []inventory.Node - behind map[string][]string - sources map[string]inventory.Source + // notAnnounced are the owners some module reads the state of that have announced nothing yet (novox/hq ADR 0495, + // issue 1790): read from `reads` as before until they do; notAnnouncedWhy why that could not be read. + notAnnounced []string + notAnnouncedWhy string + wrong []inventory.Doing + nodes []inventory.Node + quiet []inventory.Node + behind map[string][]string + sources map[string]inventory.Source // waiting is every machine not running what the mesh would send it. waiting []inventory.Machine // recordedWaits is, per machine, every move of a recorded module that waits there for a person (novox/hq ADR diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 9eab8a5e..0b506f2a 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -54,6 +54,8 @@ func run() error { return fmt.Errorf("no command given") } + // The bus's users are composed from what each module announced it keeps (novox/hq ADR 0495, issue 1790). + inventory.Announcements = readAnnouncements // Every connection this process dials says what it is (novox/hq issue 327). broker.ConnectionName = connectionName(args[0], os.Getenv(verbVar)) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index e3997d88..510f650a 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -289,6 +289,14 @@ func serve(ctx context.Context) (err error) { // And the modules the catalogue does not hold yet, registered from the builder's first done key of each, woken by // the builder's and the indexer's writes (novox/hq ADR 0501 §6, issue 1851): nothing asks a first build. watchNewModules(ctx, open) + // And says what it keeps, and keeps what every module says it keeps (novox/hq ADR 0495, issue 1790): a changed word + // sends the machine holding the bus its new user list. + if err := announceOwnStates(ctx, server.JetStream()); err != nil { + fmt.Printf("states: %v\n", err) + } + if err := keepAnnouncements(ctx, server.JetStream(), open); err != nil { + fmt.Printf("states: the states the modules announce are not kept: %v\n", err) + } // Every call carries the lease's epoch, and its record is written only under the lease (novox/hq // to-be 45 §6). link.Calls.UnderLease(func() (uint64, error) { return theLease.epoch(ctx) }) diff --git a/cmd/mesh-controller/readable.go b/cmd/mesh-controller/readable.go index 82657485..30a32f0d 100644 --- a/cmd/mesh-controller/readable.go +++ b/cmd/mesh-controller/readable.go @@ -53,6 +53,11 @@ type meshStatus struct { // Per machine, and data. One node failing must never take the document away from a reader // asking about the others. Unresolved []machineUnresolved `json:"unresolved"` + // NotAnnounced are the modules others read the state of that have not said what they keep (novox/hq ADR 0495, + // issue 1790): their readers are granted from `reads` as before until they do. NotAnnouncedWhy is why it could not + // be read. + NotAnnounced []string `json:"not-announced,omitempty"` + NotAnnouncedWhy string `json:"not-announced-why,omitempty"` // Network is why the private network could not be computed, when it could not; absent when it // could. Almost always a consequence of Unresolved: a node that does not resolve is not on the // network, and a mesh whose hub is that node has no hub. @@ -247,7 +252,8 @@ func statusAsJSON(asked answers) ([]byte, error) { out := meshStatus{Machines: len(nodes), Wrong: []machineDoing{}, Quiet: []machineQuiet{}, Behind: []moduleBehind{}, Waiting: []machineWaiting{}, Reported: []machineReported{}, Unresolved: []machineUnresolved{}, - Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds)} + Network: asked.network, Adopted: adoptedNodes(nodes), Plans: planStatuses(asked.plans, time.Now(), asked.paused, asked.tierBounds), + NotAnnounced: asked.notAnnounced, NotAnnouncedWhy: asked.notAnnouncedWhy} // In a stated order, so two readings of an unchanged mesh are the same document. untakenNodes := make([]string, 0, len(asked.untaken)) for name := range asked.untaken { diff --git a/cmd/mesh-controller/states_announced.go b/cmd/mesh-controller/states_announced.go new file mode 100644 index 00000000..8e4012a1 --- /dev/null +++ b/cmd/mesh-controller/states_announced.go @@ -0,0 +1,225 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "git.novox.be/novox/mesh-sdk/go/announce" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" +) + +// What each module keeps, and who may read it (novox/hq ADR 0495 clauses 2 to 5, ADR 0513 §3; issue 1790, D1-D3). +// +// Every module says on the bus which states it keeps and which of them others may read (announce.Subject), from its +// code; the controller says its own (D1). The serving controller keeps the last word of each module in its bucket +// (D2), and the bus's users are composed from it: a reader's `reads: .` is granted only when the owner +// announced that state readable; an owner that has announced nothing yet is read from `reads` as before (the +// transition), and `status` names it. A changed word composes the bus's users again: the machine holding the bus is +// sent its new user list (D3). No timer: the words wake it. + +// announceOwnStates says what the controller keeps, on its own event subject (D1). +func announceOwnStates(ctx context.Context, js *broker.JetStream) error { + host, _ := os.Hostname() + body, err := announce.Encode(broker.ControllerAnnouncement(host)) + if err != nil { + return err + } + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + if _, err := api.Publish(ctx, announce.Subject(broker.ControllerSeat), body); err != nil { + return fmt.Errorf("saying what the controller keeps: %w", err) + } + return nil +} + +// keepAnnouncements follows every module's word on what it keeps (D2) and keeps the last of each, from the last each +// module said: one module's word in the bucket under its name. A word whose module is not the subject's is refused, +// said; a changed word composes the bus's users again (D3). +func keepAnnouncements(ctx context.Context, js *broker.JetStream, open *stores) error { + api, err := jetstream.New(js.Conn()) + if err != nil { + return err + } + kv, err := api.KeyValue(ctx, broker.StatesAnnouncedBucket) + if err != nil { + return fmt.Errorf("the bucket of the states announced: %w", err) + } + stream, err := api.Stream(ctx, broker.EventsStream) + if err != nil { + return fmt.Errorf("the events stream: %w", err) + } + consumer, err := stream.OrderedConsumer(ctx, jetstream.OrderedConsumerConfig{ + FilterSubjects: []string{announce.SubjectAll}, DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy}) + if err != nil { + return fmt.Errorf("following the states announced: %w", err) + } + consuming, err := consumer.Consume(func(m jetstream.Msg) { + changed, err := keepAnnouncement(ctx, kv, m.Subject(), m.Data()) + if err != nil { + fmt.Printf("states: %v\n", err) + return + } + if changed { + go composeAnnouncedReads(context.WithoutCancel(ctx), open) + } + }) + if err != nil { + return fmt.Errorf("following the states announced: %w", err) + } + go func() { + <-ctx.Done() + consuming.Stop() + }() + return nil +} + +// announcedKeeper is the bucket of the states announced, as keepAnnouncement writes it. +type announcedKeeper interface { + Get(ctx context.Context, key string) (jetstream.KeyValueEntry, error) + Put(ctx context.Context, key string, value []byte) (uint64, error) +} + +// keepAnnouncement keeps one module's word, and says whether it changed what the bucket held. +func keepAnnouncement(ctx context.Context, kv announcedKeeper, subject string, body []byte) (bool, error) { + s, err := announce.Decode(body) + if err != nil { + return false, fmt.Errorf("a word on %s is not read: %w", subject, err) + } + if subject != announce.Subject(s.Module) { + return false, fmt.Errorf("%s says what %s keeps, and a module says only what it keeps itself: refused", subject, + s.Module) + } + key, err := announce.Key(s.Module) + if err != nil { + return false, err + } + stored, err := announce.Encode(s) + if err != nil { + return false, err + } + if was, err := kv.Get(ctx, key); err == nil { + if string(was.Value()) == string(stored) { + return false, nil + } + } else if !errors.Is(err, jetstream.ErrKeyNotFound) { + return false, fmt.Errorf("the last word of %s cannot be read: %w", s.Module, err) + } + if _, err := kv.Put(ctx, key, stored); err != nil { + return false, fmt.Errorf("the word of %s cannot be kept: %w", s.Module, err) + } + fmt.Printf("states: %s keeps %s\n", s.Module, statesWords(s)) + return true, nil +} + +// statesWords is an announcement as a person reads it: each state, readable ones marked. +func statesWords(s announce.States) string { + if len(s.States) == 0 { + return "nothing" + } + var out []string + for _, k := range s.States { + if k.Readable { + out = append(out, k.Name+" (readable)") + } else { + out = append(out, k.Name) + } + } + return strings.Join(out, ", ") +} + +// readAnnouncements is the last word of every module, by module (what inventory.Announcements reads): none when the +// bucket is not there yet, which reads every owner as not yet announced. +func readAnnouncements(ctx context.Context) (map[string]announce.States, error) { + js, err := aBus() + if err != nil { + return nil, err + } + defer js.Close() + api, err := jetstream.New(js.Conn()) + if err != nil { + return nil, err + } + kv, err := api.KeyValue(ctx, broker.StatesAnnouncedBucket) + if errors.Is(err, jetstream.ErrBucketNotFound) { + // empty-on-error: not an error; no module announced yet, so every read is granted from `reads` as before + return map[string]announce.States{}, nil + } + if err != nil { + return nil, err + } + keys, err := kv.Keys(ctx) + if errors.Is(err, jetstream.ErrNoKeysFound) { + // empty-on-error: not an error; no module announced yet + return map[string]announce.States{}, nil + } + if err != nil { + return nil, err + } + out := map[string]announce.States{} + for _, key := range keys { + e, err := kv.Get(ctx, key) + if errors.Is(err, jetstream.ErrKeyNotFound) { + continue + } + if err != nil { + return nil, err + } + s, err := announce.Decode(e.Value()) + if err != nil { + return nil, fmt.Errorf("the kept word of %s is not read: %w", key, err) + } + out[s.Module] = s + } + return out, nil +} + +// composeAnnouncedReads sends the machine holding the bus its user list when a changed word changed it (D3): as a new +// controller's user list is carried (carryUserList), not when that machine is held. +func composeAnnouncedReads(ctx context.Context, open *stores) { + holder, behind, err := brokerBehind(ctx, open, nil) + if err != nil { + fmt.Printf("states: whether the bus's user list is behind the states announced cannot be read: %v\n", err) + return + } + if holder == "" || !behind { + return + } + held, err := heldMachines(ctx, open, []string{holder}) + if err != nil { + fmt.Printf("states: whether %s may be sent its new user list cannot be read: %v\n", holder, err) + return + } + if why, isHeld := held[holder]; isHeld { + fmt.Printf("states: the user list the states announced compose is not sent to %s, which holds the bus: %s — "+ + "`push %s` sends it\n", holder, strings.Join(why, "; "), holder) + return + } + if _, err := sendRollout(ctx, open, []string{holder}); err != nil { + fmt.Printf("states: the user list the states announced compose could not be sent to %s: %v\n", holder, err) + return + } + fmt.Printf("states: sent %s, which holds the bus, the user list the states announced compose\n", holder) +} + +// notYetAnnounced are the owners some module reads the state of and that have announced nothing yet: what `status` +// names while the transition lasts. +func notYetAnnounced(ctx context.Context, inv *inventory.Inventory) ([]string, error) { + if inventory.Announcements == nil { + return nil, nil + } + records, err := inv.BusRecords(ctx) + if err != nil { + return nil, err + } + if records.Announced == nil { + return nil, errors.New("the states announced cannot be read") + } + return broker.NotAnnounced(records), nil +} diff --git a/cmd/mesh-controller/states_announced_integration_test.go b/cmd/mesh-controller/states_announced_integration_test.go new file mode 100644 index 00000000..0e64ca7f --- /dev/null +++ b/cmd/mesh-controller/states_announced_integration_test.go @@ -0,0 +1,118 @@ +//go:build integration + +package main + +import ( + "context" + "slices" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/announce" + "git.novox.be/novox/mesh-sdk/go/catalogue" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +// D1-D3 whole against a real bus and the store (ADR 0495 checks §2, §3, §5; ADR 0513 §3; issue 1790): the controller +// says what it keeps and keeps what every module says; a reader of a state its owner announced readable is granted it, +// one its owner announced and did not make readable is refused and said, and one of an owner that announced nothing +// yet is granted as before and named as not yet announced. +func TestTheStatesAnnouncedGrantTheReads(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + was := inventory.Announcements + inventory.Announcements = readAnnouncements + t.Cleanup(func() { inventory.Announcements = was }) + + // The owner keeps two states, one readable; the reader reads both and a state of a module that says nothing. + register := func(m catalogue.Manifest) { + t.Helper() + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + m.Module, BuiltFrom: "c0"}); err != nil { + t.Fatal(err) + } + } + register(catalogue.Manifest{Module: "owner", Version: "1", State: []catalogue.StateDeclaration{{Name: "open"}, {Name: "closed"}}}) + register(catalogue.Manifest{Module: "quiet", Version: "1", State: []catalogue.StateDeclaration{{Name: "any"}}}) + register(catalogue.Manifest{Module: "reader", Version: "1", Reads: []string{"owner.open", "owner.closed", "quiet.any"}}) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + for _, m := range []string{"owner", "quiet", "reader"} { + if _, err := inv.Assign(ctx, "one", m); err != nil { + t.Fatal(err) + } + } + + watching, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + if err := announceOwnStates(ctx, js); err != nil { + t.Fatal(err) + } + if err := keepAnnouncements(watching, js, open); err != nil { + t.Fatal(err) + } + api, _ := jetstream.New(js.Conn()) + word, _ := announce.Encode(announce.States{Module: "owner", Node: "one", + States: []announce.State{{Name: "open", Readable: true}, {Name: "closed"}}}) + if _, err := api.Publish(ctx, announce.Subject("owner"), word); err != nil { + t.Fatal(err) + } + becomes(t, 10*time.Second, "both words kept", func() bool { + a, err := readAnnouncements(ctx) + return err == nil && len(a) == 2 && a["owner"].Readable("open") && a[broker.ControllerSeat].Readable(broker.PlansBuilding) + }) + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + var reader broker.Declared + for _, d := range records.Assigned["one"] { + if d.Module == "reader" { + reader = d + } + } + if !slices.Equal(reader.Reads, []string{"owner.open", "quiet.any"}) || !slices.Equal(reader.ReadsRefused, []string{"owner.closed"}) { + t.Fatalf("0495 §3: the reader is granted %v and refused %v", reader.Reads, reader.ReadsRefused) + } + if got := broker.NotAnnounced(records); !slices.Equal(got, []string{"quiet"}) { + t.Fatalf("not yet announced: %v", got) + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + for _, u := range users { + if u.Kind != broker.KindModule || u.Module != "reader" { + continue + } + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + if slices.ContainsFunc(perms.Publish, func(s string) bool { return s == "$JS.API.DIRECT.GET.KV_owner_closed.>" }) { + t.Fatal("0495 §3: the reader may read the state its owner did not announce readable") + } + if !slices.Contains(perms.Publish, "$JS.API.DIRECT.GET.KV_owner_open.>") { + t.Fatal("0495 §3: the reader may not read the state its owner announced readable") + } + } +} diff --git a/cmd/mesh-controller/states_announced_test.go b/cmd/mesh-controller/states_announced_test.go new file mode 100644 index 00000000..eb3f920f --- /dev/null +++ b/cmd/mesh-controller/states_announced_test.go @@ -0,0 +1,56 @@ +package main + +import ( + "context" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/announce" + "github.com/nats-io/nats.go/jetstream" +) + +// announcedInMemory is the bucket of the states announced, in memory. +type announcedInMemory map[string][]byte + +type announcedEntry struct { + jetstream.KeyValueEntry + value []byte +} + +func (e announcedEntry) Value() []byte { return e.value } + +func (k announcedInMemory) Get(_ context.Context, key string) (jetstream.KeyValueEntry, error) { + v, ok := k[key] + if !ok { + return nil, jetstream.ErrKeyNotFound + } + return announcedEntry{value: v}, nil +} + +func (k announcedInMemory) Put(_ context.Context, key string, value []byte) (uint64, error) { + k[key] = value + return uint64(time.Now().UnixNano()), nil +} + +// D2 (ADR 0495 clause 2; issue 1790): a module's word is kept under its name, changed only when it changes, and a word +// about another module than its subject's is refused. +func TestAModulesWordIsKeptUnderItsNameAndOnlyItsOwn(t *testing.T) { + k := announcedInMemory{} + word, _ := announce.Encode(announce.States{Module: "keeper", States: []announce.State{{Name: "things", Readable: true}}}) + changed, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), word) + if err != nil || !changed || string(k["keeper"]) != string(word) { + t.Fatalf("the first word: %v %v %q", changed, err, k["keeper"]) + } + if changed, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), word); err != nil || changed { + t.Fatalf("the same word again changed the bucket: %v %v", changed, err) + } + other, _ := announce.Encode(announce.States{Module: "victim", States: []announce.State{{Name: "secrets", Readable: true}}}) + if _, err := keepAnnouncement(t.Context(), k, announce.Subject("keeper"), other); err == nil || + !strings.Contains(err.Error(), "only what it keeps itself") { + t.Fatalf("a word about another module was kept: %v", err) + } + if _, found := k["victim"]; found { + t.Fatal("a word about another module was kept") + } +} diff --git a/cmd/mesh-controller/status.go b/cmd/mesh-controller/status.go index 5d262e36..93191908 100644 --- a/cmd/mesh-controller/status.go +++ b/cmd/mesh-controller/status.go @@ -85,6 +85,13 @@ func printStatus(asked answers) error { // **What is wrong leads** (novox/hq to-be 45 §2): every open condition, urgent first, oldest // first, silenced ones with when their silence ends. printConditions(asked.conditions, asked.conditionsUnread, time.Now()) + if len(asked.notAnnounced) > 0 { + fmt.Printf("%d module(s) others read the state of have not said what they keep, and are read from `reads` as "+ + "before until they do (ADR 0495): %s\n\n", len(asked.notAnnounced), strings.Join(asked.notAnnounced, ", ")) + } + if asked.notAnnouncedWhy != "" { + fmt.Printf("whose states are read without an announcement cannot be said: %s\n\n", asked.notAnnouncedWhy) + } if len(asked.refused) > 0 { // First of what follows. A machine that cannot be worked out is not running an old @@ -435,6 +442,11 @@ func theThreeQuestions(ctx context.Context, open *stores) (answers, error) { if err != nil { return answers{}, err } + // And whose states are still read without an announcement (novox/hq ADR 0495, issue 1790): said, never a reason + // for the rest of the answer to fail. + if out.notAnnounced, err = notYetAnnounced(ctx, inv); err != nil { + out.notAnnounced, out.notAnnouncedWhy = nil, err.Error() + } // And what each machine is holding rather than running, by the module that would run it. Read // from what the machine itself last reported, not from what take-time computed: the machine is // the only thing that knows what it found (novox/hq 04-ISSUES/125). diff --git a/internal/broker/announced.go b/internal/broker/announced.go new file mode 100644 index 00000000..9d1fdcae --- /dev/null +++ b/internal/broker/announced.go @@ -0,0 +1,81 @@ +package broker + +import ( + "sort" + "strings" + + "git.novox.be/novox/mesh-sdk/go/announce" +) + +// Who may read a module's state is what the module announced (novox/hq ADR 0495 clauses 2 and 3, ADR 0513 §3; issue +// 1790). A module says on the bus which states it keeps and which others may read (announce.Subject); the controller +// keeps the last word of each (announce.KeptBucket) and grants a reader's `reads: .` only when the owner +// announced that state readable. An owner that has announced nothing yet is read as before, from the reader's `reads` +// (the transition, until every module announces), and is named as not yet announced. + +// StatesAnnouncedBucket is the controller's bucket of the last announcement of each module (announce.KeptBucket). +const StatesAnnouncedBucket = announce.KeptBucket + +// controllerReadable are the controller's states others may read: the record mirrors, the chain's buckets, the checks +// running and the plans building. +var controllerReadable = []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, + ChecksRunning, PlansBuilding} + +// ControllerAnnouncement is what the controller announces it keeps (ADR 0495 clause 2): every bucket of its own by its local +// name, the ones others may read marked readable. It announces itself, as any module does from its code. +func ControllerAnnouncement(node string) announce.States { + s := announce.States{Module: ControllerSeat, Node: node} + readable := map[string]bool{} + for _, n := range controllerReadable { + readable[n] = true + } + for _, bucket := range ControllerBuckets() { + local, mine := strings.CutPrefix(bucket, ControllerSeat+"_") + if !mine { + continue + } + s.States = append(s.States, announce.State{Name: local, Readable: readable[local]}) + } + sort.Slice(s.States, func(i, j int) bool { return s.States[i].Name < s.States[j].Name }) + return s +} + +// ReadsAnnounced splits a reader's `reads` (`.`) by what each owner announced: granted when the owner +// announced the state readable, or announced nothing yet (the transition); refused when the owner announced and did +// not announce that state readable. +func ReadsAnnounced(reads []string, announced map[string]announce.States) (granted, refused []string) { + for _, r := range reads { + owner, name, ok := strings.Cut(r, ".") + s, said := announced[owner] + switch { + case !ok || !said || s.Readable(name): + granted = append(granted, r) + default: + refused = append(refused, r) + } + } + return granted, refused +} + +// NotAnnounced are the owners some module reads the state of and that have announced nothing yet, sorted: read from the +// readers' `reads` as before, until they announce. +func NotAnnounced(r Records) []string { + seen := map[string]bool{} + var out []string + for _, ds := range r.Assigned { + for _, d := range ds { + for _, read := range append(append([]string(nil), d.Reads...), d.ReadsRefused...) { + owner, _, ok := strings.Cut(read, ".") + if !ok || seen[owner] { + continue + } + if _, said := r.Announced[owner]; !said { + seen[owner] = true + out = append(out, owner) + } + } + } + } + sort.Strings(out) + return out +} diff --git a/internal/broker/announced_test.go b/internal/broker/announced_test.go new file mode 100644 index 00000000..cca553df --- /dev/null +++ b/internal/broker/announced_test.go @@ -0,0 +1,85 @@ +package broker + +import ( + "slices" + "testing" + + "git.novox.be/novox/mesh-sdk/go/announce" +) + +// D3 (ADR 0495 clause 3, ADR 0513 §3; issue 1790): a reader's `reads` is granted where the owner announced the state +// readable, refused where it announced and did not, and granted as before where the owner announced nothing yet. +func TestAReadIsGrantedFromItsOwnersAnnouncement(t *testing.T) { + announced := map[string]announce.States{ + "owner": {Module: "owner", States: []announce.State{{Name: "open", Readable: true}, {Name: "closed"}}}, + } + granted, refused := ReadsAnnounced([]string{"owner.open", "owner.closed", "owner.unsaid", "quiet.any"}, announced) + if !slices.Equal(granted, []string{"owner.open", "quiet.any"}) || !slices.Equal(refused, []string{"owner.closed", "owner.unsaid"}) { + t.Fatalf("granted %v, refused %v", granted, refused) + } + r := Records{Announced: announced, Assigned: map[string][]Declared{"one": {{Module: "reader", Reads: granted, + ReadsRefused: refused}}}} + if got := NotAnnounced(r); !slices.Equal(got, []string{"quiet"}) { + t.Fatalf("not yet announced: %v", got) + } +} + +// D1 (ADR 0495 clause 2): the controller announces every bucket of its own, and readable exactly the record mirrors, +// the chain's buckets, the checks running and the plans building. +func TestTheControllerAnnouncesWhatItKeeps(t *testing.T) { + s := ControllerAnnouncement("novox") + if _, err := announce.Encode(s); err != nil { + t.Fatal(err) + } + var readable []string + for _, k := range s.States { + if k.Readable { + readable = append(readable, k.Name) + } + } + want := []string{ChainBuilds, ChecksRunning, MirrorDeclaredConfig, PlansBuilding, MirrorPredefined, ChainRollouts, + MirrorAssignments} + slices.Sort(want) + if !slices.Equal(readable, want) { + t.Fatalf("readable %v, want %v", readable, want) + } + for _, name := range []string{"lease", "calls", "states-announced", "merge-window"} { + if !s.Keeps(name) || s.Readable(name) { + t.Errorf("%s: kept %v, readable %v", name, s.Keeps(name), s.Readable(name)) + } + } + perms, err := PermissionsFor(Principal{Kind: KindController}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(perms.Publish, announce.Subject(ControllerSeat)) { + t.Fatal("the controller may not say what it keeps") + } +} + +// D2: every account that keeps state may say so on its own subject, a process account included, and no other's. +func TestAnAccountKeepingStateMaySayWhatItKeeps(t *testing.T) { + keeper, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "keeper", PasswordHash: "x", + State: []string{"things"}}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(keeper.Publish, announce.Subject("keeper")) || slices.Contains(keeper.Publish, announce.Subject("other")) { + t.Fatalf("the keeper's announcement grants: %v", keeper.Publish) + } + process, err := PermissionsFor(Principal{Kind: KindProcess, Node: "one", Module: "keeper", Process: "send", + PasswordHash: "x", State: []string{"sends"}}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(process.Publish, announce.Subject("keeper")) { + t.Fatal("a process keeping state may not say what it keeps") + } + none, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "stateless", PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + if slices.Contains(none.Publish, announce.Subject("stateless")) { + t.Fatal("a module keeping nothing may say it keeps something") + } +} diff --git a/internal/broker/announced_test_helpers_test.go b/internal/broker/announced_test_helpers_test.go new file mode 100644 index 00000000..e8b1b5a2 --- /dev/null +++ b/internal/broker/announced_test_helpers_test.go @@ -0,0 +1,13 @@ +package broker + +// controllerReadableStates are the controller's states its announcement makes readable (ADR 0495 clause 2): what a +// module may name in `reads` after `mesh-controller.`. +func controllerReadableStates() []string { + var out []string + for _, s := range ControllerAnnouncement("").States { + if s.Readable { + out = append(out, s.Name) + } + } + return out +} diff --git a/internal/broker/checks_running_mirror_test.go b/internal/broker/checks_running_mirror_test.go index a19f1dcf..aec691ea 100644 --- a/internal/broker/checks_running_mirror_test.go +++ b/internal/broker/checks_running_mirror_test.go @@ -8,8 +8,8 @@ import ( // 1155/AC1: a module whose manifest reads mesh-controller.checks-running may watch and get every key of the bucket and // write none. func TestAModuleReadingChecksRunningReadsAndWritesNone(t *testing.T) { // 1155/AC1 - if !slices.Contains(ControllerMirrors(), "checks-running") || BucketName(ControllerSeat, "checks-running") != CheckRunningBucket { - t.Fatalf("a module cannot name %s in `reads`: %v", CheckRunningBucket, ControllerMirrors()) + if !slices.Contains(controllerReadableStates(), "checks-running") || BucketName(ControllerSeat, "checks-running") != CheckRunningBucket { + t.Fatalf("a module cannot name %s in `reads`: %v", CheckRunningBucket, controllerReadableStates()) } perms, err := PermissionsFor(Principal{Kind: KindModule, Node: "one", Module: "mesh-delivery-watchman", PasswordHash: "x", Reads: []string{"mesh-controller.checks-running"}}) diff --git a/internal/broker/controller_buckets.go b/internal/broker/controller_buckets.go index fe588c01..f3aad638 100644 --- a/internal/broker/controller_buckets.go +++ b/internal/broker/controller_buckets.go @@ -150,12 +150,6 @@ const ( // ChainBuckets are the three buckets of the chain: builds, rollouts and assignments. func ChainBuckets() []string { return []string{BuildsBucket, RolloutsBucket, AssignmentsBucket} } -// ControllerMirrors are the local names of the controller's buckets a module may read: the record mirrors and the -// chain's buckets. -func ControllerMirrors() []string { - return []string{MirrorAssignments, MirrorDeclaredConfig, MirrorPredefined, ChainBuilds, ChainRollouts, ChecksRunning, PlansBuilding} -} - // MirrorsWrittenByAnother are the controller's mirrors whose writer is not the controller, each with its reason: the rule // that a mirror is written by the controller alone (TestTheMirrorsListAndTheWritersTableAgree) has these exceptions and no // other. @@ -224,14 +218,14 @@ func IsControllerBucket(bucket string) bool { bucket == AssignmentsBucket || bucket == DeclaredConfigBucket || bucket == PredefinedBucket || bucket == MergeWindowBucket || bucket == RolloutSilentBucket || bucket == CheckRunningBucket || bucket == DeliveryMergesBucket || bucket == BuildsBucket || bucket == RolloutsBucket || - bucket == PlansBuildingBucket || bucket == delivery.HaveBucket + bucket == PlansBuildingBucket || bucket == delivery.HaveBucket || bucket == StatesAnnouncedBucket } // ControllerBuckets are the controller's own buckets, in the order they are asserted. func ControllerBuckets() []string { return []string{LeaseBucket, CallsBucket, HandActsBucket, ConditionsBucket, ConditionHistoryBucket, AskedBucket, AssignmentsBucket, DeclaredConfigBucket, PredefinedBucket, MergeWindowBucket, RolloutSilentBucket, CheckRunningBucket, - DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket} + DeliveryMergesBucket, PlansBuildingBucket, BuildsBucket, RolloutsBucket, delivery.HaveBucket, StatesAnnouncedBucket} } // ControllerBucketsAsserter is what raising the controller's buckets needs of a connection. @@ -390,6 +384,18 @@ func (j *JetStream) EnsureControllerBuckets() error { }); err != nil { return fmt.Errorf("asserting bucket %s: %w", PlansBuildingBucket, err) } + // **The states each module announced** (novox/hq ADR 0495 clauses 2 and 3, issue 1790): one key a module, its last word. + if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ + Bucket: StatesAnnouncedBucket, + Description: "the states each module announced it keeps and which of them others may read (novox/hq ADR 0495): " + + "`` holds its last announcement; what a reader's `reads` is granted from; written by the controller alone", + History: 1, + MaxValueSize: 64 << 10, + MaxBytes: 16 << 20, + Storage: jetstream.FileStorage, + }); err != nil { + return fmt.Errorf("asserting bucket %s: %w", StatesAnnouncedBucket, err) + } // **The merges mesh-delivery announced: keys that age** (ADR 0420 §7, issue 1025). One message a key; a week. if _, err := js.CreateOrUpdateKeyValue(ctx, jetstream.KeyValueConfig{ Bucket: DeliveryMergesBucket, diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 228f05c0..f0637377 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -100,7 +100,7 @@ func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) { // stands for their declaration, so the report of state nothing declares never names one. func TestAListedMirrorIsNotReportedAsUndeclared(t *testing.T) { names := []string{"gone_state"} - for _, m := range ControllerMirrors() { + for _, m := range controllerReadableStates() { names = append(names, BucketName(ControllerSeat, m)) } undeclared, err := RaiseBuckets(fakeBuckets{names: names}, nil) @@ -132,7 +132,7 @@ func TestTheMirrorsListAndTheWritersTableAgree(t *testing.T) { } } listed := map[string]bool{} - for _, m := range ControllerMirrors() { + for _, m := range controllerReadableStates() { bucket := BucketName(ControllerSeat, m) listed[bucket] = true if why, other := MirrorsWrittenByAnother[m]; other { // 1155/AC3: the exception, with its reason @@ -292,7 +292,7 @@ func TestEveryChainBucketIsOnTheControllersListAndHasAWriterRow(t *testing.T) { t.Errorf("%s is not on the controller's list of buckets", bucket) } local := strings.TrimPrefix(bucket, ControllerSeat+"_") - if !slices.Contains(ControllerMirrors(), local) { + if !slices.Contains(controllerReadableStates(), local) { t.Errorf("a module cannot name %s in `reads`", bucket) } rows := 0 diff --git a/internal/broker/delivery_parts_grant_test.go b/internal/broker/delivery_parts_grant_test.go index fbfbe001..49edefeb 100644 --- a/internal/broker/delivery_parts_grant_test.go +++ b/internal/broker/delivery_parts_grant_test.go @@ -2,6 +2,7 @@ package broker import ( "encoding/json" + "git.novox.be/novox/mesh-sdk/go/announce" "slices" "sort" "strings" @@ -407,7 +408,8 @@ func withPrefix(patterns []string, prefixes ...string) []string { func eventsSaid(p Permissions) []string { var out []string for _, s := range withPrefix(p.Publish, "mesh.mod.", "mesh.stream.") { - if strings.Contains(s, ".event.") { + // A module keeping state says what it keeps (ADR 0495 clause 2): no event of its declaration. + if strings.Contains(s, ".event.") && !strings.HasSuffix(s, ".event."+announce.Event) { out = append(out, s) } } diff --git a/internal/broker/nats.go b/internal/broker/nats.go index 0f8466fa..e61dfbd0 100644 --- a/internal/broker/nats.go +++ b/internal/broker/nats.go @@ -15,6 +15,7 @@ package broker import ( "errors" "fmt" + "git.novox.be/novox/mesh-sdk/go/announce" "regexp" "slices" "sort" @@ -389,6 +390,16 @@ type DegradableGrant struct { // works. Every other grant stays required — missing one of those is a controller that comes up and is // refused on the first thing it tries, with no fallback that covers it. var DegradableGrants = []DegradableGrant{ + { + Subject: announce.Subject(ControllerSeat), + Reason: "refused: the controller's own announcement waits for its first push, and until it is kept every reader of " + + "its states is granted from its `reads` as before (novox/hq ADR 0495, issue 1790)", + }, + { + Subject: "$KV." + StatesAnnouncedBucket + ".>", + Reason: "refused: no announcement is kept until the controller's first push, and until then every reader is " + + "granted from its `reads` as before (novox/hq ADR 0495, issue 1790)", + }, { Subject: "mesh.seat." + ControllerSeat + ".tool.push", Reason: "refused: the ask of a held move (Release) is answered, and the refusal told, while the operator's own " + @@ -751,6 +762,8 @@ func PermissionsFor(p Principal) (Permissions, error) { // **And its own buckets** (novox/hq to-be 45 §1): the calls it served and the acts done by // hand, which it alone writes. A put is a publish to the bucket's subject, which `$JS.API.>` // does not cover; each bucket named, not `$KV.>`, which would let it write any module's state. + // And says what it keeps, as every module does (novox/hq ADR 0495 clause 2, issue 1790). + pub = append(pub, announce.Subject(ControllerSeat)) for _, bucket := range ControllerBuckets() { if bucket == delivery.HaveBucket { continue // each node's node-engine writes its own keys of it, the controller none (ADR 0429 §1) diff --git a/internal/broker/plans_building_mirror_test.go b/internal/broker/plans_building_mirror_test.go index 43efd6db..9b57f4d3 100644 --- a/internal/broker/plans_building_mirror_test.go +++ b/internal/broker/plans_building_mirror_test.go @@ -8,8 +8,8 @@ import ( // Issue 1687 (ADR 0483 clause 2): a module whose manifest reads mesh-controller.plans-building may name it in `reads`, and // reads the bucket and writes none; the controller writes it, so it is no mirror written by another. func TestAModuleReadingPlansBuildingReadsAndWritesNone(t *testing.T) { - if !slices.Contains(ControllerMirrors(), "plans-building") || BucketName(ControllerSeat, PlansBuilding) != PlansBuildingBucket { - t.Fatalf("a module cannot name %s in `reads`: %v", PlansBuildingBucket, ControllerMirrors()) + if !slices.Contains(controllerReadableStates(), "plans-building") || BucketName(ControllerSeat, PlansBuilding) != PlansBuildingBucket { + t.Fatalf("a module cannot name %s in `reads`: %v", PlansBuildingBucket, controllerReadableStates()) } if _, other := MirrorsWrittenByAnother[PlansBuilding]; other { t.Error("the controller writes the plans building itself") diff --git a/internal/broker/process_grants_test.go b/internal/broker/process_grants_test.go index 7866df30..93259427 100644 --- a/internal/broker/process_grants_test.go +++ b/internal/broker/process_grants_test.go @@ -48,6 +48,8 @@ func TestAProcessIsGrantedExactlyItsBusDeclaration(t *testing.T) { "$KV.mesh-mail_sends.>", "mesh.mod.mesh-mail.event.mail.send.failed", "mesh.mod.mesh-mail.event.mail.send.sent", + // It keeps state, so it says what it keeps (novox/hq ADR 0495 clause 2, issue 1790). + "mesh.mod.mesh-mail.event.states", "mesh.seat.operator-channel.call.tell.*.novox~mesh-mail~send", "mesh.seat.operator-channel.call.tell.novox~mesh-mail~send", "mesh.seat.operator-channel.tool.tell", diff --git a/internal/broker/state.go b/internal/broker/state.go index 25c7c7ab..b89b3217 100644 --- a/internal/broker/state.go +++ b/internal/broker/state.go @@ -2,6 +2,7 @@ package broker import ( "fmt" + "git.novox.be/novox/mesh-sdk/go/announce" "sort" "strings" "time" @@ -147,6 +148,11 @@ func stateGrants(a stateAccess) []string { read(bucket, "") out = append(out, "$KV."+bucket+".>") } + // **And it says what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): a module keeping state announces it on its + // own event subject, from its code; refused, its readers are read as before. + if len(a.Keeps) > 0 && safeSubject.MatchString(a.Module) { + out = append(out, announce.Subject(a.Module)) + } for _, r := range a.Reads { if bucket, ok := bucketOfRead(r); ok { read(bucket, "") diff --git a/internal/broker/testdata/composed.conf b/internal/broker/testdata/composed.conf index 4e2ba482..c3f0da3e 100644 --- a/internal/broker/testdata/composed.conf +++ b/internal/broker/testdata/composed.conf @@ -29,7 +29,7 @@ accounts { ] users = [ { user: "controller", password: "$2a$11$cccccccccccccccccccccc", permissions: { - publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } + publish: { allow: ["$JS.ACK.CONTROL.controller.>", "$JS.ACK.DEAD_LETTER_NOTICES.controller.>", "$JS.ACK.EVENTS.controller-merges.>", "$JS.ACK.EVENTS.controller.>", "$JS.API.>", "$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>", "$KV.SEAT_NODE_BUILD_AGENT_cancelled.>", "$KV.mesh-controller_asked.>", "$KV.mesh-controller_assignments.>", "$KV.mesh-controller_builds.>", "$KV.mesh-controller_calls.>", "$KV.mesh-controller_checks-running.>", "$KV.mesh-controller_condition-history.>", "$KV.mesh-controller_conditions.>", "$KV.mesh-controller_declared-config.>", "$KV.mesh-controller_delivery-merges.>", "$KV.mesh-controller_hand-acts.>", "$KV.mesh-controller_lease.>", "$KV.mesh-controller_merge-window.>", "$KV.mesh-controller_plans-building.>", "$KV.mesh-controller_predefined.>", "$KV.mesh-controller_rollout-silent.>", "$KV.mesh-controller_rollouts.>", "$KV.mesh-controller_states-announced.>", "$SRV.INFO", "_INBOX.enrol.>", "mesh.again.>", "mesh.assignment.>", "mesh.events.dead.>", "mesh.mod.*.tool.>", "mesh.mod.mesh-controller.event.states", "mesh.node.>", "mesh.seat.mesh-build-machine.accept.>", "mesh.seat.mesh-build-machine.tool.>", "mesh.seat.mesh-controller.event.applied", "mesh.seat.mesh-controller.event.assignment-changed", "mesh.seat.mesh-controller.event.built-before", "mesh.seat.mesh-controller.event.checked", "mesh.seat.mesh-controller.event.condition-changed", "mesh.seat.mesh-controller.event.condition-cleared", "mesh.seat.mesh-controller.event.condition-raised", "mesh.seat.mesh-controller.event.declared-config-changed", "mesh.seat.mesh-controller.event.doctor-heartbeat", "mesh.seat.mesh-controller.event.healer-acted", "mesh.seat.mesh-controller.event.plan-moved", "mesh.seat.mesh-controller.event.predefined-changed", "mesh.seat.mesh-controller.event.refused", "mesh.seat.mesh-controller.event.rolled-back", "mesh.seat.mesh-controller.event.secret-replaced", "mesh.seat.mesh-controller.event.sent", "mesh.seat.mesh-controller.tool.plans", "mesh.seat.mesh-controller.tool.push", "mesh.seat.mesh-controller.tool.upgrade", "mesh.seat.mesh-delivery.tool.close", "mesh.seat.mesh-delivery.tool.groups", "mesh.seat.mesh-delivery.tool.release", "mesh.seat.mesh-delivery.tool.stalled", "mesh.seat.mesh-delivery.tool.stop", "mesh.seat.mesh-delivery.tool.times", "mesh.seat.node-backup.tool.backed-up.*", "mesh.seat.node-backup.tool.now.*", "mesh.seat.node-build-agent.accept.>", "mesh.seat.node-build-agent.check.>", "mesh.seat.node-build-agent.tool.>", "mesh.seat.node-intrusion-prevention.tool.banned.*", "mesh.seat.node-launcher.tool.secret.*", "mesh.seat.node-service-manager.tool.restart.*", "mesh.seat.operator-channel.tool.tell"] } subscribe: { allow: ["$JS.API.>", "$JS.EVENT.ADVISORY.CONSUMER.DELETED.>", "$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>", "$SRV.INFO", "$SRV.INFO.mesh-controller", "$SRV.INFO.mesh-controller.>", "$SRV.PING", "$SRV.PING.mesh-controller", "$SRV.PING.mesh-controller.>", "$SRV.STATS", "$SRV.STATS.mesh-controller", "$SRV.STATS.mesh-controller.>", "_DELIVER.controller", "_DELIVER.controller.>", "_INBOX.controller.>", "mesh.control.>", "mesh.mod.*.event.provisioner.failing", "mesh.mod.*.event.provisioner.recovered", "mesh.mod.*.event.provisioner.retirement", "mesh.mod.gitea.event.pull.merged", "mesh.mod.gitea.event.pull.updated", "mesh.mod.mesh-catalog.event.catching-up", "mesh.mod.mesh-catalog.event.upgraded", "mesh.seat.mesh-build-machine.event.built", "mesh.seat.mesh-controller.tool.>", "mesh.seat.node-build-agent.event.built", "mesh.seat.operator-channel.event.decided.mesh-controller"] } allow_responses: { max: 1, ttl: "1m" } } } diff --git a/internal/broker/users.go b/internal/broker/users.go index a63e281e..123e436e 100644 --- a/internal/broker/users.go +++ b/internal/broker/users.go @@ -2,6 +2,7 @@ package broker import ( "fmt" + "git.novox.be/novox/mesh-sdk/go/announce" "slices" "sort" ) @@ -56,6 +57,9 @@ type Declared struct { HearsAStream bool // Reads are other modules' state it reads, each `.` (novox/hq ADR 0201). Reads []string + // ReadsRefused are the reads its manifest names that the owner's announcement does not make readable (novox/hq ADR + // 0495 clause 3, issue 1790): granted nothing, kept to be said. + ReadsRefused []string // KeyedReads are keys of other modules' state it reads, each one key alone: what a seat's holder is // granted for the pieces the modules beside it offer (novox/hq ADR 0260). KeyedReads []KeyedRead @@ -126,6 +130,9 @@ type Records struct { // account agents run as, judged unable to become root by its node-engine, and serves no login shell // execute. A machine absent is not free: no record of a pass is no pass. RootFree map[string]bool + // Announced is each module's last announcement of the states it keeps (novox/hq ADR 0495, issue 1790), as the + // controller keeps it; a module absent has announced nothing yet. Nil where none was read. + Announced map[string]announce.States } // Users is every user the composed file should contain, in the order it will be written. diff --git a/internal/broker/writers.go b/internal/broker/writers.go index 9dd7ed3f..b31d0192 100644 --- a/internal/broker/writers.go +++ b/internal/broker/writers.go @@ -87,6 +87,22 @@ func ownModule(p Principal, pattern string) bool { return false } +// ownAnnouncer is a module, a process of it or the node tools carrying it saying what that module keeps, or the +// controller saying what it keeps itself (novox/hq ADR 0495 clause 2): each under its own name alone. +func ownAnnouncer(p Principal, pattern string) bool { + tokens := strings.Split(pattern, ".") + if len(tokens) < 3 { + return false + } + switch p.Kind { + case KindController: + return tokens[2] == ControllerSeat + case KindProcess: + return p.Module == tokens[2] + } + return ownModule(p, pattern) +} + // ownProvisioner is a machine's provisioner client saying a report about its own machine: the third token of // `prov.applied..…` is its machine (novox/hq ADR 0349 §4, design 57 P3). func ownProvisioner(p Principal, pattern string) bool { @@ -181,6 +197,14 @@ var WritersTable = []WriterRow{ KeptIn: "key-value " + PlansBuildingBucket + ", aged out when not refreshed", Others: "the build seat's holders read it and take no new check while a key is there (ADR 0483)", Subjects: kvOf(PlansBuildingBucket), Writes: isController}, + // What a module keeps and who may read it (novox/hq ADR 0495 clauses 2 and 3, issue 1790): said by the module, kept by + // the controller. + {State: "a module's states announced", Writer: "the module itself, from its code, under its own name", + KeptIn: "the bus (EVENTS)", Others: "the controller keeps the last of each and grants readers from it", + Subjects: []string{"mesh.mod.*.event.states"}, Writes: ownAnnouncer}, + {State: "the states announced", Writer: "controller, from each module's announcement", + KeptIn: "key-value " + StatesAnnouncedBucket, Others: "the composition of the bus's users reads it", + Subjects: kvOf(StatesAnnouncedBucket), Writes: isController}, {State: "a merge announced", Writer: "one announcer per forge (the hook, or the poll when the hook is absent — never both)", KeptIn: "the bus", Others: "—", Subjects: []string{"mesh.mod.*.event.pull.merged"}, Writes: ownModule}, // A pull request's head, before it merges (novox/hq to-be 45 §9): the same one announcer. diff --git a/internal/broker/writers_test.go b/internal/broker/writers_test.go index 24128be7..d01005ce 100644 --- a/internal/broker/writers_test.go +++ b/internal/broker/writers_test.go @@ -31,6 +31,9 @@ var designRows = []string{ "a pull request's check running", // The plans in their build phase, a bucket the build seat's holders read (ADR 0483 clause 2, issue 1687). "the plans-building mirror", + // What each module keeps and who may read it, said by the module and kept by the controller (ADR 0495, issue 1790). + "a module's states announced", + "the states announced", "a merge announced", "a pull request's head announced", "a pull request's merge check", diff --git a/internal/inventory/busrecords.go b/internal/inventory/busrecords.go index a3fdd616..de25fb28 100644 --- a/internal/inventory/busrecords.go +++ b/internal/inventory/busrecords.go @@ -3,6 +3,7 @@ package inventory import ( "context" "fmt" + "git.novox.be/novox/mesh-sdk/go/announce" "slices" "strings" @@ -16,6 +17,10 @@ import ( // apart for the reason that package keeps its own types: a permission must be a function of what a // module declared, and a query that decided anything would be a second place authority came from. +// Announcements reads the last announcement of each module (novox/hq ADR 0495, issue 1790), set by the controller that +// reaches the bus; nil, nothing is read and every read is granted from `reads`. +var Announcements func(ctx context.Context) (map[string]announce.States, error) + // BusRecords is every fact the composer needs about who may reach the bus. // // **A module's authority comes from the manifest, not from the assignment.** The assignment says @@ -110,6 +115,29 @@ func (i *Inventory) BusRecords(ctx context.Context) (broker.Records, error) { } } + // **Who may read a state is what its owner announced** (novox/hq ADR 0495 clause 3, issue 1790): a read the owner's + // announcement does not make readable is granted nothing, and said; an owner that announced nothing yet is read as + // before. Announcements that cannot be read are said, and every read is granted as before: the bus's users are + // composed either way. + if Announcements != nil { + announced, err := Announcements(ctx) + if err != nil { + fmt.Printf("the states the modules announced cannot be read, so every read is granted from `reads` as before: %v\n", err) + } else { + out.Announced = announced + for node, ds := range out.Assigned { + for k := range ds { + granted, refused := broker.ReadsAnnounced(ds[k].Reads, announced) + ds[k].Reads, ds[k].ReadsRefused = granted, refused + for _, r := range refused { + fmt.Printf("%s on %s reads %s, which its owner does not announce readable: not granted\n", + ds[k].Module, node, r) + } + } + } + } + } + enrolling, err := i.NodesWithALiveToken(ctx) if err != nil { return broker.Records{}, err diff --git a/internal/recordmirror/readers_test.go b/internal/recordmirror/readers_test.go index 91736ad0..83a9faf2 100644 --- a/internal/recordmirror/readers_test.go +++ b/internal/recordmirror/readers_test.go @@ -28,6 +28,9 @@ var mayNameAMirror = []string{ // The grant of a machine's own keys of the assignments mirror to its provisioner client (design 57 P3). "internal/broker/provisioner.go", "internal/broker/writers.go", + // The controller's announcement of what it keeps, the mirrors readable (novox/hq ADR 0495, issue 1790): it names them + // to announce them, never reads them. + "internal/broker/announced.go", } // The packages whose names stand for a mirror, by import path, and the names in each. @@ -38,7 +41,7 @@ const ( var mirrorNames = map[string][]string{ brokerPath: {"AssignmentsBucket", "DeclaredConfigBucket", "MirrorAssignments", "MirrorDeclaredConfig", - "ControllerMirrors"}, + "controllerReadable"}, recordmirrorPath: {"Assignments", "DeclaredConfig", "Mirrors"}, } @@ -54,8 +57,10 @@ var mirrorNames = map[string][]string{ // allowed file. Those are deliberate evasions rather than ordinary code; a reviewer judges them. func TestOnlyTheAuditAndAMirrorsWriterReadAMirror(t *testing.T) { var buckets []string - for _, m := range broker.ControllerMirrors() { - buckets = append(buckets, broker.BucketName(broker.ControllerSeat, m)) + for _, s := range broker.ControllerAnnouncement("").States { + if s.Readable { + buckets = append(buckets, broker.BucketName(broker.ControllerSeat, s.Name)) + } } root := filepath.Join("..", "..") found := mirrorReaders(t, root, buckets) From 791c2a3d48b4d11d2df6de1adc122f6bb528135b Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:36:18 +0200 Subject: [PATCH 23/38] Refuse a removed verb by name before reading its arguments, as the SDK member's table now offers none (issue 1851, A11a; issue 1318, C3) --- cmd/mesh-controller/delivery_test.go | 2 -- cmd/mesh-controller/queue_test.go | 2 -- cmd/mesh-controller/seatverbs.go | 46 ++++++++++----------------- cmd/mesh-controller/seatverbs_test.go | 21 ++++++------ 4 files changed, 28 insertions(+), 43 deletions(-) diff --git a/cmd/mesh-controller/delivery_test.go b/cmd/mesh-controller/delivery_test.go index e21c319e..30478a90 100644 --- a/cmd/mesh-controller/delivery_test.go +++ b/cmd/mesh-controller/delivery_test.go @@ -138,8 +138,6 @@ func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { {"delivery-order", map[string]any{"members": "[]"}, []string{"delivery", "order", "--members", "[]"}}, {"delivery-check", map[string]any{"group": "feat/x", "members": "[]"}, []string{"delivery", "check", "--group", "feat/x", "--members", "[]"}}, - {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x", "head": "c0"}, - []string{"delivery", "plan"}}, {"plans", map[string]any{"go": "plan-1", "why": "down"}, []string{"plans", "go", "plan-1", "--why", "down"}}, } { got, err := argvFor(c.verb, c.args) diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index da2fea1e..c5a4e281 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -158,9 +158,7 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) { {"cancel", map[string]any{"id": "build-1"}, []string{"cancel", "build-1"}}, {"clear", nil, []string{"clear"}}, {"clear", map[string]any{"dead": "true"}, []string{"clear", "--dead"}}, - {"rebuild", map[string]any{"what": "gitea"}, []string{"rebuild"}}, {"replay", map[string]any{"id": "build-1"}, []string{"replay", "build-1"}}, - {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, []string{"replay", "build-1", "--register"}}, {"kill", map[string]any{"id": "build-1"}, []string{"kill", "build-1"}}, {"pause", nil, []string{"pause"}}, {"resume", map[string]any{"node": "ace"}, []string{"resume", "ace"}}, diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index 2f73d897..b27187f3 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -52,6 +52,17 @@ type verbAnswer struct { // verb without the machine and ran as a push of every machine behind; a verb that answers "I did // not take that" would have stopped it before anything was sent. func argvFor(verb string, args map[string]any) ([]string, error) { + // **A removed verb is refused by name, naming its replacement, whatever was asked of it** (novox/hq ADR 0501 §5, + // ADR 0503 §3, ADR 0450 §8): before its arguments are read, so an argument a table of before offered is not what the + // caller is told. + if removedVerbs[verb] { + return nil, removedByName(verb) + } + if verb == "replay" { + if v, given := args["register"]; given && v != false && v != "false" { + return nil, removedByName("replay --register") + } + } a, err := readArguments(verb, args) if err != nil { return nil, err @@ -86,13 +97,9 @@ type verbArguments struct { misread []string // arguments the command line read that the schema does not declare: a bug here } -// passOver takes every argument given as read: a removed verb is refused by name whatever was asked of it (novox/hq ADR -// 0501 §5), so no argument of a table of before is refused as passed over in its place. -func (a *verbArguments) passOver() { - for k := range a.given { - a.used[k] = true - } -} +// removedVerbs are the controller seat's verbs R4e removed, kept in its table with no input so a caller is answered +// by name (ADR 0501 §5, ADR 0503 §3, ADR 0450 §8). +var removedVerbs = map[string]bool{"build": true, "rebuild": true, "forget-checks": true, "delivery-plan": true} // controllerVerb is this binary's own definition of a verb: what it runs is what it declares, so the // arguments are checked against the table compiled beside argvFor, not a row a newer or older build @@ -411,10 +418,6 @@ func (a *verbArguments) commandLine() ([]string, error) { } return argv, nil // The delivery's owner's verbs (novox/hq ADR 0239). - case "delivery-plan": - // Removed (ADR 0450 §8; issue 1318): the command refuses it by name, whatever was asked. - a.passOver() - return []string{"delivery", "plan"}, nil case "delivery-order": if err := need("members"); err != nil { return nil, err @@ -485,10 +488,6 @@ func (a *verbArguments) commandLine() ([]string, error) { case "queue": // As data, the table its output (novox/hq issue 455). return []string{"queue", "--json"}, nil - case "forget-checks": - // Removed (ADR 0503 §3; issue 1318): the command refuses it by name, whatever was asked. - a.passOver() - return []string{"forget-checks"}, nil case "try": // A branch tried before its pull request (novox/hq issue 686): asked, or read by the id it answered. if id := str("id"); id != "" { @@ -521,21 +520,12 @@ func (a *verbArguments) commandLine() ([]string, error) { return []string{"clear", "--dead"}, nil } return []string{"clear"}, nil - case "rebuild": - // Removed (ADR 0501 §5): the command refuses it by name, naming its replacement, whatever was asked. - a.passOver() - return []string{"rebuild"}, nil case "replay": if err := need("id"); err != nil { return nil, err } - argv := []string{"replay", str("id")} - // A table of before still offers register: passed on, so the command refuses it by name (ADR 0501 §5). - if on("register") { - argv = append(argv, "--register") - } - a.passOver() - return argv, nil + // A dry run alone (ADR 0429 §18): register is refused by name before this (argvFor). + return []string{"replay", str("id")}, nil case "pause", "resume": if n := str("node"); n != "" { return []string{verb, n}, nil @@ -1073,10 +1063,6 @@ func (a *verbArguments) commandLine() ([]string, error) { return nil, err } return []string{"module", "issue", str("module"), "--node", str("node")}, nil - case "build": - // Removed (ADR 0501 §5): the command refuses it by name, naming its replacement, whatever was asked. - a.passOver() - return []string{"build"}, nil } return nil, fmt.Errorf("%q is a verb of the %s seat's table that this binary has no command line for", verb, catalogue.ControllerSeatName) diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index 3b88d5c5..b41492dc 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -120,21 +120,24 @@ func TestTheRemovedVerbsAreRefusedByName(t *testing.T) { for _, c := range []struct { verb string args map[string]any - want string + said string }{ - {"build", map[string]any{"repository": "novox/x", "path": "modules/x"}, "build"}, - {"build", map[string]any{"on": "mesh-tools"}, "build"}, - {"rebuild", map[string]any{"what": "gitea"}, "rebuild"}, - {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, "replay build-1 --register"}, + {"build", map[string]any{"repository": "novox/x", "path": "modules/x"}, "`build` is removed"}, + {"build", map[string]any{}, "`build` is removed"}, + {"rebuild", map[string]any{"what": "gitea"}, "`rebuild` is removed"}, + {"replay", map[string]any{"id": "build-1", "register": "true", "older": "true"}, "`replay --register` is removed"}, // 1318: delivery-plan and forget-checks go too. - {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x"}, "delivery plan"}, - {"forget-checks", map[string]any{"owner": "novox", "repo": "a", "number": "7"}, "forget-checks"}, + {"delivery-plan", map[string]any{"repository": "novox/a", "paths": "x"}, "`delivery-plan` is removed"}, + {"forget-checks", map[string]any{"owner": "novox", "repo": "a", "number": "7"}, "`forget-checks` is removed"}, } { argv, err := argvFor(c.verb, c.args) - if err != nil || strings.Join(argv, " ") != c.want { - t.Errorf("%s %v: %v %v, want %q", c.verb, c.args, argv, err, c.want) + if err == nil || !strings.Contains(err.Error(), c.said) { + t.Errorf("%s %v: %v %v, want refused saying %q", c.verb, c.args, argv, err, c.said) } } + if argv, err := argvFor("replay", map[string]any{"id": "build-1"}); err != nil || strings.Join(argv, " ") != "replay build-1" { + t.Errorf("replay as a dry run: %v %v", argv, err) + } for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery-plan"} { err := removedByName(removed) if err == nil || !strings.Contains(err.Error(), "`"+removed+"` is removed") || !strings.Contains(err.Error(), "ADR 05") { From ab33f015ecf294982855a3d4c68742d9ee71b970 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:37:31 +0200 Subject: [PATCH 24/38] Say again what the controller keeps at every push, so a word refused before its grant is kept once a push carries it (issue 1790, D1) --- cmd/mesh-controller/push.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 510f650a..2769c902 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -1490,6 +1490,13 @@ func issueMemberships(ctx context.Context, open *stores, server *link.Server, se if reheard > 0 { fmt.Printf(" issued %d membership(s) again for what they hear of a module's stream\n", reheard) } + // **And says again what it keeps** (novox/hq ADR 0495 clause 2, issue 1790): its word at start is refused until a push + // carries its grant; kept only when it changed, so saying it at every push changes nothing else. + if js := servingBus.Load(); js != nil { + if err := announceOwnStates(ctx, js); err != nil { + fmt.Printf(" states: %v\n", err) + } + } if len(refused) > 0 { // Returned, never passed over (novox/hq issue 249): the declarations of the machines they are // for are not sent, and the rollout that asked is tried again rather than waiting for a push. From dfc2f2aa98123cf8281e88a88cab7738399694e7 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 18:38:44 +0200 Subject: [PATCH 25/38] Take a walk's key in again when it was recorded at an earlier merge, so the module is registered at the commit the walk carries before it is built (issue 1851, A6) --- cmd/mesh-controller/plan_keys.go | 13 ++++++------ cmd/mesh-controller/plan_keys_test.go | 30 +++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go index 16cecbf1..f3fe8d35 100644 --- a/cmd/mesh-controller/plan_keys.go +++ b/cmd/mesh-controller/plan_keys.go @@ -160,13 +160,13 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven keyTakes.Unlock() registered := taken if !registered { - if registered, err = registeredFrom(ctx, inv, m, b); err != nil { + if registered, err = registeredAt(ctx, inv, m, trunk.Commit); err != nil { return changed, err } } if registered { at := b.At - s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, b.Commit, id + s.State, s.BuiltAt, s.Commit, s.Build = "built", &at, trunk.Commit, id s.Made = fmt.Sprintf("taken from the builder's key %s (revision %d), no build asked", short(trunk.FP2), k.Revision) s.Why = "" fmt.Printf("%s: %s taken from the builder's key %s\n", p.ID, m, short(trunk.FP2)) @@ -192,9 +192,10 @@ func settleKeys(ctx context.Context, inv keyRecords, keys planKeyReads, p *inven return changed, nil } -// registeredFrom says the module's registration stands on the build: the catalogue holds the module at the build's -// commit, or the build was registered as one with an unchanged source (no move). -func registeredFrom(ctx context.Context, inv keyRecords, module string, b inventory.Build) (bool, error) { +// registeredAt says the catalogue holds the module registered at the commit the walk takes it at: its key's take-in +// registered it there, whether with the key's artifacts or, unchanged, with the build it stands on (no move). A key +// recorded at an earlier commit (the same fingerprint at an older merge) is taken in again, to register it here. +func registeredAt(ctx context.Context, inv keyRecords, module, commit string) (bool, error) { src, err := inv.SourceOf(ctx, module) if errors.Is(err, inventory.ErrNoSuchModule) { return false, nil @@ -202,7 +203,7 @@ func registeredFrom(ctx context.Context, inv keyRecords, module string, b invent if err != nil { return false, err } - return sameCommitID(src.BuiltFrom, b.Commit), nil + return sameCommitID(src.BuiltFrom, commit), nil } func sameCommitID(a, b string) bool { diff --git a/cmd/mesh-controller/plan_keys_test.go b/cmd/mesh-controller/plan_keys_test.go index e11da201..29190f32 100644 --- a/cmd/mesh-controller/plan_keys_test.go +++ b/cmd/mesh-controller/plan_keys_test.go @@ -188,3 +188,33 @@ func TestARefusedTakeInFailsTheWalk(t *testing.T) { t.Fatalf("a refused take-in did not fail the walk: %s %q", p.State, p.Note) } } + +// A6: a walk's module whose fingerprint did not change since an earlier merge has its key recorded at that merge's +// commit: the walk takes it in again, so the module is registered at the commit the walk carries, before it is built. +func TestAKeyRecordedAtAnEarlierCommitIsTakenInAgain(t *testing.T) { + t.Cleanup(func() { + keyTakes.Lock() + keyTakes.running, keyTakes.waits, keyTakes.taken = map[string]bool{}, map[string]string{}, map[string]bool{} + keyTakes.Unlock() + }) + p, byName := aWalkOfOne("sudo") + askModule(t.Context(), p, "sudo", byName) + fp := "src2:" + strings.Repeat("e", 64) + k := builderkeys.Key{Module: "sudo", Fingerprint: fp, Found: true, Revision: 1, Build: sdkdelivery.BuilderBuild{ + State: sdkdelivery.BuildDone, Manifest: strings.Repeat("b", 64), Artifacts: []sdkdelivery.BuiltArtifact{{Name: "a", + Digest: "sha256:1", Reference: "artifact-store://sudo/a/blobs/sha256:1"}}}} + keys := planKeysOf{values: map[string]sdkdelivery.Module{"sudo@c0ffee0123456789": {FP2: fp, Commit: "c0ffee0123456789"}}, + keys: map[string]builderkeys.Key{"sudo": k}} + id := keyBuildID(k) + records := recordsOf{builds: map[string]inventory.Build{id: {ID: id, Module: "sudo", Commit: "0ld0ld0ld0ld"}}, + sources: map[string]inventory.Source{"sudo": {BuiltFrom: "0ld0ld0ld0ld"}}} + var taken []string + if _, err := settleKeys(t.Context(), records, keys, p, p.Tiers[0], func(m string, _ sdkdelivery.Module, _ string) { + taken = append(taken, m) + }); err != nil { + t.Fatal(err) + } + if s := p.Modules["sudo"]; s.State != "asked" || len(taken) != 1 { + t.Fatalf("a key recorded at an earlier commit settled the walk at that commit: %+v, taken %v", s, taken) + } +} From a10ef8995b7079a8adb298506dd3b455d7fd3451 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 19:05:21 +0200 Subject: [PATCH 26/38] Vendor what go.mod pins after the rebase on main (issue 1790; the SDK's 1833 fix returns with E's re-pin) --- .../novox/mesh-sdk/go/compose/network.go | 23 +++---------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/compose/network.go b/vendor/git.novox.be/novox/mesh-sdk/go/compose/network.go index afc8363c..1e66c516 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/compose/network.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/compose/network.go @@ -74,10 +74,7 @@ func (c *composer) overlayRange() string { } // network builds the private network over the machines that resolved the module for it. A machine is on it because it -// was given the module, or because it is joining with a key and an address (novox/hq ADR 0169), or because its own set -// could not be resolved and it holds a place (a key and an address): that place is its last good one on the network, and -// it does not change because its modules failed, so every other machine keeps its path to it -- the hub above all -- and -// only its own declaration is refused (issue 1833). A machine refused with no place is left off. +// was given the module, or because it is joining with a key and an address (novox/hq ADR 0169). func (c *composer) network(on map[string]bool, refused map[string]string) (*overlay.Generator, error) { tunnels := c.in.Tunnels carried := c.in.CarriedPeers @@ -88,15 +85,9 @@ func (c *composer) network(on map[string]bool, refused map[string]string) (*over var nodes []overlay.Node for _, node := range c.in.sorted() { p := node.Place - placed := p.Key != "" && p.Address != "" - _, wasRefused := refused[node.Name] - if !on[node.Name] && !((isJoining[node.Name] || wasRefused) && placed) { + if !on[node.Name] && !(isJoining[node.Name] && p.Key != "" && p.Address != "") { continue } - if !on[node.Name] && wasRefused && !isJoining[node.Name] { - c.say("%s could not be resolved and keeps its place on the private network at %s, so the others keep their path to it", - node.Name, p.Address) - } n := overlay.Node{ Name: node.Name, Key: p.Key, Endpoint: p.Endpoint, Site: p.Site, Hub: p.Hub, Address: p.Address, @@ -242,9 +233,7 @@ func (c *composer) whereEveryoneIs() (map[string]string, error) { } // onTheNetwork is every placed machine that resolves the private network: has an address AND runs what puts it there -// (novox/hq issue 079). A machine that does not resolve keeps its place when it holds one (a key and an address), by the -// same rule as network(), so its name and its address stay where the tunnel to it does (issue 1833); one without is not -// counted, and said. +// (novox/hq issue 079). A machine that does not resolve is not counted, and said. func (c *composer) onTheNetwork() []Node { var out []Node for _, n := range c.in.sorted() { @@ -258,12 +247,6 @@ func (c *composer) onTheNetwork() []Node { catalogue.Node{Name: n.Name, Site: n.Place.Site, Capabilities: n.Capabilities}, catalogue.World{Unchecked: true, Holdings: c.in.Holdings}) if err != nil { - if n.Place.Key != "" { - c.say("%s could not be resolved and keeps its place on the private network at %s, so the others keep their path to it", - n.Name, n.Place.Address) - out = append(out, n) - continue - } c.say("%s is not counted as on the network: it does not resolve: %v", n.Name, err) continue } From 7c0e1d223ff0b3869f2fe997937ef6294e191323 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 19:38:02 +0200 Subject: [PATCH 27/38] Serve a removed verb as its refusal, not as a verb this build is behind on, and fit the tests of the verbs and the stopped checks (issue 1851, A11a; issue 1299, C1; issue 1318, C3) --- .../checks_stop_integration_test.go | 6 +++ cmd/mesh-controller/main.go | 8 +++- cmd/mesh-controller/pending.go | 2 +- cmd/mesh-controller/replay867_test.go | 41 ----------------- cmd/mesh-controller/seatverbs.go | 3 +- .../seatverbs_schema_integration_test.go | 6 +-- cmd/mesh-controller/seatverbs_schema_test.go | 7 +-- internal/inventory/seats_test.go | 45 ------------------- 8 files changed, 21 insertions(+), 97 deletions(-) diff --git a/cmd/mesh-controller/checks_stop_integration_test.go b/cmd/mesh-controller/checks_stop_integration_test.go index d6f32d1d..0ed5b21a 100644 --- a/cmd/mesh-controller/checks_stop_integration_test.go +++ b/cmd/mesh-controller/checks_stop_integration_test.go @@ -33,6 +33,12 @@ func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { t.Fatal(err) } + // hq is cloned as a module built from it is, from its URL: no forge is held in this test. + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "hq-docs", Version: "1"}, + inventory.Source{Repository: "http://forge.invalid/novox/hq.git", Ref: "main", Path: "docs", BuiltFrom: "c0", + Head: "c0"}); err != nil { + t.Fatal(err) + } required := map[string][]string{"novox/mesh-controller": {"mesh/check"}, "novox/hq": {"mesh/repo-check"}} was := requiredStatusesOf requiredStatusesOf = func(_ context.Context, owner, repo, _ string) ([]string, error) { diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 0b506f2a..1bc06227 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -495,5 +495,11 @@ func removedByName(command string) error { "forget-checks": "build-agent makes no check build from R4e (ADR 0503 §3), so there is nothing to let go of", "delivery-plan": "a delivery's plan is read with `delivery-order` and the checker's verdict (ADR 0503 §3)", }[command] - return fmt.Errorf("`%s` is removed: %s", command, replacement) + return &removedCommand{msg: fmt.Sprintf("`%s` is removed: %s", command, replacement)} } + +// removedCommand is a command or verb the controller no longer has: a refusal that is the verb's whole answer, not a +// verb of a newer build this one cannot run. +type removedCommand struct{ msg string } + +func (r *removedCommand) Error() string { return r.msg } diff --git a/cmd/mesh-controller/pending.go b/cmd/mesh-controller/pending.go index 724a6b1a..0a48d957 100644 --- a/cmd/mesh-controller/pending.go +++ b/cmd/mesh-controller/pending.go @@ -241,7 +241,7 @@ func notRegistered(ctx context.Context, open *stores, node, module string, opts "at the controller's terminal, and the indexer holds no %s", inventory.ErrNoSuchModule, where.said, module) } answer := strings.TrimPrefix(where.said, inventory.ErrNoSuchModule.Error()) + - "\n a module in a repository is registered from the builder's done key once a merge adds it, then assigned" + "\n a module in a repository is registered from the builder's done key of it, then assigned" if opts.Build { answer += "\n build \"true\" asks for nothing: the controller asks no build (ADR 0501 §5), and the indexer " + "holds no module of that name" diff --git a/cmd/mesh-controller/replay867_test.go b/cmd/mesh-controller/replay867_test.go index a8316cee..1918e261 100644 --- a/cmd/mesh-controller/replay867_test.go +++ b/cmd/mesh-controller/replay867_test.go @@ -323,44 +323,3 @@ exit 1 t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) t.Setenv("GOTOOLCHAIN", "local") } - -// A recheck runs as a command of its own (`delivery check` with one member), where the controller is not the serving -// one and holds no connection: it dials the bus for the moment and asks mesh-delivery's `groups` as the daemon does. -// This drives the verb on a test bus with a fake mesh-delivery answering `groups`, with no stub of the seam between. -func TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn(t *testing.T) { - realAsk := askDeliveryOwner - r := theGroupOfIssue867(t) - askDeliveryOwner = realAsk - js, _ := aBuildQueue(t) - if servingBus.Load() != nil { - t.Fatal("this test is of a process that is not the serving controller") - } - asked := 0 - wrapped, _ := json.Marshal(map[string]any{"content": []map[string]any{{"type": "text", "text": r.groups}}}) - sub, err := js.Conn().Subscribe(link.SeatToolSubject(catalogue.DeliverySeat, "groups"), func(m *nats.Msg) { - asked++ - body, _ := json.Marshal(link.Answer{Result: wrapped}) - _ = m.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - member := `[{"id":"novox/mesh-host@` + r.hostHead[:12] + `","repository":"novox/mesh-host","base":"main","head":"` + - r.hostHead + `","number":114,"paths":["notes.txt"],"module_dirs_said":true}]` - if err := deliveryCommand(t.Context(), []string{"check", "--members", member}); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine) - if err != nil || len(q.Asks) == 0 { - t.Fatalf("nothing was asked of the build seat: %v", err) - } - c := q.Asks[len(q.Asks)-1].Request.Check - if c.SiblingsUnknown != "" || len(c.Siblings) != 1 || c.Siblings[0].Repository != "novox/mesh-controller" || - c.Siblings[0].Number != 346 || c.Siblings[0].Head != r.controllerHead { - t.Fatalf("the recheck learned siblings %+v, unknown %q", c.Siblings, c.SiblingsUnknown) - } - if asked != 1 { - t.Fatalf("mesh-delivery was asked %d times", asked) - } -} diff --git a/cmd/mesh-controller/seatverbs.go b/cmd/mesh-controller/seatverbs.go index b27187f3..737c89ec 100644 --- a/cmd/mesh-controller/seatverbs.go +++ b/cmd/mesh-controller/seatverbs.go @@ -1325,7 +1325,8 @@ func seatToolHandlers() (map[string]link.ToolHandler, []string, error) { continue } var policy *heldAtTheTerminal - if _, err := argvFor(verb, sampleArguments(v)); err != nil && !errors.As(err, &policy) { + var removed *removedCommand + if _, err := argvFor(verb, sampleArguments(v)); err != nil && !errors.As(err, &policy) && !errors.As(err, &removed) { // **A row ahead of this binary is not a reason to go silent.** // // The row is the store's and a control plane follows it (novox/hq ADR 0154), so a verb diff --git a/cmd/mesh-controller/seatverbs_schema_integration_test.go b/cmd/mesh-controller/seatverbs_schema_integration_test.go index c955119d..10ac2c9f 100644 --- a/cmd/mesh-controller/seatverbs_schema_integration_test.go +++ b/cmd/mesh-controller/seatverbs_schema_integration_test.go @@ -38,7 +38,7 @@ func servedSchemas(t *testing.T) map[string]catalogue.Verb { func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) { for name, v := range servedSchemas(t) { - if inProcess[name] { + if inProcess[name] || removedVerbs[name] { continue } names, switches := declaredArguments(v) @@ -78,7 +78,7 @@ func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { args := sampleArguments(v) args[stranger] = "x" _, err := argvFor(name, args) - if inProcess[name] { + if inProcess[name] || removedVerbs[name] { _, err = readArguments(name, args) } if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) { @@ -105,7 +105,7 @@ func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) { reached := map[string]map[string]bool{} // verb → flag sets its command lines reach served := servedSchemas(t) for name, v := range served { - if inProcess[name] || name == "command" { + if inProcess[name] || removedVerbs[name] || name == "command" { continue } names, switches := declaredArguments(v) diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index a14f1041..0aed1a96 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -177,11 +177,8 @@ var accountedFlags = map[string]map[string]string{ "all": "withheld: the verb naming no machine is --behind, and a push of every machine is said at a " + "shell (novox/hq ADR 0217); `command` reaches it", }, - "build": { - "wait": "set by the verb to 0: the id follows the build (issue 176)", - "self": "set by the verb from the repository's form: a path on the forge, or a URL", - "dry-run": "withheld: a dry run answers only when the build ends, which a call cannot wait for; `command` reaches it", - }, + // Kept only to be refused by name, naming what replaced it (novox/hq ADR 0501 §5, ADR 0429 §18). + "replay": {"register": "withheld: refused by name, a registered replay being removed (ADR 0501 §5)"}, "builds": {"n": "=limit"}, "plans": {"n": "=limit", "what-if": "=repository"}, // The machine's tunnel key, named as the verb's other arguments are (novox/hq ADR 0169). diff --git a/internal/inventory/seats_test.go b/internal/inventory/seats_test.go index c300e97c..fb6cb9c7 100644 --- a/internal/inventory/seats_test.go +++ b/internal/inventory/seats_test.go @@ -230,48 +230,3 @@ func TestASeatsVerbGainsTheArgumentsTheBinaryNames(t *testing.T) { } } } - -// **A stored row's build verb takes on and behind** (novox/hq ADR 0246: arguments added to a verb are -// additive): a row seeded before them described build with a repository it required, so the console -// would refuse both and a call naming neither would be refused for want of a repository. Re-seeding -// gives the row this binary's build. -func TestAStoredRowsBuildVerbTakesOnAndBehind(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - old := `[{"name":"build","description":"Have the build machine build a repository.","input":{"type":"object", - "properties":{"repository":{"type":"string"},"path":{"type":"string"},"ref":{"type":"string"}}, - "required":["repository"]}}]` - if _, err := inv.store.Pool().Exec(ctx, `update seat set serves = $1 where name = $2`, - []byte(old), catalogue.ControllerSeatName); err != nil { - t.Fatal(err) - } - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - seats, err := inv.Seats(ctx) - if err != nil { - t.Fatal(err) - } - var build catalogue.Verb - for _, s := range seats { - if s.Name == catalogue.ControllerSeatName { - for _, v := range s.Serves { - if v.Name == "build" { - build = v - } - } - } - } - props, _ := build.Input["properties"].(map[string]any) - for _, arg := range []string{"repository", "path", "ref", "on", "behind"} { - if _, has := props[arg]; !has { - t.Errorf("build in the row does not take %s: %v", arg, props) - } - } - if req, _ := build.Input["required"].([]any); len(req) != 0 { - t.Errorf("build in the row still requires %v, so on and behind alone are refused", req) - } -} From 420ae3d0e771d390fc217eb9742a74c2dceb9935 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 20:19:57 +0200 Subject: [PATCH 28/38] Test the group check's answer without swapping standard output, and wait for a take-in to forget a new module's wait (issue 1299, C2; issue 1851, A5) --- .../checks_stop_integration_test.go | 47 ------------------- cmd/mesh-controller/delivery.go | 10 +++- cmd/mesh-controller/group_check_test.go | 12 +++++ .../new_module_on_merge_test.go | 7 +-- 4 files changed, 24 insertions(+), 52 deletions(-) create mode 100644 cmd/mesh-controller/group_check_test.go diff --git a/cmd/mesh-controller/checks_stop_integration_test.go b/cmd/mesh-controller/checks_stop_integration_test.go index 0ed5b21a..df8a2e64 100644 --- a/cmd/mesh-controller/checks_stop_integration_test.go +++ b/cmd/mesh-controller/checks_stop_integration_test.go @@ -6,9 +6,6 @@ import ( "context" "encoding/json" "errors" - "io" - "os" - "strings" "testing" "git.novox.be/novox/mesh-sdk/go/catalogue" @@ -101,47 +98,3 @@ func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t t.Fatalf("0503 §5: the verdict said names a gate layer, or no repo-check layer: %+v", c) } } - -// C2 (ADR 0503 §3; issue 1299): a delivery group's composed check asks build-agent nothing and answers that the checker -// judges the group. -func TestTheGroupCheckAsksNothingAndNamesTheChecker(t *testing.T) { - aMesh(t) - ctx := t.Context() - js, _ := aBuildQueue(t) - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(nil) }) - members := `[{"id":"novox/a@111111111111","repository":"novox/a","base":"main","head":"1111111111111111111111111111111111111111","number":1},` + - `{"id":"novox/b@222222222222","repository":"novox/b","base":"main","head":"2222222222222222222222222222222222222222","number":2}]` - out := capturedStdout(t, func() error { - return deliveryCommand(ctx, []string{"check", "--group", "feat/x", "--members", members}) - }) - if !strings.Contains(out, `"judged-by": "mesh-delivery-checker"`) || !strings.Contains(out, `"asked": ""`) { - t.Fatalf("0503 §3: the group check answered %s", out) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 0 { - t.Fatalf("0503 §3: the group check asked %+v", q.Asks) - } -} - -// capturedStdout is what run writes to standard output. -func capturedStdout(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - was := os.Stdout - os.Stdout = w - runErr := run() - os.Stdout = was - w.Close() - out, _ := io.ReadAll(r) - if runErr != nil { - t.Fatal(runErr) - } - return string(out) -} diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 17a9a16c..44d6c0f0 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -901,8 +901,7 @@ func deliveryCommand(ctx context.Context, args []string) error { } // **The group's composed check is the checker's** (novox/hq ADR 0503 §3; issue 1299, C2): nothing is asked of // build-agent, and the answer says who judges it. - return answer(map[string]any{"asked": "", "group": *group, "judged-by": checkerModule, - "said": "the checker judges a delivery group's heads merged together (ADR 0503 §3): nothing was asked"}) + return answer(groupCheckAnswer(*group)) case "go": if len(positionals) != 1 { return errors.New("delivery go [--by ] [--why ]") @@ -1258,3 +1257,10 @@ func recheckOneHead(member string, answer func(any) error, ask func() (string, e } return answer(recheckAnswer(asked, member)) } + +// groupCheckAnswer is what `delivery check` of a group answers from R4e: who judges it, and that nothing was asked +// (novox/hq ADR 0503 §3; issue 1299, C2). +func groupCheckAnswer(group string) map[string]any { + return map[string]any{"asked": "", "group": group, "judged-by": checkerModule, + "said": "the checker judges a delivery group's heads merged together (ADR 0503 §3): nothing was asked"} +} diff --git a/cmd/mesh-controller/group_check_test.go b/cmd/mesh-controller/group_check_test.go new file mode 100644 index 00000000..080f1ba1 --- /dev/null +++ b/cmd/mesh-controller/group_check_test.go @@ -0,0 +1,12 @@ +package main + +import "testing" + +// C2 (ADR 0503 §3; issue 1299): a delivery group's composed check is answered as the checker's, and nothing is asked +// of build-agent: the command answers before any ask is composed. +func TestTheGroupCheckNamesTheCheckerAndAsksNothing(t *testing.T) { + a := groupCheckAnswer("feat/x") + if a["judged-by"] != "mesh-delivery-checker" || a["asked"] != "" || a["group"] != "feat/x" { + t.Fatalf("the group check answered %v", a) + } +} diff --git a/cmd/mesh-controller/new_module_on_merge_test.go b/cmd/mesh-controller/new_module_on_merge_test.go index eb693721..4a763321 100644 --- a/cmd/mesh-controller/new_module_on_merge_test.go +++ b/cmd/mesh-controller/new_module_on_merge_test.go @@ -58,9 +58,10 @@ func TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild(t *testing.T) _, known := inventoryHolds(t, inv, "systemd-resolved") return known }) - if obs := newModuleObservations(time.Now().Add(time.Hour)); len(obs) != 0 { - t.Fatalf("a registered module is still named waiting: %+v", obs) - } + // The take-in forgets the wait just after the registration it made is seen. + becomes(t, 5*time.Second, "the registered module no longer named waiting", func() bool { + return len(newModuleObservations(time.Now().Add(time.Hour))) == 0 + }) requests, err := inv.RequestsNamed(ctx, inventory.BuildRequest{Repository: "novox/mesh-catalog", Path: "modules/systemd-resolved"}.Name()) if err != nil || len(requests) != 0 { From bc80683b9eecd578ff790941927ff4cba50e1ce1 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 20:41:30 +0200 Subject: [PATCH 29/38] Count the work the take-ins start in the background, so a test's mesh closes only once that work has ended (issue 1851, A5-A7; issue 1790, D3) --- cmd/mesh-controller/first_builds.go | 2 +- cmd/mesh-controller/mesh_for_test.go | 2 ++ cmd/mesh-controller/new_modules.go | 19 ++++++++++++++++--- cmd/mesh-controller/plan_keys.go | 2 +- cmd/mesh-controller/states_announced.go | 2 +- 5 files changed, 21 insertions(+), 6 deletions(-) diff --git a/cmd/mesh-controller/first_builds.go b/cmd/mesh-controller/first_builds.go index c6f6322c..b6c06bfc 100644 --- a/cmd/mesh-controller/first_builds.go +++ b/cmd/mesh-controller/first_builds.go @@ -94,7 +94,7 @@ func askFirstBuilds(ctx context.Context, open *stores, m link.SourceMoved, now t fmt.Printf(" the indexer's modules of %s cannot be read now (%v): they are registered when the builder's next "+ "write of one wakes the take-in\n", repository, err) case len(modules) > 0: - go takeNewModules(context.WithoutCancel(ctx), open, modules...) + inBackground(func() { takeNewModules(context.WithoutCancel(ctx), open, modules...) }) } return nil } diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index 43f73c75..4cfe3ff7 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -56,6 +56,8 @@ func aMeshOnTheStore(t *testing.T) *stores { t.Fatal(err) } t.Cleanup(open.Close) + // The work this mesh started in the background ends before its stores close (cleanups run last first). + t.Cleanup(background.Wait) // A condition store of its own, held in memory (novox/hq to-be 45 §2): status leads with what is // open, and a mesh with no bus would otherwise read as one whose conditions cannot be read. withConditionsInMemory(t) diff --git a/cmd/mesh-controller/new_modules.go b/cmd/mesh-controller/new_modules.go index 58d88cd3..aa870fe7 100644 --- a/cmd/mesh-controller/new_modules.go +++ b/cmd/mesh-controller/new_modules.go @@ -200,10 +200,10 @@ func newModuleObservations(now time.Time) []conditions.Observation { // watchNewModules wakes the take-in of new modules for each module the builder or the indexer writes, until ctx ends, and // looks at every module once now. A bucket not there yet is said: the look at start and a first merge's word still work. func watchNewModules(ctx context.Context, open *stores) { - go func() { + inBackground(func() { answerHeldMerges(ctx, open.inventory) takeNewModules(ctx, open) - }() + }) keys, closeKeys, err := openBuilderKeys() if err != nil { fmt.Printf("new modules: the builder's keys are not watched: %v\n", err) @@ -223,7 +223,7 @@ func watchNewModules(ctx context.Context, open *stores) { // wakeNewModule takes module in when the catalogue does not hold it: never on the watch's own goroutine, which must keep // draining. func wakeNewModule(ctx context.Context, open *stores, module string) { - go func() { + inBackground(func() { entries, err := open.inventory.Catalogued(ctx) if err != nil { return @@ -236,6 +236,19 @@ func wakeNewModule(ctx context.Context, open *stores, module string) { } } takeNewModules(ctx, open, module) + }) +} + +// background is the work the take-ins of keys and of announcements start beside the event that woke them, so a test +// ends only once the work its mesh started has. +var background sync.WaitGroup + +// inBackground runs work beside the caller, counted in background. +func inBackground(work func()) { + background.Add(1) + go func() { + defer background.Done() + work() }() } diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go index f3fe8d35..c3677e6e 100644 --- a/cmd/mesh-controller/plan_keys.go +++ b/cmd/mesh-controller/plan_keys.go @@ -73,7 +73,7 @@ var keyTakes = struct { // the buckets and the records; a done key not recorded yet is taken in off the plans' hold, and its end asks the look. func settleFromKeys(ctx context.Context, open *stores, keys planKeyReads, p *inventory.Plan, tier []string) (bool, error) { return settleKeys(ctx, open.inventory, keys, p, tier, func(module string, trunk sdkdelivery.Module, id string) { - go takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id) + inBackground(func() { takeKeyOffTheHold(context.WithoutCancel(ctx), open, module, trunk, id) }) }) } diff --git a/cmd/mesh-controller/states_announced.go b/cmd/mesh-controller/states_announced.go index 8e4012a1..f4f05611 100644 --- a/cmd/mesh-controller/states_announced.go +++ b/cmd/mesh-controller/states_announced.go @@ -67,7 +67,7 @@ func keepAnnouncements(ctx context.Context, js *broker.JetStream, open *stores) return } if changed { - go composeAnnouncedReads(context.WithoutCancel(ctx), open) + inBackground(func() { composeAnnouncedReads(context.WithoutCancel(ctx), open) }) } }) if err != nil { From d5d53893a4986af96a1ea8f7e1dd3cc481f843df Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 20:51:08 +0200 Subject: [PATCH 30/38] Send the bus its user list before a starting controller writes (issue 1800) A new controller's grants reached the bus only after its gate, so the buckets it newly writes were refused until then. At start it now sends the bus's machine the user list alone, every build there kept. --- cmd/mesh-controller/own_grants.go | 69 ++++++++++++++++++++++++ cmd/mesh-controller/own_grants_test.go | 73 ++++++++++++++++++++++++++ cmd/mesh-controller/push.go | 4 +- 3 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/own_grants.go create mode 100644 cmd/mesh-controller/own_grants_test.go diff --git a/cmd/mesh-controller/own_grants.go b/cmd/mesh-controller/own_grants.go new file mode 100644 index 00000000..8964a078 --- /dev/null +++ b/cmd/mesh-controller/own_grants.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + "fmt" +) + +// A new controller's own grants reach the bus before it writes (novox/hq issue 1800). +// +// A controller's grants are composed by the controller binary itself (broker.PermissionsFor), so the controller that +// walks a delivery cannot put its successor's grants in the bus's user list: the list that carries them was sent only +// once the new controller passed its gate on its first machine (carryUserList). Until then the new controller wrote +// buckets its account was not granted, and the bus refused them: plans-building on 2026-10-21, declared-packages on +// 2026-10-22, and R4e's states-announced would be the third. +// +// So a controller that starts, holding the lease and before it starts anything that writes its buckets, sends the +// machine holding the bus its user list when the list it composes is not the one that machine was last sent: the grants +// step's send (release.go grantsStep), every build there kept at the one it runs, refused unsent when the declaration +// would change more than the user list (errNotGrantsOnly). No build of a running plan reaches that machine before the +// plan's own tier sends it (ADR 0503 §8). A send that cannot be made is said once, and the controller serves on. +// +// It does not ask heldMachines, as carryUserList does: every reason that gives is a build that would move with the +// list (a policy, a plan not yet sent there, no gate passed), and here none can. The running plan that brought this +// controller holds the bus's machine until its tier sends it; asking would leave the list behind exactly when 1800 +// meets it. The refusal unsent (errNotGrantsOnly) is the guard instead. + +// ownGrantsCarrier is what the start's send reads and does: whether the bus's list is behind and on which machine, and +// the grants-only send to it. Variables so a test needs no store and no bus. +type ownGrantsCarrier struct { + behind func(ctx context.Context) (holder string, behind bool, err error) + send func(ctx context.Context, holder string) error +} + +// carry sends the machine holding the bus the controller's own user list when it is behind, and says what it did. +func (c ownGrantsCarrier) carry(ctx context.Context) string { + holder, behind, err := c.behind(ctx) + switch { + case err != nil: + return fmt.Sprintf("whether the bus's user list carries this controller's grants cannot be read, so it is not sent "+ + "at start; a bucket it newly writes may be refused until the gate carries it: %v", err) + case holder == "" || !behind: + return "" + } + if err := c.send(ctx, holder); err != nil { + return fmt.Sprintf("this controller's grants could not be sent to %s, which holds the bus, at its start; a bucket "+ + "it newly writes may be refused until the gate carries them: %v", holder, err) + } + return fmt.Sprintf("sent %s, which holds the bus, this controller's user list alone, every build there kept, "+ + "before it writes", holder) +} + +// startWithOwnGrants carries the controller's own grants to the bus and only then starts what writes (then). +func startWithOwnGrants(ctx context.Context, c ownGrantsCarrier, then func()) { + if said := c.carry(ctx); said != "" { + fmt.Println(said) + } + then() +} + +// ownGrantsOf is the carrier of the serving controller: the grants step's reading and its grants-only send. +func ownGrantsOf(open *stores) ownGrantsCarrier { + return ownGrantsCarrier{ + behind: func(ctx context.Context) (string, bool, error) { return brokerBehindOf(ctx, open, nil) }, + send: func(ctx context.Context, holder string) error { + _, err := sendRollout(keepingEveryBuild(withScope(ctx, sendScope{}), holder), open, []string{holder}) + return err + }, + } +} diff --git a/cmd/mesh-controller/own_grants_test.go b/cmd/mesh-controller/own_grants_test.go new file mode 100644 index 00000000..d7a8ea4f --- /dev/null +++ b/cmd/mesh-controller/own_grants_test.go @@ -0,0 +1,73 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" +) + +// 1800 (1): a controller starting with the bus's list behind its own sends the bus's machine once, before anything +// that writes its buckets starts. +func TestAStartingControllerSendsItsGrantsBeforeItWrites(t *testing.T) { + var order []string + c := ownGrantsCarrier{ + behind: func(context.Context) (string, bool, error) { return "novox", true, nil }, + send: func(_ context.Context, holder string) error { order = append(order, "send "+holder); return nil }, + } + startWithOwnGrants(t.Context(), c, func() { order = append(order, "writers") }) + if strings.Join(order, ", ") != "send novox, writers" { + t.Fatalf("the start did %v", order) + } +} + +// 1800 (3), (4): no send when the list is current or there is no bus with a list; one that cannot be read or sent is +// said once, and the controller goes on. +func TestAStartingControllerSendsNothingWhenTheListIsCurrentAndGoesOnWhenRefused(t *testing.T) { + sends := 0 + send := func(context.Context, string) error { sends++; return nil } + for _, c := range []ownGrantsCarrier{ + {behind: func(context.Context) (string, bool, error) { return "novox", false, nil }, send: send}, + {behind: func(context.Context) (string, bool, error) { return "", true, nil }, send: send}, + } { + if said := c.carry(t.Context()); said != "" || sends != 0 { + t.Fatalf("a current list sent %d and said %q", sends, said) + } + } + refused := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "novox", true, nil }, + send: func(context.Context, string) error { return errNotGrantsOnly }} + went := false + startWithOwnGrants(t.Context(), refused, func() { went = true }) + if !went { + t.Fatal("a refused send stopped the controller") + } + if said := refused.carry(t.Context()); !strings.Contains(said, "could not be sent to novox") || + !strings.Contains(said, errNotGrantsOnly.Error()) { + t.Fatalf("a refused send is said as %q", said) + } + unread := ownGrantsCarrier{behind: func(context.Context) (string, bool, error) { return "", false, errors.New("no store") }, + send: send} + if said := unread.carry(t.Context()); !strings.Contains(said, "cannot be read") || sends != 0 { + t.Fatalf("an unread list: sent %d, said %q", sends, said) + } +} + +// 1800 (2): the start's send is the grants step's: composed with every build of the bus's machine kept at the one it +// runs, so the declaration differs from its last only by the user list (and is refused unsent otherwise, +// errNotGrantsOnly, release.go), whatever a running plan registered since. +func TestAStartingControllersSendKeepsEveryBuildOfTheBusMachine(t *testing.T) { + was := sendRollout + t.Cleanup(func() { sendRollout = was }) + var kept bool + var named []string + sendRollout = func(ctx context.Context, _ *stores, names []string) ([]string, error) { + kept, named = everyBuildKept(ctx, "novox"), names + return names, nil + } + if err := ownGrantsOf(nil).send(t.Context(), "novox"); err != nil { + t.Fatal(err) + } + if !kept || len(named) != 1 || named[0] != "novox" { + t.Fatalf("the start's send keeps every build: %v, names %v", kept, named) + } +} diff --git a/cmd/mesh-controller/push.go b/cmd/mesh-controller/push.go index 61cfcb9f..41d88d4f 100644 --- a/cmd/mesh-controller/push.go +++ b/cmd/mesh-controller/push.go @@ -189,7 +189,9 @@ func serve(ctx context.Context) (err error) { // Open plans move on events (novox/hq ADR 0406 §5, issue 974): a tier is asked on the `built` and the send of the // tier before it, a walk ends on its machines' reports, a silent machine is a key that expired, and a plan left // by a replaced controller resumes from the first look, which this asks for. No timer looks every so often. - startRolloutEvents(ctx, open, server.JetStream()) + // **Its own grants on the bus before it writes** (novox/hq issue 1800): a bucket this controller newly writes is + // granted before the first write, not after its gate. + startWithOwnGrants(ctx, ownGrantsOf(open), func() { startRolloutEvents(ctx, open, server.JetStream()) }) // And the merge window (novox/hq ADR 0276, ADR 0407): a batch is cut into its walk when the window closes, // which no merge or outcome says. The window is two keys on the bus that expire and a watcher that hears it; // nothing polls (issue 973). From 93fc2f48a16d43d24f08c09fee3a67ece6917f46 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:05:01 +0200 Subject: [PATCH 31/38] Name a retry's modules still waiting for the bus, read as the plan it resumes (issue 1851 A11a, issue 1403) With the hand rebuild gone, a retry of a plan failed past the bus's bound read the waiting modules while the plan still said failed, named none, and answered "asked" with nothing after it. --- cmd/mesh-controller/ask_bus_wait_test.go | 23 +++++++++++++++++++++++ cmd/mesh-controller/plan_retry.go | 10 +++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 cmd/mesh-controller/ask_bus_wait_test.go diff --git a/cmd/mesh-controller/ask_bus_wait_test.go b/cmd/mesh-controller/ask_bus_wait_test.go new file mode 100644 index 00000000..3df88f5a --- /dev/null +++ b/cmd/mesh-controller/ask_bus_wait_test.go @@ -0,0 +1,23 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A retry of a plan failed past the bus's bound, whose module the bus still does not answer, names that module as waiting: +// read while the plan still says failed, it was named nowhere, and the retry said "asked" with nothing after it (issue +// 1403, met again once the hand rebuild no longer resumed the plan first). +func TestARetryNamesTheModulesStillWaitingForTheBus(t *testing.T) { + since := time.Now().UTC() + p := inventory.Plan{ID: "plan-bus", State: inventory.PlanFailed, Tiers: [][]string{{"a"}}, + Modules: map[string]*inventory.PlanModule{"a": {BusWaitFrom: &since}}} + waiting := busWaitingOnRetry(p) + _, said := retryWords(p, []string{"a"}, nil, waiting) + if !strings.Contains(said, "a not asked yet: the bus did not answer") || strings.Contains(said, "asked ;") { + t.Fatalf("a retry left for the bus is said as %q", said) + } +} diff --git a/cmd/mesh-controller/plan_retry.go b/cmd/mesh-controller/plan_retry.go index 6cca81db..5db2d366 100644 --- a/cmd/mesh-controller/plan_retry.go +++ b/cmd/mesh-controller/plan_retry.go @@ -408,7 +408,7 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { asked = append(asked, m+" as "+s.Build) } } - waiting := busWaitingIn(p) + waiting := busWaitingOnRetry(p) note, said := retryWords(p, failed, asked, waiting) resumed(&p, note) if err := inv.SavePlan(ctx, &p); err != nil { @@ -420,6 +420,14 @@ func retryPlan(ctx context.Context, open *stores, id string) (string, error) { return said, nil } +// busWaitingOnRetry are the modules a retry left waiting for the bus. Read before the plan is resumed, while it still says +// failed, so read as the plan the retry resumes: a failed plan is not open, and busWaitingIn would name none, leaving the +// answer "asked" with nothing after it (issue 1403). +func busWaitingOnRetry(p inventory.Plan) []string { + p.State = inventory.PlanBuilding + return busWaitingIn(p) +} + // retryWords are the plan's note and the verb's answer for a retry: what was asked again (as the builds it got), and what waits // for the bus. A retry whose modules all wait for the bus asked none, and says so rather than "asked" with nothing after it // (issue 1403). From 9713ceb84d0aa8711335716afad35954723ac92e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:06:33 +0200 Subject: [PATCH 32/38] Ask build-agent for itself and its toolchain images, never a builder key (ADR 0520, issue 1851) The builder writes no key of build-agent or of the toolchain bases its builds stand on, so a walk moving one would hold its tier for ever. Until R4g they are build-agent's own, as the builder's module is; the bases are read from the toolchains, not copied. --- cmd/mesh-controller/builders_own_test.go | 45 ++++++++++++++++++++++++ cmd/mesh-controller/plan_keys.go | 23 +++++++++--- internal/builder/toolchain.go | 18 ++++++++++ 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/cmd/mesh-controller/builders_own_test.go b/cmd/mesh-controller/builders_own_test.go index 3c2d717b..05b7589c 100644 --- a/cmd/mesh-controller/builders_own_test.go +++ b/cmd/mesh-controller/builders_own_test.go @@ -59,3 +59,48 @@ func TestAWalkAsksBuildAgentForTheBuildersOwnModuleAlone(t *testing.T) { t.Fatalf("the other module is not taken from its key: %+v", s) } } + +// ADR 0520 (issue 1851): until R4g build-agent and the toolchain images its builds stand on are build-agent's own, as the +// builder's module is: the builder writes no key of them, so a walk asks build-agent for them and reads no key; any other +// module of the same tier is still taken from its key. +func TestAWalkAsksBuildAgentForItselfAndItsToolchainsAndReadsNoKeyOfThem(t *testing.T) { + own := []string{"build-agent", "mesh-tools", "mesh-tools-go", "mesh-tools-python"} + wasModule := askedModuleOf + askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) { + return strings.TrimPrefix(path, "modules/"), nil + } + t.Cleanup(func() { askedModuleOf = wasModule }) + for _, m := range own { + if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x", Seat: "git"}, "modules/"+m); err != nil { + t.Fatalf("an ask of %s is refused: %v", m, err) + } + } + var asked []string + was := askABuild + askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) { + asked = append(asked, path) + return "build-" + strings.TrimPrefix(path, "modules/"), nil + } + t.Cleanup(func() { askABuild = was }) + p, byName := aWalkOfOne("sudo") + p.Tiers = [][]string{append(append([]string{}, own...), "sudo")} + for _, m := range own { + e := byName["sudo"] + e.Source.Path = "modules/" + m + byName[m] = e + } + for _, m := range p.Tiers[0] { + askModule(t.Context(), p, m, byName) + } + if strings.Join(asked, " ") != "modules/build-agent modules/mesh-tools modules/mesh-tools-go modules/mesh-tools-python" { + t.Fatalf("build-agent was asked for %v", asked) + } + for _, m := range own { + if s := p.Modules[m]; s.Key || s.Build != "build-"+m || s.State != "asked" { + t.Fatalf("%s is not asked of build-agent, or is read from a key: %+v", m, s) + } + } + if s := p.Modules["sudo"]; !s.Key || s.Build != "" { + t.Fatalf("the other module is not taken from its key: %+v", s) + } +} diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go index c3677e6e..cf8f6a30 100644 --- a/cmd/mesh-controller/plan_keys.go +++ b/cmd/mesh-controller/plan_keys.go @@ -8,6 +8,8 @@ import ( "time" sdkdelivery "git.novox.be/novox/mesh-sdk/go/delivery" + + "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/builderkeys" "github.com/novox/mesh-controller/internal/inventory" ) @@ -22,13 +24,24 @@ import ( // yet, or waiting for a reference, holds the tier and says so; a failed key fails the walk in the builder's own words. // **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules). -// isBuildersOwn says a module is the builder's own: the one module build-agent still builds, asked on a merge, until R4g -// (ADR 0453 §5, ADR 0498 §7). +// isBuildersOwn says a module is build-agent's own: asked of build-agent on a merge and taken from its build record, never +// from the builder's key, until R4g (ADR 0453 §5, ADR 0498 §7, ADR 0520). They are the builder's module, build-agent itself +// (its image's context is another repository's) and the toolchain images build-agent's builds stand on +// (builder.Bases): the builder writes no key of any of them, and a walk waiting for one would hold its tier for ever. func isBuildersOwn(module string) bool { return buildersOwn(module) } -// buildersOwn is isBuildersOwn's rule: a variable only so a test of the one ask left (its bus waits, its plan key) may -// name modules of its own as the builder's. -var buildersOwn = func(module string) bool { return module == sdkdelivery.BuilderModule } +// buildAgentsOwn are those modules: the builder's, build-agent's and every toolchain's base, read from the toolchains. +var buildAgentsOwn = func() map[string]bool { + own := map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true} + for _, b := range builder.Bases() { + own[b] = true + } + return own +}() + +// buildersOwn is isBuildersOwn's rule: a variable only so a test of the asks left (their bus waits, their plan key) may +// name modules of its own as build-agent's. +var buildersOwn = func(module string) bool { return buildAgentsOwn[module] } // planKeyReads is what a walk reads of the indexer and the builder (builderkeys.Reader, or a test's). type planKeyReads interface { diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index f5a304da..1cb0a381 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -234,6 +234,24 @@ func spoken() string { } // Languages is every language the mesh can build, for anything that wants to say so. +// AgentModule is build-agent's own module: the machine's builder, whose image is built from another repository's context. +const AgentModule = "build-agent" + +// Bases are the modules whose artifacts provide the toolchains' compilers (Toolchain.Base), each once, sorted: images +// build-agent's builds stand on. +func Bases() []string { + seen := map[string]bool{} + var out []string + for _, t := range toolchains { + if t.Base != "" && !seen[t.Base] { + seen[t.Base] = true + out = append(out, t.Base) + } + } + sort.Strings(out) + return out +} + func Languages() []string { names := make([]string, 0, len(toolchains)) for _, t := range toolchains { From 61f7e89ad545d626ccf32d48818767ae5ab23327 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:07:55 +0200 Subject: [PATCH 33/38] =?UTF-8?q?Ask=20a=20repository=20check=20whenever?= =?UTF-8?q?=20the=20trunk=20requires=20it,=20beside=20mesh/check=20(issue?= =?UTF-8?q?=201299=20C1,=20ADR=200513=20=C2=A74)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A trunk requiring both contexts (hq) matched mesh/check first and had its own merge-check.sh never asked. --- cmd/mesh-controller/checks.go | 33 ++++++++++++------- .../checks_stop_integration_test.go | 9 +++++ cmd/mesh-controller/checks_test.go | 18 ++++++++++ 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index 1cca29e7..bbd6edc3 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -163,9 +163,9 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri } scope := pullScope(p, entries, read, edges) // **The merge gate's checks stop** (novox/hq ADR 0503 §3, §5; issue 1299): the checker judges every repository whose - // trunk requires `mesh/check`, and the controller asks nothing of it and says nothing. Only a repository whose trunk - // requires `mesh/repo-check` alone (hq) has its own merge-check.sh asked of build-agent, and said with that layer - // alone; what the forge's protection requires is read, never listed (lead decision L11). + // trunk requires `mesh/check`. A repository whose trunk requires `mesh/repo-check` (hq), whatever else it requires + // (ADR 0513 §4), has its own merge-check.sh asked of build-agent, and said with that layer alone; any other is asked + // nothing and said nothing. What the forge's protection requires is read, never listed (lead decision L11). // Mesh-config is still told what the head declares (ADR 0502), whatever judges it. this := pullAtOf(p) if this.Truncated { @@ -180,20 +180,31 @@ func (f following) pullUpdated(ctx context.Context, p link.PullUpdated, w io.Wri fmt.Fprintf(w, "%s/%s#%d (%.8s): what %s's protection requires cannot be read, so nothing is asked and nothing is "+ "said: %v\n", p.Owner, p.Repo, p.Number, p.Commit, p.Base, err) return "", nil - case slices.Contains(required, sdkdelivery.StatusContextCheck): - fmt.Fprintf(w, "%s/%s#%d (%.8s): %s requires %s: the checker judges it; nothing is asked\n", p.Owner, p.Repo, - p.Number, p.Commit, p.Base, sdkdelivery.StatusContextCheck) - return "", nil - case !slices.Contains(required, sdkdelivery.StatusContextRepoCheck): - fmt.Fprintf(w, "%s/%s#%d (%.8s): %s requires neither %s nor %s: nothing is asked and nothing is said\n", p.Owner, - p.Repo, p.Number, p.Commit, p.Base, sdkdelivery.StatusContextCheck, sdkdelivery.StatusContextRepoCheck) + case !asksRepoCheck(required): + fmt.Fprintf(w, "%s/%s#%d (%.8s): %s does not require %s, and %s; nothing is asked\n", p.Owner, p.Repo, p.Number, + p.Commit, p.Base, sdkdelivery.StatusContextRepoCheck, checkerWords(required)) return "", nil } return f.repoCheckAlone(ctx, w, p, scope, entries) } +// asksRepoCheck says what a trunk's protection has the controller ask: its repository's own merge-check.sh whenever the +// trunk requires `mesh/repo-check`, whatever else it requires (ADR 0513 §4: a trunk requiring both has the checker judge +// `mesh/check` and the repository check asked beside it), and nothing otherwise. +func asksRepoCheck(required []string) bool { + return slices.Contains(required, sdkdelivery.StatusContextRepoCheck) +} + +// checkerWords says who judges a trunk that requires no `mesh/repo-check`. +func checkerWords(required []string) string { + if slices.Contains(required, sdkdelivery.StatusContextCheck) { + return "the checker judges its " + sdkdelivery.StatusContextCheck + } + return "does not require " + sdkdelivery.StatusContextCheck + " either, so nothing is said" +} + // repoCheckAlone asks build-agent for a repository's own merge-check.sh alone, for a pull request of a repository whose -// trunk requires `mesh/repo-check` and not `mesh/check` (ADR 0503 §5): no gate, no module built, its verdict said with the +// trunk requires `mesh/repo-check`, whether or not it also requires `mesh/check` (ADR 0503 §5, ADR 0513 §4): no gate, no module built, its verdict said with the // repo-check layer alone. func (f following) repoCheckAlone(ctx context.Context, w io.Writer, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) { diff --git a/cmd/mesh-controller/checks_stop_integration_test.go b/cmd/mesh-controller/checks_stop_integration_test.go index df8a2e64..d0496f3a 100644 --- a/cmd/mesh-controller/checks_stop_integration_test.go +++ b/cmd/mesh-controller/checks_stop_integration_test.go @@ -97,4 +97,13 @@ func TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge(t if c.Gate != nil || c.RepoCheck == nil || c.RepoCheck.Verdict != "warning" { t.Fatalf("0503 §5: the verdict said names a gate layer, or no repo-check layer: %+v", c) } + // A trunk requiring both (hq, ADR 0513 §4): the checker judges mesh/check, and the repository check is still asked. + required["novox/hq"] = []string{"mesh/check", "mesh/repo-check"} + if err := f.PullUpdated(ctx, pull("hq", 11, true)); err != nil { + t.Fatal(err) + } + if q, err := link.ReadQueue(ctx, js, link.TheBuildMachine); err != nil || len(q.Asks) != 2 || + q.Asks[1].Request.Check == nil || q.Asks[1].Request.Check.Repo != "hq" { + t.Fatalf("0513 §4: a trunk requiring both had no repository check asked: %+v %v", q.Asks, err) + } } diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index d7e821eb..f804eb06 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -316,3 +316,21 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) { // A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the // live head, and the newer request is the one that judges. A check that failed for any other reason still says its error. + +// C2 (ADR 0513 §4, ADR 0503 §5; issue 1299): a trunk requiring mesh/repo-check has its repository check asked, whether or +// not it also requires mesh/check (hq requires both); a trunk requiring mesh/check alone, or neither, is asked nothing. +func TestARepositoryCheckIsAskedWheneverTheTrunkRequiresIt(t *testing.T) { + for _, c := range []struct { + required []string + asks bool + }{ + {[]string{"mesh/check", "mesh/repo-check"}, true}, + {[]string{"mesh/repo-check"}, true}, + {[]string{"mesh/check"}, false}, + {nil, false}, + } { + if got := asksRepoCheck(c.required); got != c.asks { + t.Fatalf("a trunk requiring %v asks its repository check: %v, want %v", c.required, got, c.asks) + } + } +} From 61449249a8fc9d6c08eb2fd850bb59a2c7401d86 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:08:47 +0200 Subject: [PATCH 34/38] =?UTF-8?q?Name=20build-agent's=20own=20as=20the=20b?= =?UTF-8?q?uilder's=20module,=20build-agent=20and=20mesh-tools=20(ADR=2005?= =?UTF-8?q?20=20=C2=A71,=20issue=201851)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ADR 0520 narrowed after its review: mesh-tools-go builds from its own directory on a pinned base and has its key from mesh-build-image, and mesh-tools-python is no module, so the set is named, not derived from the toolchain bases. --- cmd/mesh-controller/builders_own_test.go | 26 +++++++++++++++--------- cmd/mesh-controller/plan_keys.go | 17 ++++++---------- internal/builder/toolchain.go | 15 -------------- 3 files changed, 22 insertions(+), 36 deletions(-) diff --git a/cmd/mesh-controller/builders_own_test.go b/cmd/mesh-controller/builders_own_test.go index 05b7589c..55aabaa9 100644 --- a/cmd/mesh-controller/builders_own_test.go +++ b/cmd/mesh-controller/builders_own_test.go @@ -60,11 +60,12 @@ func TestAWalkAsksBuildAgentForTheBuildersOwnModuleAlone(t *testing.T) { } } -// ADR 0520 (issue 1851): until R4g build-agent and the toolchain images its builds stand on are build-agent's own, as the -// builder's module is: the builder writes no key of them, so a walk asks build-agent for them and reads no key; any other -// module of the same tier is still taken from its key. -func TestAWalkAsksBuildAgentForItselfAndItsToolchainsAndReadsNoKeyOfThem(t *testing.T) { - own := []string{"build-agent", "mesh-tools", "mesh-tools-go", "mesh-tools-python"} +// ADR 0520 §1 (issue 1851): until R4g build-agent and mesh-tools, the toolchain image its builds stand on, are build-agent's +// own, as the builder's module is: the builder writes no key of them, so a walk asks build-agent for them and reads no key; +// mesh-tools-go (its key from mesh-build-image) and any other module of the same tier are still taken from their keys. +func TestAWalkAsksBuildAgentForItselfAndMeshToolsAndReadsNoKeyOfThem(t *testing.T) { + own := []string{"build-agent", "mesh-tools"} + keyed := []string{"mesh-tools-go", "sudo"} wasModule := askedModuleOf askedModuleOf = func(_ context.Context, _ buildSource, path string) (string, error) { return strings.TrimPrefix(path, "modules/"), nil @@ -75,6 +76,9 @@ func TestAWalkAsksBuildAgentForItselfAndItsToolchainsAndReadsNoKeyOfThem(t *test t.Fatalf("an ask of %s is refused: %v", m, err) } } + if err := onlyTheBuildersOwn(t.Context(), buildSource{Repository: "novox/x", Seat: "git"}, "modules/mesh-tools-go"); !errors.Is(err, errNotTheBuildersOwn) { + t.Fatalf("an ask of mesh-tools-go went on: %v", err) + } var asked []string was := askABuild askABuild = func(_ context.Context, _ buildSource, path, _ string) (string, error) { @@ -83,8 +87,8 @@ func TestAWalkAsksBuildAgentForItselfAndItsToolchainsAndReadsNoKeyOfThem(t *test } t.Cleanup(func() { askABuild = was }) p, byName := aWalkOfOne("sudo") - p.Tiers = [][]string{append(append([]string{}, own...), "sudo")} - for _, m := range own { + p.Tiers = [][]string{append(append([]string{}, own...), keyed...)} + for _, m := range append(append([]string{}, own...), keyed...) { e := byName["sudo"] e.Source.Path = "modules/" + m byName[m] = e @@ -92,7 +96,7 @@ func TestAWalkAsksBuildAgentForItselfAndItsToolchainsAndReadsNoKeyOfThem(t *test for _, m := range p.Tiers[0] { askModule(t.Context(), p, m, byName) } - if strings.Join(asked, " ") != "modules/build-agent modules/mesh-tools modules/mesh-tools-go modules/mesh-tools-python" { + if strings.Join(asked, " ") != "modules/build-agent modules/mesh-tools" { t.Fatalf("build-agent was asked for %v", asked) } for _, m := range own { @@ -100,7 +104,9 @@ func TestAWalkAsksBuildAgentForItselfAndItsToolchainsAndReadsNoKeyOfThem(t *test t.Fatalf("%s is not asked of build-agent, or is read from a key: %+v", m, s) } } - if s := p.Modules["sudo"]; !s.Key || s.Build != "" { - t.Fatalf("the other module is not taken from its key: %+v", s) + for _, m := range keyed { + if s := p.Modules[m]; !s.Key || s.Build != "" { + t.Fatalf("%s is not taken from its key: %+v", m, s) + } } } diff --git a/cmd/mesh-controller/plan_keys.go b/cmd/mesh-controller/plan_keys.go index cf8f6a30..453bcbc2 100644 --- a/cmd/mesh-controller/plan_keys.go +++ b/cmd/mesh-controller/plan_keys.go @@ -25,19 +25,14 @@ import ( // **No timer**: the builder's and the indexer's writes wake the look (builderkeys.Watch, watchNewModules). // isBuildersOwn says a module is build-agent's own: asked of build-agent on a merge and taken from its build record, never -// from the builder's key, until R4g (ADR 0453 §5, ADR 0498 §7, ADR 0520). They are the builder's module, build-agent itself -// (its image's context is another repository's) and the toolchain images build-agent's builds stand on -// (builder.Bases): the builder writes no key of any of them, and a walk waiting for one would hold its tier for ever. +// from the builder's key, until R4g (ADR 0453 §5, ADR 0498 §7, ADR 0520 §1). func isBuildersOwn(module string) bool { return buildersOwn(module) } -// buildAgentsOwn are those modules: the builder's, build-agent's and every toolchain's base, read from the toolchains. -var buildAgentsOwn = func() map[string]bool { - own := map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true} - for _, b := range builder.Bases() { - own[b] = true - } - return own -}() +// buildAgentsOwn are those modules, named (ADR 0520 §1): the builder's; build-agent itself, whose image's context is another +// repository's; and mesh-tools, the toolchain image build-agent's builds stand on. The builder writes no key of any of them, +// and a walk waiting for one would hold its tier for ever. mesh-tools-go is not among them: it builds from a Dockerfile in +// its own directory on a pinned base, and has its key from mesh-build-image. +var buildAgentsOwn = map[string]bool{sdkdelivery.BuilderModule: true, builder.AgentModule: true, "mesh-tools": true} // buildersOwn is isBuildersOwn's rule: a variable only so a test of the asks left (their bus waits, their plan key) may // name modules of its own as build-agent's. diff --git a/internal/builder/toolchain.go b/internal/builder/toolchain.go index 1cb0a381..2b77567e 100644 --- a/internal/builder/toolchain.go +++ b/internal/builder/toolchain.go @@ -237,21 +237,6 @@ func spoken() string { // AgentModule is build-agent's own module: the machine's builder, whose image is built from another repository's context. const AgentModule = "build-agent" -// Bases are the modules whose artifacts provide the toolchains' compilers (Toolchain.Base), each once, sorted: images -// build-agent's builds stand on. -func Bases() []string { - seen := map[string]bool{} - var out []string - for _, t := range toolchains { - if t.Base != "" && !seen[t.Base] { - seen[t.Base] = true - out = append(out, t.Base) - } - } - sort.Strings(out) - return out -} - func Languages() []string { names := make([]string, 0, len(toolchains)) for _, t := range toolchains { From f9014452d3a8307274b3b20541a20724fb98d018 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:10:20 +0200 Subject: [PATCH 35/38] List register and switch in the help, and none of the removed commands (issue 1851 A11a, A11b) The help still offered build, rebuild and replay --register, each now refused as removed, and did not name the terminal's register. --- cmd/mesh-controller/main.go | 20 +++++++++++--------- cmd/mesh-controller/usage_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 9 deletions(-) create mode 100644 cmd/mesh-controller/usage_test.go diff --git a/cmd/mesh-controller/main.go b/cmd/mesh-controller/main.go index 1bc06227..4382d52a 100644 --- a/cmd/mesh-controller/main.go +++ b/cmd/mesh-controller/main.go @@ -237,8 +237,8 @@ func run() error { } } -func usage() { - fmt.Fprint(os.Stderr, `mesh-controller — the control plane +// usageText is the terminal's help: every command it has, and none it refuses as removed (removedByName). +var usageText = `mesh-controller — the control plane On a node the operator types these as 'mesh-cli ' (in zsh, 'nox '): asked through the node's engine, and run as the controller's terminal only on the control-node @@ -312,15 +312,14 @@ func usage() { operator key make [--out ] make the operator's sealing key, off the mesh; private half to the file only operator key set [--replace] tell the mesh which operator key to seal to operator key show the operator key, and what it can recover - build [--ref R] have a build machine build it, and record what came out - build --behind build every module the mesh holds older than its source - build --on rebuild every module that stands on this module's artifacts, bases first + register / register a module from a repository the catalogue does not + build, from the builder's done key of its fingerprint (ADR 0501 §4) + switch [--json] the switch's list: each module, the build it runs, and what the builder holds builds [] what has been built lately, and what came of it queue [--json] every ask in the build queue: waiting, in flight (where, how long), dead cancel drop a waiting or dead ask; recorded failed, cancelled by hand clear [--dead] cancel every waiting ask (and the dead ones); never one in flight - rebuild ask the module's source again, or that build's, under a new id - replay [--register [--older]] that build's commit again; a dry run unless --register + replay that build's commit again, a dry run: what a build of it makes kill end a build where it runs; recorded failed, killed by hand pause [] / resume [] the build seat's holder there, or every holder, takes nothing new / again plans retry ask a failed plan's failed builds again, and carry the plan on @@ -355,10 +354,13 @@ func usage() { version what this binary is Each context reaches its own store through its own credential (novox/hq ADR 0008), named -`+store.Variable("")+` — or `+store.FileVariable("")+`, naming a file that holds +` + store.Variable("") + ` — or ` + store.FileVariable("") + `, naming a file that holds the same thing and keeps the password out of the environment. This process holds: -`) +` + +func usage() { + fmt.Fprint(os.Stderr, usageText) for _, c := range held { fmt.Fprintf(os.Stderr, " %-12s database %-12s from %s\n", c.name, store.Database(c.name), store.Variable(c.name)) diff --git a/cmd/mesh-controller/usage_test.go b/cmd/mesh-controller/usage_test.go new file mode 100644 index 00000000..3dd8a0c9 --- /dev/null +++ b/cmd/mesh-controller/usage_test.go @@ -0,0 +1,25 @@ +package main + +import ( + "regexp" + "strings" + "testing" +) + +// C3 (issue 1851 A11a, A11b, A13; ADR 0501 §4, §5): the terminal's help lists no command it refuses as removed, and lists +// the ones R4e added: `register` and `switch`. +func TestTheHelpListsNoRemovedCommandAndTheNewOnes(t *testing.T) { + for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery plan"} { + if regexp.MustCompile(`(?m)^ `+regexp.QuoteMeta(removed)+` `).MatchString(usageText) { + t.Errorf("the help still lists %q, which is refused as removed", removed) + } + } + if strings.Contains(usageText, "--register") { + t.Error("the help still lists replay --register, which is refused as removed") + } + for _, added := range []string{"register / ", "switch [--json]"} { + if !strings.Contains(usageText, " "+added) { + t.Errorf("the help does not list %q", added) + } + } +} From 1fd833bfe8b2777802a3f85b0af7d1919a7a1d22 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:18:58 +0200 Subject: [PATCH 36/38] List the store tests R4e adds, and drop those of the hand builds and the stopped checks it deleted (issue 1851, issue 1299) The merge check refused the eight deleted tests as listed and not run. --- testdata/store-tests | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/testdata/store-tests b/testdata/store-tests index 93092c0f..fbda4108 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -24,7 +24,6 @@ ./cmd/mesh-controller TestAWalkRetriedByAPersonIsSaidStartedAgain ./cmd/mesh-controller TestABacklogReleasedByAPersonIsSaidStarted ./cmd/mesh-controller TestAWalkEndedByCancelOrClearIsSaidEnded -./cmd/mesh-controller TestAWalkRestartedByARebuildIsSaidStarted ./cmd/mesh-controller TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain ./cmd/mesh-controller TestReplay237 ./cmd/mesh-controller TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer @@ -47,7 +46,6 @@ ./cmd/mesh-controller TestReplay348AFaultFromBeforeTheSendFailsNoGate ./cmd/mesh-controller TestReplay239AModuleNameIsNotTakenOverByAnotherRepository ./cmd/mesh-controller TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns -./cmd/mesh-controller TestAWalkWhoseEveryBuildCameOutTheSameIsDone ./cmd/mesh-controller TestReplay614ANewSDKPlansTheToolchainAndThenItsBundles ./internal/inventory TestACredentialIssuedOutsideIsNeverMade ./internal/inventory TestOneAcceptForAProviderIsSealedToEveryConsumer @@ -123,9 +121,6 @@ ./cmd/mesh-controller TestARecordedModulesMoveThatTouchesDataOrAProviderWaits ./cmd/mesh-controller TestAPersonsRecordAndTheBusHoldEveryMove ./cmd/mesh-controller TestARecordedMoveThatCannotBeJudgedWaits -./cmd/mesh-controller TestReplay867 -./cmd/mesh-controller TestReplay867Recheck -./cmd/mesh-controller TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn ./cmd/mesh-controller TestAssigningTheClientIssuesItsAccount ./cmd/mesh-controller TestAnUnrelatedBatchWalksBesideAnOpenWalk ./cmd/mesh-controller TestARelatedBatchWaitsBehindTheOpenWalk @@ -135,7 +130,6 @@ ./cmd/mesh-controller TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid ./cmd/mesh-controller TestARolloutAdvancesOnEventsAndAReplacedControllerResumesIt ./cmd/mesh-controller TestAWalkLetGoByTheDeliverVerbIsAskedWithNoOtherEvent -./cmd/mesh-controller TestTwoMergesWithinAMinuteLeaveEachOpenPullRequestOneQueuedCheck ./cmd/mesh-controller TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate ./cmd/mesh-controller TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt ./cmd/mesh-controller TestAModuleBuiltFromANewRepositoryIsRegisteredOnlyOnTheOperatorsApproveOnTheBus @@ -164,7 +158,11 @@ ./internal/inventory TestWhetherAMachineReadsModulesIsItsLatestWord ./cmd/mesh-controller TestAComposedDeclarationCarriesModulesAndBuildsOnlyForANodeThatReadsThem ./cmd/mesh-controller TestAPlanPassesATierWhoseMachineNeverReportsAfterTheBoundAndSendsTheNextToTheOthers -./cmd/mesh-controller TestAMergeOnTheApproveIntoANewRepositoryAsksTheFirstBuildAndItsOutcomeRegistersTheModule -./cmd/mesh-controller TestTheApprovedFirstBuildPassesTheGuardOfAVerbAndIsQueued ./cmd/mesh-controller TestAPlanGoesOnWithoutAMachineAwayWhenItsTierIsSent ./internal/inventory TestARefusedRegistrationIsKeptByItsBuild +./cmd/mesh-controller TestThePullRequestChecksAreAskedOnlyForARepositoryTheCheckerDoesNotJudge +./cmd/mesh-controller TestAMergeOnTheApproveIntoANewRepositoryRegistersItsModuleFromItsKey +./cmd/mesh-controller TestAMergeAddingAModuleEndsRegisteredFromItsKeyAndAsksNoBuild +./cmd/mesh-controller TestANewModuleWithAnUnreadWordWaitsForTheControllerThatReadsIt +./cmd/mesh-controller TestTheStatesAnnouncedGrantTheReads +./cmd/mesh-controller TestATryIsAskedOfTheChecker From 58512a9c6d482c3d169bd1a229cd08be4891ec43 Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:19:46 +0200 Subject: [PATCH 37/38] Pin mesh-sdk at the R4e member's final head 104533e8 and vendor it (R4e E) The SDK's S1 and S2 review fixes landed after 875c680d. --- go.mod | 2 +- go.sum | 10 ++-------- modules/route-proxy/go.mod | 2 +- modules/route-proxy/go.sum | 4 ++-- modules/route-proxy/vendor/modules.txt | 2 +- testdata/beside/CAPTURED | 2 +- .../git.novox.be/novox/mesh-sdk/go/buildstep/narrow.go | 6 ++++-- .../git.novox.be/novox/mesh-sdk/go/buildstep/trees.go | 4 ++-- vendor/modules.txt | 2 +- 9 files changed, 15 insertions(+), 19 deletions(-) diff --git a/go.mod b/go.mod index 6fbf57ab..bc54493b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-controller go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/jackc/pgx/v5 v5.10.0 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 diff --git a/go.sum b/go.sum index 60fa576d..bea58f3a 100644 --- a/go.sum +++ b/go.sum @@ -1,13 +1,7 @@ git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5 h1:E5stfIeCN6uYgA9TCv+4K8JgaURg3zQppvUgJLaDMP4= git.novox.be/novox/mesh-host v0.0.0-20261018171808-9d9e3a46c4f5/go.mod h1:4wyoCr6IIIIarEUuROThWDj7DAvJvIidqMjWcpSnOwE= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c h1:YdSbkI42rlVmClybH8BS717ut0vouKIMavO691fpdps= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022075101-c61212eba77c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b h1:cfJ2LK/etUiV1eL58pnaEjmSi34+EVmO4ud9J7pXKOs= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022152521-dfc2c858d94b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b h1:OjlhfiGr+wKLWxafV+BRjZx3TbBtwRYcm9mgoam/jv8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022161855-e76a5a31a07b/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 h1:dtiIJzrX7ey1wmsdzapiFgCKDllCph7/vxfUp1ib/t8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/modules/route-proxy/go.mod b/modules/route-proxy/go.mod index a1f29bc9..f33ea3f1 100644 --- a/modules/route-proxy/go.mod +++ b/modules/route-proxy/go.mod @@ -3,7 +3,7 @@ module route-proxy go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 ) diff --git a/modules/route-proxy/go.sum b/modules/route-proxy/go.sum index b72ac074..47599506 100644 --- a/modules/route-proxy/go.sum +++ b/modules/route-proxy/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 h1:dtiIJzrX7ey1wmsdzapiFgCKDllCph7/vxfUp1ib/t8= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 h1:Ufmd2cYEXICQ817IGflccP9bexABE4WhIO49HPbaoSo= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/modules/route-proxy/vendor/modules.txt b/modules/route-proxy/vendor/modules.txt index c65612ed..8fff12d3 100644 --- a/modules/route-proxy/vendor/modules.txt +++ b/modules/route-proxy/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go git.novox.be/novox/mesh-sdk/go/announce diff --git a/testdata/beside/CAPTURED b/testdata/beside/CAPTURED index e9c65105..84ac613c 100644 --- a/testdata/beside/CAPTURED +++ b/testdata/beside/CAPTURED @@ -9,7 +9,7 @@ a second place to clean besides the catalogue itself. mesh-catalog e0bc919d4945bb5f13f971f74643a0b714cf45da modules/*/module.json, modules/nats/Dockerfile mesh-host 9d9e3a46c4f5943cfffb413d962ce48b75e2fd39 examples/foundation-first-node.lock -mesh-sdk 875c680d9876863a3817bbf253d5fe4e54d62c2e conformance/events/module-event.json +mesh-sdk 104533e8c0c9116fd1cbb02c474bafbbe229ce22 conformance/events/module-event.json To move them, from this repository's root, with the two repositories checked out beside it: diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/narrow.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/narrow.go index 9bc6e00f..e27050f1 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/narrow.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/narrow.go @@ -9,8 +9,10 @@ import ( ) // narrowKept are the files of a module's directory a narrowed build keeps beside its named paths: its manifest and -// its lock files. -var narrowKept = []string{"module.json", "package.json", "package-lock.json", "go.mod", "go.sum"} +// its lock files, each an input of the module's key (ModuleTree's Manifest and Locks). package.json and go.mod are +// not kept: neither is in the named tree, so a build that needs one names it among its artifact's paths, which puts +// it in the key (novox/hq ADR 0499 clause 4). +var narrowKept = append([]string{"module.json"}, lockFiles...) // Narrow leaves in the fetched tree only what a build of an artifact that names its paths may see (novox/hq ADR 0499 // clause 4): r.Dir's named paths, its manifest and lock files, and .git; everything else is removed, so a file outside diff --git a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go index fc31c62d..27d35ed5 100644 --- a/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go +++ b/vendor/git.novox.be/novox/mesh-sdk/go/buildstep/trees.go @@ -76,10 +76,10 @@ func ModuleTrees(ctx context.Context, run forgekey.Runner, env []string, dir, co // narrow sets mt's Paths and its named Tree when the module's manifest names paths on every artifact (novox/hq ADR 0499 // clause 2): one `git ls-tree -z` gives each named path's object at commit. The merge preview reads the manifest here, as -// the indexer does, and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse +// the indexer does, with catalogue.ParseManifestForInputs (issue 1751), and no other build step reads paths from one (ADR 0495 clause 1). A manifest that does not parse // is not narrowed: the indexer gives it none. A named path absent at commit fails, naming the module and the path. func narrow(ctx context.Context, run forgekey.Runner, env []string, dir, commit, mdir string, raw []byte, mt *ModuleTree) error { - m, err := catalogue.ParseManifest(raw) + m, err := catalogue.ParseManifestForInputs(raw) if err != nil { return nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index ca0e1ff9..585419a2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,4 +1,4 @@ -# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022183749-875c680d9876 +# git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261022190740-104533e8c0c9 ## explicit; go 1.26.0 git.novox.be/novox/mesh-sdk/go git.novox.be/novox/mesh-sdk/go/announce From d51324df45184e049757436d7795e43237590a9e Mon Sep 17 00:00:00 2001 From: jochen Date: Thu, 22 Oct 2026 21:20:06 +0200 Subject: [PATCH 38/38] Format the help's test (issue 1851) --- cmd/mesh-controller/usage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mesh-controller/usage_test.go b/cmd/mesh-controller/usage_test.go index 3dd8a0c9..034ac0b2 100644 --- a/cmd/mesh-controller/usage_test.go +++ b/cmd/mesh-controller/usage_test.go @@ -10,7 +10,7 @@ import ( // the ones R4e added: `register` and `switch`. func TestTheHelpListsNoRemovedCommandAndTheNewOnes(t *testing.T) { for _, removed := range []string{"build", "rebuild", "forget-checks", "delivery plan"} { - if regexp.MustCompile(`(?m)^ `+regexp.QuoteMeta(removed)+` `).MatchString(usageText) { + if regexp.MustCompile(`(?m)^ ` + regexp.QuoteMeta(removed) + ` `).MatchString(usageText) { t.Errorf("the help still lists %q, which is refused as removed", removed) } }