diff --git a/cmd/mesh-builder/release_test.go b/cmd/mesh-builder/check_spec_test.go similarity index 64% rename from cmd/mesh-builder/release_test.go rename to cmd/mesh-builder/check_spec_test.go index 17eef10f..56f40d21 100644 --- a/cmd/mesh-builder/release_test.go +++ b/cmd/mesh-builder/check_spec_test.go @@ -7,16 +7,8 @@ import ( "github.com/novox/mesh-controller/internal/link" ) -// **A pull request into a release branch reaches the check as one** (novox/hq ADR 0358): the build seat hands the -// check the release branch and, beside it, the trunk each repository is read at when it has no branch of that name; -// and it hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own. -func TestTheBuildSeatHandsTheCheckItsReleaseBranch(t *testing.T) { - spec := checkSpecOf(link.BuildRequest{ID: "b", Repository: "r", Ref: "abc", Check: &link.CheckRequest{ - Owner: "novox", Repo: "mesh-catalog", Base: "feat/57", Release: "feat/57", - Beside: map[string]link.CheckedOut{"mesh-host": {Repository: "h", Ref: "feat/57", Else: "main"}}}}) - if spec.Release != "feat/57" || spec.Beside["mesh-host"].Ref != "feat/57" || spec.Beside["mesh-host"].Else != "main" { - t.Fatalf("the check was handed release %q and mesh-host %+v", spec.Release, spec.Beside["mesh-host"]) - } +// The build seat hands the check the judge cache the build agent declares (MESH_JUDGE_CACHE), never a path of its own. +func TestTheBuildSeatHandsTheCheckTheJudgeCacheItDeclares(t *testing.T) { t.Setenv("MESH_JUDGE_CACHE", "/var/lib/mesh/build-agent/judge-cache") if got := judgeCache(); got != "/var/lib/mesh/build-agent/judge-cache" { t.Fatalf("the judge cache the manifest declares is read as %q", got) diff --git a/cmd/mesh-builder/main.go b/cmd/mesh-builder/main.go index 4216b495..468f2a4b 100644 --- a/cmd/mesh-builder/main.go +++ b/cmd/mesh-builder/main.go @@ -358,8 +358,7 @@ func answer(ctx context.Context, publisher builder.Publisher, on, workspace stri result.Check = &link.CheckOutcome{Verdict: v.Verdict, Summary: v.Summary, Report: v.Report, Took: v.Took.Round(time.Second).String(), Gate: layerOf(v.Gate), RepoCheck: layerOf(v.Repo), Head: v.Head, Trunk: v.Trunk, Beside: v.Beside, Built: checkBuiltOf(v.Built), - Release: v.Release, JudgedBy: v.JudgedBy, BesideAt: v.BesideAt, Validator: v.Validator, - Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom} + Validator: v.Validator, Siblings: siblingsOut(v.Siblings), SiblingsUnknown: v.SiblingsUnknown, JudgeFrom: v.JudgeFrom} // What each of a group's heads was judged as (novox/hq ADR 0323 §6). for _, m := range v.Members { result.Check.Members = append(result.Check.Members, link.CheckedMember{Owner: m.Owner, Repo: m.Repo, @@ -498,13 +497,13 @@ func checkSpecOf(request link.BuildRequest) builder.CheckSpec { Owner: c.Owner, Repo: c.Repo, Number: c.Number, Paths: c.Paths, Beside: map[string]builder.Beside{}, Modules: c.Modules, New: c.New, Manifests: c.Manifests, Judge: c.Judge, Base: c.Base, Toolchain: builder.ToolchainOf(request.Held), Toolchains: builder.ToolchainsOf(request.Held), - Tests: c.Tests, StoreOnly: c.StoreOnly, Release: c.Release, SiblingsUnknown: c.SiblingsUnknown} + Tests: c.Tests, StoreOnly: c.StoreOnly, SiblingsUnknown: c.SiblingsUnknown} for _, s := range c.Siblings { spec.Siblings = append(spec.Siblings, builder.Sibling{Repository: s.Repository, Number: s.Number, Head: s.Head, Base: s.Base, Dir: s.Dir}) } for dir, b := range c.Beside { - spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref, Else: b.Else} + spec.Beside[dir] = builder.Beside{Repository: b.Repository, Ref: b.Ref} } for _, b := range c.Builds { spec.Builds = append(spec.Builds, builder.CheckBuildSpec{Module: b.Module, Repository: b.Repository, Path: b.Path, diff --git a/cmd/mesh-controller/acts_test.go b/cmd/mesh-controller/acts_test.go index a2e99a66..2e8e6261 100644 --- a/cmd/mesh-controller/acts_test.go +++ b/cmd/mesh-controller/acts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/addresses_integration_test.go b/cmd/mesh-controller/addresses_integration_test.go new file mode 100644 index 00000000..3d85873d --- /dev/null +++ b/cmd/mesh-controller/addresses_integration_test.go @@ -0,0 +1,228 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/licences" +) + +func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aStore()) + if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "distribution", + map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { + t.Fatal(err) + } + // A module the mesh built, recorded by digest and path, running on the other machine. + if err := open.inventory.RecordBuild(ctx, inventory.Build{ + ID: "b1", Repository: "r", Module: "app", Commit: "abc", + Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, + }); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "app", Version: "1", + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", + "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + // The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it + // asks where this machine reaches the store, as the docker module does. + register(t, open, catalogue.Manifest{Module: "runtime", Version: "1", + Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", + "into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}}) + if _, err := assign(ctx, open, "laptop", "runtime"); err != nil { + t.Fatal(err) + } + + on := map[string]bool{"anchor": true, "laptop": true} + node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) + if err != nil || !found || node != "anchor" || port != "5101" { + t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) + } + + var trust string + for _, r := range composed(t, open, "laptop").Resources { + if r["path"] == "/etc/docker/daemon.json" { + trust, _ = r["content"].(string) + } + if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { + t.Errorf("the image the mesh built is fetched as %v", r["image"]) + } + } + if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { + t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) + } + + // And a replay to the catalogue says where the store is now. + announced, err := following{open: open}.Announceable(ctx) + if err != nil { + t.Fatal(err) + } + if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { + t.Fatalf("the replay announces %+v", announced) + } +} + +func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + raw, err := os.ReadFile("../../module.json") + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle, + Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}}) + if err != nil { + t.Fatal(err) + } + register(t, open, control) + register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", + Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) + register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", + Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, + {Port: 5672, From: catalogue.FromMesh}}, + Guards: []int{15672}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", + "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) + // The control plane's own bus user is the installer's, seeded at genesis before the controller + // runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203). + if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller", + Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + // The store's module is registered and given its port, and NOT assigned: the state genesis + // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). + if err := open.inventory.SetSettings(ctx, "anchor", "postgres", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", + map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { + t.Fatal(err) + } + + var env map[string]any + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "mesh-controller.controller" { + env, _ = r["env"].(map[string]any) + } + } + if env == nil { + t.Fatal("the control plane's process is not in its own node's declaration") + } + for key, want := range map[string]string{ + "MESH_STORE_INVENTORY_PORT": "6852", + "MESH_STORE_IDENTITY_PORT": "6852", + "MESH_STORE_LICENCES_PORT": "6852", + "MESH_BROKER_AMQP_PORT": "5679", + // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, + // because the sealed value beside it carries the port genesis wrote (finding F3). + "MESH_BROKER_ADDRESS_PORT": "", + "MESH_BROKER_MANAGEMENT_PORT": "", + } { + if env[key] != want { + t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) + } + } +} + +func aLoneNode(t *testing.T) *stores { + t.Helper() + inventory.ForTest(t) + licences.ForTest(t) + open, err := openStores(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(open.Close) + for _, m := range provided { + if err := open.inventory.Provide(t.Context(), m); err != nil { + t.Fatal(err) + } + } + record, err := open.inventory.AddNode(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ + {"name": "container-runtime", "present": true}}}) + var profile map[string]any + _ = json.Unmarshal(reported, &profile) + if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + return open +} + +func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { + open := aLoneNode(t) + ctx := t.Context() + register(t, open, aStore()) + register(t, open, catalogue.Manifest{Module: "builder", Version: "1", + Requires: []string{catalogue.ArtifactStoreProvision}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", + "image": "registry.example/mesh-builder@" + aDigest}}}) + if err := open.inventory.RecordBuild(ctx, inventory.Build{ + ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", + Made: []inventory.Artifact{{Name: "runtime", Kind: "image", + Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, + }); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", + Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", + "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) + for _, module := range []string{"distribution", "builder", "postgres"} { + if _, err := assign(ctx, open, "anchor", module); err != nil { + t.Fatal(err) + } + } + if err := open.inventory.SetSettings(ctx, "anchor", "distribution", + map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { + t.Fatal(err) + } + + var image any + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "postgres.runtime" { + image = r["image"] + } + } + if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { + t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) + } + held := heldBy(ctx) + if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest { + t.Fatalf("a builder beside the store is handed the base %q", got) + } +} diff --git a/cmd/mesh-controller/addresses_test.go b/cmd/mesh-controller/addresses_test.go index bc02b37b..5eb8d617 100644 --- a/cmd/mesh-controller/addresses_test.go +++ b/cmd/mesh-controller/addresses_test.go @@ -2,13 +2,10 @@ package main import ( "encoding/json" - "os" "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/licences" "github.com/novox/mesh-controller/internal/link" ) @@ -86,150 +83,12 @@ func aStore() catalogue.Manifest { // **The trust a machine writes for the store, and the address every built image is fetched // through, say the port the node gave the store** — not the catalogue's number. -func TestTheRegistryTrustAndEveryImageFollowThePortTheNodeGaveTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aStore()) - if _, err := assign(ctx, open, "anchor", "distribution"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "distribution", - map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5101}}); err != nil { - t.Fatal(err) - } - // A module the mesh built, recorded by digest and path, running on the other machine. - if err := open.inventory.RecordBuild(ctx, inventory.Build{ - ID: "b1", Repository: "r", Module: "app", Commit: "abc", - Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}, - }); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "app", Version: "1", - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-app", - "image": catalogue.ArtifactStoreScheme + "app/server@" + aDigest}}}) - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - // The runtime's trust is the runtime's module's to write (novox/hq ADR 0222): a stand-in for it - // asks where this machine reaches the store, as the docker module does. - register(t, open, catalogue.Manifest{Module: "runtime", Version: "1", - Resources: []map[string]any{{"id": "daemon", "type": "file", "path": "/etc/docker/daemon.json", - "into": "json", "content": `{"insecure-registries": ["${seat:mesh-artifact-store:reach}"]}` + "\n"}}}) - if _, err := assign(ctx, open, "laptop", "runtime"); err != nil { - t.Fatal(err) - } - - on := map[string]bool{"anchor": true, "laptop": true} - node, port, found, err := artifactStoreOnNetwork(ctx, open.inventory, on) - if err != nil || !found || node != "anchor" || port != "5101" { - t.Fatalf("the store is found on %q:%q (%v, %v); the node put it on 5101", node, port, found, err) - } - - var trust string - for _, r := range composed(t, open, "laptop").Resources { - if r["path"] == "/etc/docker/daemon.json" { - trust, _ = r["content"].(string) - } - if r["id"] == "app.server" && r["image"] != "anchor.internal:5101/app/server@"+aDigest { - t.Errorf("the image the mesh built is fetched as %v", r["image"]) - } - } - if !strings.Contains(trust, "anchor.internal:5101") || strings.Contains(trust, ":5000") { - t.Fatalf("the runtime is told to trust %q; the node put the store on 5101", trust) - } - - // And a replay to the catalogue says where the store is now. - announced, err := following{open: open}.Announceable(ctx) - if err != nil { - t.Fatal(err) - } - if len(announced) != 1 || announced[0].Made[0].Reference != "anchor.internal:5101/app/server@"+aDigest { - t.Fatalf("the replay announces %+v", announced) - } -} // **The control plane's own connections say the port the node gave the store and the broker.** // // Composed from the control plane's own manifest against a real inventory: the store's module is // given 6852 on this node the way genesis or an operator gives it, and the control plane's // process is told so beside the sealed connection genesis wrote. -func TestTheControlPlaneIsToldWhereTheNodePutTheStoreAndTheBroker(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - raw, err := os.ReadFile("../../module.json") - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - t.Fatal(err) - } - control, err := withSeatPorts(m).Resolve([]catalogue.Built{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "https://registry.example/mesh-controller/controller.tar.gz", Digest: aDigest}}) - if err != nil { - t.Fatal(err) - } - register(t, open, control) - register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, "image": "pg@" + aDigest}}}) - register(t, open, catalogue.Manifest{Module: "lavinmq", Version: "1", - Claims: []catalogue.Claim{{Name: "mesh-broker", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5671, From: catalogue.FromMesh}, - {Port: 5672, From: catalogue.FromMesh}}, - Guards: []int{15672}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-broker", - "ports": []any{"5671:5671", "5672:5672", "127.0.0.1:15672:15672"}, "image": "mq@" + aDigest}}}) - // The control plane's own bus user is the installer's, seeded at genesis before the controller - // runs (SeedBusUser); without it a push now refuses the credential nobody issued (issue 203). - if err := open.inventory.SeedBusUser(ctx, inventory.BusUser{Username: "anchor.mesh-controller", - Kind: inventory.BusController, Node: "anchor", Module: "mesh-controller"}, "bootstrap"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - // The store's module is registered and given its port, and NOT assigned: the state genesis - // leaves a given-port node in before the foundation is adopted as modules (04-ISSUES/085). - if err := open.inventory.SetSettings(ctx, "anchor", "postgres", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 6852}}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "lavinmq"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "lavinmq", - map[string]any{catalogue.PortsSetting: map[string]any{"5672": 5679}}); err != nil { - t.Fatal(err) - } - - var env map[string]any - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "mesh-controller.controller" { - env, _ = r["env"].(map[string]any) - } - } - if env == nil { - t.Fatal("the control plane's process is not in its own node's declaration") - } - for key, want := range map[string]string{ - "MESH_STORE_INVENTORY_PORT": "6852", - "MESH_STORE_IDENTITY_PORT": "6852", - "MESH_STORE_LICENCES_PORT": "6852", - "MESH_BROKER_AMQP_PORT": "5679", - // Neither given nor assigned by the mesh: the manifest's own number is NOT the answer, - // because the sealed value beside it carries the port genesis wrote (finding F3). - "MESH_BROKER_ADDRESS_PORT": "", - "MESH_BROKER_MANAGEMENT_PORT": "", - } { - if env[key] != want { - t.Errorf("the control plane is told %s=%v; the node says %q", key, env[key], want) - } - } -} // withSeatPorts is the control plane's manifest with the seat placeholders in its environment — // added here until module.json carries them (the manifest lands one commit after the code that @@ -274,79 +133,7 @@ func withSeatPorts(m catalogue.Manifest) catalogue.Manifest { // aLoneNode is one capable machine with nothing placed on any network — the control-node during // genesis, before the "network" step, which is after the store, the broker, the vault and the // catalogue have each been built and pushed (finding F2). -func aLoneNode(t *testing.T) *stores { - t.Helper() - inventory.ForTest(t) - licences.ForTest(t) - open, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(open.Close) - for _, m := range provided { - if err := open.inventory.Provide(t.Context(), m); err != nil { - t.Fatal(err) - } - } - record, err := open.inventory.AddNode(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - reported, _ := json.Marshal(map[string]any{"capabilities": []map[string]any{ - {"name": "container-runtime", "present": true}}}) - var profile map[string]any - _ = json.Unmarshal(reported, &profile) - if err := open.inventory.RecordProfile(t.Context(), record.ID, profile); err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(t.Context(), record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - return open -} // **Before the network exists, the store's own node reaches it by loopback** — never refused, // never handed the scheme: a genesis pushes the store, the broker, the vault and the catalogue to // a node on no network, and builds the catalogue on a base it must be able to pull. -func TestOnANodeWithNoNetworkTheStoreIsReachedByLoopback(t *testing.T) { - open := aLoneNode(t) - ctx := t.Context() - register(t, open, aStore()) - register(t, open, catalogue.Manifest{Module: "builder", Version: "1", - Requires: []string{catalogue.ArtifactStoreProvision}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-builder", - "image": "registry.example/mesh-builder@" + aDigest}}}) - if err := open.inventory.RecordBuild(ctx, inventory.Build{ - ID: "b1", Repository: "r", Module: "postgres", Commit: "abc", - Made: []inventory.Artifact{{Name: "runtime", Kind: "image", - Reference: catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}, - }); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "postgres", Version: "1", - Resources: []map[string]any{{"id": "runtime", "type": "container", "name": "mesh-postgres", - "image": catalogue.ArtifactStoreScheme + "postgres/runtime@" + aDigest}}}) - for _, module := range []string{"distribution", "builder", "postgres"} { - if _, err := assign(ctx, open, "anchor", module); err != nil { - t.Fatal(err) - } - } - if err := open.inventory.SetSettings(ctx, "anchor", "distribution", - map[string]any{catalogue.PortsSetting: map[string]any{"5000": 5100}}); err != nil { - t.Fatal(err) - } - - var image any - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "postgres.runtime" { - image = r["image"] - } - } - if image != "127.0.0.1:5100/postgres/runtime@"+aDigest { - t.Fatalf("on the store's own node, off any network, the image is fetched as %v", image) - } - held := heldBy(ctx) - if got := held["postgres/runtime"]; got != "127.0.0.1:5100/postgres/runtime@"+aDigest { - t.Fatalf("a builder beside the store is handed the base %q", got) - } -} diff --git a/cmd/mesh-controller/adopting_test.go b/cmd/mesh-controller/adopting_test.go index 3589fec0..5561bb05 100644 --- a/cmd/mesh-controller/adopting_test.go +++ b/cmd/mesh-controller/adopting_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/agent_account_integration_test.go b/cmd/mesh-controller/agent_account_integration_test.go new file mode 100644 index 00000000..1d615a1c --- /dev/null +++ b/cmd/mesh-controller/agent_account_integration_test.go @@ -0,0 +1,161 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-host/rootsearch" +) + +func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.NodeByName(ctx, n); err != nil { + if _, err := inv.AddNode(ctx, n); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAccount(ctx, n, "ops", ""); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { + t.Fatal(err) + } + d := &doctor{open: open} + found, err := probeAgentAccounts(ctx, d) + if err != nil { + t.Fatal(err) + } + found = onlyMachine(found, "anchor") + if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent || + !strings.Contains(found[0].Said, "not judged") { + t.Fatalf("a named agent account with no verdict: %+v", found) + } + w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) + if w.Headline == "" || w.Needs == "" || w.Resolved == "" { + t.Errorf("the condition has no plain words: %+v", w) + } + + healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", + Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"} + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, + SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil { + t.Fatal(err) + } + if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 { + t.Fatalf("a judged agent account still fails: %+v %v", found, err) + } + if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined { + t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err) + } + if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named || + !strings.Contains(why, "operator account") { + t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err) + } +} + +func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) { + open, _ := aMeshWithSecrets(t) + ctx := t.Context() + if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil { + t.Fatal(err) + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + body, _ := f.Encode() + if strings.Contains(string(body), "warden") { + t.Error("the agent account's name is in the snapshot") + } + m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor")) + if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount { + t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account) + } +} + +func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { + if searchQuietFor != rootsearch.Quiet { + t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet) + } + // A daily search that finishes within the quiet never leaves the account unjudged between two of them. + if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet { + t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)", + rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet) + } + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.NodeByName(ctx, "anchor"); err != nil { + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + } + if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { + t.Fatal(err) + } + d := &doctor{open: open} + say := func(state, reason string) []conditions.Observation { + t.Helper() + // The engine's since is always now: it was just restarted. + v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", + Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, + Account: "agent", Since: time.Now()} + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, + SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { + t.Fatal(err) + } + found, err := probeAgentAccounts(ctx, d) + if err != nil { + t.Fatal(err) + } + return onlyMachine(found, "anchor") + } + running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" + healthy := func() { + t.Helper() + if found := say(link.StateHealthy, ""); len(found) != 0 { + t.Fatalf("a healthy verdict raised: %+v", found) + } + } + if found := say(link.StateUnknown, running); len(found) != 0 { + t.Fatalf("a search first seen now was raised: %+v", found) + } + if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") { + t.Fatalf("an account whose search runs was read as confined: %q", why) + } + // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) + } + healthy() // a complete verdict forgets when the waiting began … + if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { + t.Fatal(err) // … and this restart loop began past the bound + } + if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) + } + healthy() + incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + + "19:23: timeout); it is tried again later" + if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { + t.Fatalf("a search that did not finish was not urgent: %+v", found) + } + if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || + found[0].Severity != conditions.Urgent { + t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) + } +} diff --git a/cmd/mesh-controller/agent_account_test.go b/cmd/mesh-controller/agent_account_test.go index ee859684..63d5dff9 100644 --- a/cmd/mesh-controller/agent_account_test.go +++ b/cmd/mesh-controller/agent_account_test.go @@ -1,14 +1,12 @@ package main import ( - "github.com/novox/mesh-host/rootsearch" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" - snapshot "github.com/novox/mesh-controller/internal/facts" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -65,55 +63,6 @@ func TestAnAgentAccountIsConfinedOnlyOnAHealthyVerdictJudgedForRoot(t *testing.T // DA raises an urgent condition, with plain words, on a machine whose agent account is not judged unable to // become root; a machine that names none is not its to judge. -func TestTheSelfCheckSaysAnAgentAccountThatCanBecomeRoot(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.NodeByName(ctx, n); err != nil { - if _, err := inv.AddNode(ctx, n); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAccount(ctx, n, "ops", ""); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { - t.Fatal(err) - } - d := &doctor{open: open} - found, err := probeAgentAccounts(ctx, d) - if err != nil { - t.Fatal(err) - } - found = onlyMachine(found, "anchor") - if len(found) != 1 || found[0].Machine != "anchor" || found[0].Severity != conditions.Urgent || - !strings.Contains(found[0].Said, "not judged") { - t.Fatalf("a named agent account with no verdict: %+v", found) - } - w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) - if w.Headline == "" || w.Needs == "" || w.Resolved == "" { - t.Errorf("the condition has no plain words: %+v", w) - } - - healthy := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: "agent", State: link.StateHealthy, Root: link.RootNever, Account: "agent"} - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, - SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{healthy}}); err != nil { - t.Fatal(err) - } - if found, err = probeAgentAccounts(ctx, d); err != nil || len(onlyMachine(found, "anchor")) != 0 { - t.Fatalf("a judged agent account still fails: %+v %v", found, err) - } - if named, confined, why, err := agentConfined(ctx, inv, "anchor", time.Now()); err != nil || !named || !confined { - t.Fatalf("agentConfined on anchor: %v %v %q %v", named, confined, why, err) - } - if named, _, why, err := agentConfined(ctx, inv, "laptop", time.Now()); err != nil || named || - !strings.Contains(why, "operator account") { - t.Fatalf("agentConfined on a machine naming none: %v %q %v", named, why, err) - } -} func TestTheAgentRootWordsArePlain(t *testing.T) { w := plainWordings[kindAgentCanBecomeRoot](conditions.Observation{Kind: kindAgentCanBecomeRoot, Machine: "anchor"}) @@ -134,28 +83,6 @@ func onlyMachine(obs []conditions.Observation, machine string) []conditions.Obse // The snapshot a merge check composes from carries the agent account as a pseudonym (novox/hq ADR 0266), // so a change is judged against machines that name one, and the name never leaves. -func TestTheFactsCarryTheAgentAccountAsAPseudonym(t *testing.T) { - open, _ := aMeshWithSecrets(t) - ctx := t.Context() - if err := open.inventory.SetAccount(ctx, "anchor", "keeper", ""); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetAgentAccount(ctx, "anchor", "warden", ""); err != nil { - t.Fatal(err) - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - body, _ := f.Encode() - if strings.Contains(string(body), "warden") { - t.Error("the agent account's name is in the snapshot") - } - m, ok := f.Machine(snapshot.Pseudonym("machine", "anchor")) - if !ok || m.AgentAccount != snapshot.Pseudonym("account", "warden") || m.Account == m.AgentAccount { - t.Fatalf("the anchor's agent account reads as %q (operator %q)", m.AgentAccount, m.Account) - } -} // No verb runs a `node` command that sets something: through the generic `command` verb, `node account`, // `node agent-account` and every other `node` subcommand but list and show are refused, naming the terminal. @@ -205,75 +132,3 @@ func TestTheNodeVerbOnlyShows(t *testing.T) { // controller first saw it waiting — never from the engine's own "since", which a restart resets, so an engine // restarted in a loop does not keep it quiet. The account is still not confined, and `node show` still says not // judged; a search that failed, or a way to root found, is urgent at once. -func TestASearchStillRunningAfterARestartIsNotUrgent(t *testing.T) { - if searchQuietFor != rootsearch.Quiet { - t.Fatalf("the quiet is %s and the node-engine's %s: they are one value", searchQuietFor, rootsearch.Quiet) - } - // A daily search that finishes within the quiet never leaves the account unjudged between two of them. - if rootsearch.FreshFor < rootsearch.Every+rootsearch.Quiet { - t.Fatalf("a complete search judges for %s, less than a day's search (%s) and the quiet (%s)", - rootsearch.FreshFor, rootsearch.Every, rootsearch.Quiet) - } - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.NodeByName(ctx, "anchor"); err != nil { - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - } - if err := inv.SetAgentAccount(ctx, "anchor", "agent", ""); err != nil { - t.Fatal(err) - } - d := &doctor{open: open} - say := func(state, reason string) []conditions.Observation { - t.Helper() - // The engine's since is always now: it was just restarted. - v := inventory.ResourceHealth{Module: "claude-code", Resource: "claude-code.agent-account", - Kind: link.KindAccount, Target: "agent", State: state, Reason: reason, Root: link.RootNever, - Account: "agent", Since: time.Now()} - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "anchor", Contract: link.RootContract, - SaidAt: time.Now(), HeardAt: time.Now(), Resources: []inventory.ResourceHealth{v}}); err != nil { - t.Fatal(err) - } - found, err := probeAgentAccounts(ctx, d) - if err != nil { - t.Fatal(err) - } - return onlyMachine(found, "anchor") - } - running := link.ReasonRootPending + ": the search for setuid programs, started at 19:21, has not finished yet" - healthy := func() { - t.Helper() - if found := say(link.StateHealthy, ""); len(found) != 0 { - t.Fatalf("a healthy verdict raised: %+v", found) - } - } - if found := say(link.StateUnknown, running); len(found) != 0 { - t.Fatalf("a search first seen now was raised: %+v", found) - } - if _, confined, why, _ := agentConfined(ctx, inv, "anchor", time.Now()); confined || !strings.Contains(why, "not judged") { - t.Fatalf("an account whose search runs was read as confined: %q", why) - } - // The engine restarted again and again, each statement's own since fresh: the controller's clock runs on. - if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { - t.Fatal(err) - } - healthy() // a complete verdict forgets when the waiting began … - if _, err := inv.RootSearchPending(ctx, "anchor", time.Now().Add(-searchQuietFor-time.Minute)); err != nil { - t.Fatal(err) // … and this restart loop began past the bound - } - if found := say(link.StateUnknown, running); len(found) != 1 || found[0].Severity != conditions.Urgent { - t.Fatalf("a search pending past the bound, by the controller's clock, was not urgent: %+v", found) - } - healthy() - incomplete := rootsearch.ReasonIncomplete + ": the search for setuid programs did not finish (last tried at " + - "19:23: timeout); it is tried again later" - if found := say(link.StateUnknown, incomplete); len(found) != 1 || found[0].Severity != conditions.Urgent { - t.Fatalf("a search that did not finish was not urgent: %+v", found) - } - if found := say(link.StateUnhealthy, link.ReasonRoot+": in the group docker; "+running); len(found) != 1 || - found[0].Severity != conditions.Urgent { - t.Fatalf("a way to root found while the search runs was not urgent: %+v", found) - } -} diff --git a/cmd/mesh-controller/api_test.go b/cmd/mesh-controller/api_test.go index 876c5a9b..9cc5d99a 100644 --- a/cmd/mesh-controller/api_test.go +++ b/cmd/mesh-controller/api_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/asker_bus_test.go b/cmd/mesh-controller/asker_bus_test.go index 0e22c949..32c53642 100644 --- a/cmd/mesh-controller/asker_bus_test.go +++ b/cmd/mesh-controller/asker_bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/awaiting_push_integration_test.go b/cmd/mesh-controller/awaiting_push_integration_test.go new file mode 100644 index 00000000..ee881d98 --- /dev/null +++ b/cmd/mesh-controller/awaiting_push_integration_test.go @@ -0,0 +1,182 @@ +//go:build integration + +package main + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/overlay" +) + +func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + + // The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing + // can be compared about a secret that does not exist (status), and nothing here string-matches. + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + _, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading) + var notMade *catalogue.NotMadeError + if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" { + t.Fatalf("a read composition did not say which secret is not made, typed: %v", err) + } + // Foreseen, it composes, names what the push makes, and made nothing. + foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing) + if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" { + t.Fatalf("foreseen: %v %v", foreseen.foreseen, err) + } + if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held { + t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err) + } + + // Within the bound: nothing at all — no urgent, no warning, nobody notified. + if got := d1(t, open); len(got) != 0 { + t.Fatalf("a machine waiting for a push raised %+v", got) + } + + // Past the bound: a warning, not urgent, saying what the push will make. + before := awaitingPushBound + awaitingPushBound = -time.Minute + t.Cleanup(func() { awaitingPushBound = before }) + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning || + !strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") { + t.Fatalf("a machine left un-pushed past the bound: %+v", got) + } + + // Pushed: the secret is made and D1 says nothing. + pushedBy(t, open, "laptop") + if got := d1(t, open); len(got) != 0 { + t.Fatalf("a pushed machine still raised %+v", got) + } +} + +func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aTalker()) + if _, err := assign(ctx, open, "laptop", "talker"); err != nil { + t.Fatal(err) + } + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent || + !strings.Contains(got[0].Summary, "module issue talker --node laptop") { + t.Fatalf("a push that will be refused was not said urgently: %+v", got) + } + + // And a machine whose composition fails for anything else, with a secret waiting beside it, is + // uncomposable for that — the stand-in hides nothing. + register(t, open, aKeeper()) + if _, err := assign(ctx, open, "anchor", "keeper"); err != nil { + t.Fatal(err) + } + one, two := rivals() + register(t, open, one) + register(t, open, two) + _, _ = assign(ctx, open, "anchor", "rival-one") + _, _ = assign(ctx, open, "anchor", "rival-two") + keys := map[string]conditions.Severity{} + for _, o := range d1(t, open) { + keys[o.Key()] = o.Severity + } + if keys["machine.anchor.uncomposable"] != conditions.Urgent { + t.Fatalf("a real failure beside a waiting secret: %v", keys) + } +} + +func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil { + t.Fatal(err) + } + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + got := d1(t, open) + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") { + t.Fatalf("a given secret under an old key: %+v", got) + } +} + +func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") + sent := time.Now() + since, err := open.inventory.AwaitingSince(ctx, "laptop") + if err != nil || since.After(sent) { + t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent) + } + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + since, err = open.inventory.AwaitingSince(ctx, "laptop") + if err != nil || since.Before(sent.Add(-time.Second)) { + t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err) + } +} + +func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aKeeper()) + pushedBy(t, open, "laptop") + if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { + t.Fatal(err) + } + sends, err := readDeliveries(ctx, open.inventory) + if err != nil { + t.Fatal(err) + } + if sends["laptop"].settled("keeper", time.Now()) { + t.Fatal("a holder assigned and not pushed is expected to answer") + } + if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) { + t.Fatal("a module the machine was sent long ago is not expected to answer") + } + pushedBy(t, open, "laptop") + sends, err = readDeliveries(ctx, open.inventory) + if err != nil { + t.Fatal(err) + } + if sends["laptop"].settled("keeper", time.Now()) { + t.Fatal("pushed a moment ago and not reported, a holder is already expected") + } + if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) { + t.Fatal("pushed past the grace, a holder is not expected") + } + if _, ok := sends["laptop"].carried["keeper"]; !ok { + t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried) + } + pushedAndApplied(t, open, "laptop") + if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) { + t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err) + } +} diff --git a/cmd/mesh-controller/awaiting_push_test.go b/cmd/mesh-controller/awaiting_push_test.go index e0890c6f..0ec32245 100644 --- a/cmd/mesh-controller/awaiting_push_test.go +++ b/cmd/mesh-controller/awaiting_push_test.go @@ -1,15 +1,12 @@ package main import ( - "errors" - "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/overlay" ) // Between `assign` and `push` a module's own secrets are not made yet: the push makes them. D1 composed @@ -81,140 +78,14 @@ func d1(t *testing.T, open *stores) []conditions.Observation { // **THE WINDOW, REPRODUCED**: assigned and not pushed, a module whose own secret the push makes is // waiting, not uncomposable; past the bound it is a warning naming what the push makes; pushed, nothing. -func TestAModuleAssignedAndNotPushedIsAwaitingAPushNotUncomposable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") // the machine was pushed before; then the module is assigned - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - - // The read the rest of the mesh asks still refuses it, with the composer's typed error: nothing - // can be compared about a secret that does not exist (status), and nothing here string-matches. - plan, settings, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - gens, err := generators(ctx, open) - if err != nil { - t.Fatal(err) - } - _, err = declarationWith(ctx, open, "laptop", plan, settings, gens, Reading) - var notMade *catalogue.NotMadeError - if !errors.As(err, ¬Made) || notMade.Module != "keeper" || notMade.Name != "repository" { - t.Fatalf("a read composition did not say which secret is not made, typed: %v", err) - } - // Foreseen, it composes, names what the push makes, and made nothing. - foreseen, err := declarationWith(ctx, open, "laptop", plan, settings, gens, Foreseeing) - if err != nil || len(foreseen.foreseen) != 1 || foreseen.foreseen[0] != "keeper/repository" { - t.Fatalf("foreseen: %v %v", foreseen.foreseen, err) - } - if _, held, err := open.inventory.ModuleSecretIfIssued(ctx, "laptop", "keeper", "repository"); err != nil || held { - t.Fatalf("asking ahead of the push made the secret (held %v, %v)", held, err) - } - - // Within the bound: nothing at all — no urgent, no warning, nobody notified. - if got := d1(t, open); len(got) != 0 { - t.Fatalf("a machine waiting for a push raised %+v", got) - } - - // Past the bound: a warning, not urgent, saying what the push will make. - before := awaitingPushBound - awaitingPushBound = -time.Minute - t.Cleanup(func() { awaitingPushBound = before }) - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.awaiting-push" || got[0].Severity != conditions.Warning || - !strings.Contains(got[0].Summary, "keeper/repository") || !strings.Contains(got[0].Summary, "push laptop") { - t.Fatalf("a machine left un-pushed past the bound: %+v", got) - } - - // Pushed: the secret is made and D1 says nothing. - pushedBy(t, open, "laptop") - if got := d1(t, open); len(got) != 0 { - t.Fatalf("a pushed machine still raised %+v", got) - } -} // **A REAL FAILURE IS STILL URGENT**: a secret the push would be refused on is not one it will make. // A bus credential nobody issued (issue 203) fails the push, so D1 says it — urgent, in the push's words. -func TestASecretThePushCannotMakeIsStillUncomposable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aTalker()) - if _, err := assign(ctx, open, "laptop", "talker"); err != nil { - t.Fatal(err) - } - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || got[0].Severity != conditions.Urgent || - !strings.Contains(got[0].Summary, "module issue talker --node laptop") { - t.Fatalf("a push that will be refused was not said urgently: %+v", got) - } - - // And a machine whose composition fails for anything else, with a secret waiting beside it, is - // uncomposable for that — the stand-in hides nothing. - register(t, open, aKeeper()) - if _, err := assign(ctx, open, "anchor", "keeper"); err != nil { - t.Fatal(err) - } - one, two := rivals() - register(t, open, one) - register(t, open, two) - _, _ = assign(ctx, open, "anchor", "rival-one") - _, _ = assign(ctx, open, "anchor", "rival-two") - keys := map[string]conditions.Severity{} - for _, o := range d1(t, open) { - keys[o.Key()] = o.Severity - } - if keys["machine.anchor.uncomposable"] != conditions.Urgent { - t.Fatalf("a real failure beside a waiting secret: %v", keys) - } -} // A given secret sealed to a key the machine no longer has is not the mesh's to make again: the push is // refused on it, so D1 is too. -func TestAGivenSecretUnderAnOldKeyIsNotForeseen(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "keeper", "repository", "given"); err != nil { - t.Fatal(err) - } - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - got := d1(t, open) - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "issue it again") { - t.Fatalf("a given secret under an old key: %+v", got) - } -} // The bound is read from when the machine began waiting: the oldest assignment since its last send. -func TestAMachineAwaitsAPushSinceItsOldestAssignmentSinceTheLastSend(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") - sent := time.Now() - since, err := open.inventory.AwaitingSince(ctx, "laptop") - if err != nil || since.After(sent) { - t.Fatalf("nothing assigned since the send: waiting since %v (%v), the send was before %v", since, err, sent) - } - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - since, err = open.inventory.AwaitingSince(ctx, "laptop") - if err != nil || since.Before(sent.Add(-time.Second)) { - t.Fatalf("assigned after the send: waiting since %v (%v), not from the assignment", since, err) - } -} // **D3 AND D13 WAIT FOR THE SEND**: a holder is expected to answer on a machine once the machine was sent // it and had time to report — never between assign and push. @@ -242,40 +113,3 @@ func TestAHolderIsExpectedOnlyOnceSentAndReported(t *testing.T) { } // And through the stores: assigned, not in the last send; pushed and reported, carried and settled. -func TestALastSendIsReadFromTheSendAndTheReport(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aKeeper()) - pushedBy(t, open, "laptop") - if _, err := assign(ctx, open, "laptop", "keeper"); err != nil { - t.Fatal(err) - } - sends, err := readDeliveries(ctx, open.inventory) - if err != nil { - t.Fatal(err) - } - if sends["laptop"].settled("keeper", time.Now()) { - t.Fatal("a holder assigned and not pushed is expected to answer") - } - if !sends["laptop"].settled(overlay.Name, time.Now().Add(time.Hour)) { - t.Fatal("a module the machine was sent long ago is not expected to answer") - } - pushedBy(t, open, "laptop") - sends, err = readDeliveries(ctx, open.inventory) - if err != nil { - t.Fatal(err) - } - if sends["laptop"].settled("keeper", time.Now()) { - t.Fatal("pushed a moment ago and not reported, a holder is already expected") - } - if !sends["laptop"].settled("keeper", time.Now().Add(reportGrace+time.Minute)) { - t.Fatal("pushed past the grace, a holder is not expected") - } - if _, ok := sends["laptop"].carried["keeper"]; !ok { - t.Fatalf("the send's builds do not carry keeper: %v", sends["laptop"].carried) - } - pushedAndApplied(t, open, "laptop") - if sends, err = readDeliveries(ctx, open.inventory); err != nil || !sends["laptop"].settled("keeper", time.Now()) { - t.Fatalf("pushed and reported applied, a holder is not expected to answer (%v)", err) - } -} diff --git a/cmd/mesh-controller/batches.go b/cmd/mesh-controller/batches.go index 81a52afb..96235aa1 100644 --- a/cmd/mesh-controller/batches.go +++ b/cmd/mesh-controller/batches.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "os" - "regexp" "slices" "sort" "strconv" @@ -71,9 +70,9 @@ const batchLateAfter = time.Minute // made before the controller began keeping merges (owedLate). const mergeGrace = 10 * time.Minute -// edgeGroupOrder is a delivery group's order made an edge of a walk's tiers (ADR 0276 decision 5): the -// modules of the member after depend on the modules of the member before, and are gated on them. -const edgeGroupOrder = "group-order" +// edgeChangeOrder is the order the rules of a plan draw among its modules (novox/hq ADR 0419 clauses 3 and 4, delivery_edges.go) +// made an edge of its tiers: the module after depends on the module before, and is gated on it. +const edgeChangeOrder = "change-order" // mergeWindowOf is the window and its maximum, from the controller's settings; a value unreadable is its // default, and said. @@ -177,12 +176,12 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T repository, m.Base, m.Commit, already) return nil } - fmt.Printf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit) + fmt.Print(nothingReadsIt(repository, m)) return nil } // **Its delivery group's word, before the plans are held** (ADR 0306 decision 2): a slow answer holds up no walk. own := ownPath(touches) - waits, err := groupWait(ctx, inv, m, deliverySeatHeld(entries), own) + waits, holding, err := groupWaitHolding(ctx, inv, m, deliverySeatHeld(entries)) if err != nil { return notNow(err) } @@ -238,6 +237,7 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T if added, err := inv.AddMerge(ctx, heard); err != nil || !added { return notNow(err) } + holdFor(&batch, m.Head, waits, holding, own, now) if err := keepBatch(ctx, inv, &batch, now, ""); err != nil { return notNow(err) } @@ -246,6 +246,43 @@ func (f following) hearMerge(ctx context.Context, m link.SourceMoved, now time.T return cutBatchesHeld(ctx, f.open, now) } +// holdFor keeps in a batch that a merge of a group joined whether that group still has a member to merge (ADR 0418 +// clause 3): held while it has, the window running again from the moment the last hold is cleared. A group with a +// merge of the controller's own path makes its batch an own-path one: such a plan waits for nobody's word, and is +// started by the controller when the group's batch closes (clause 6). +func holdFor(batch *inventory.Plan, head, why string, holding, own bool, now time.Time) { + if batch.Delivery == nil { + batch.Delivery = &inventory.PlanDelivery{} + } + if batch.Delivery.Batch == nil { + batch.Delivery.Batch = &inventory.PlanBatch{Own: own} + } + w := batch.Delivery.Batch + w.Own = w.Own || own + if head == "" { + return + } + if holding { + if w.Holds == nil { + w.Holds = map[string]string{} + } + w.Holds[head] = why + return + } + if _, held := w.Holds[head]; held { + holds := map[string]string{} + for h, y := range w.Holds { + if h != head { + holds[h] = y + } + } + w.Holds = holds + if len(holds) == 0 { + w.Holds, w.HeldAt = nil, now + } + } +} + // owedLate says a merge that reads as history is still the record's to answer (review of this change): a cut // marks every module it moved seen at that moment, so a merge of that branch // handed over late — made before the cut, heard after it — reads as history and was dropped. One that touches @@ -350,6 +387,15 @@ func batchToJoin(ctx context.Context, inv *inventory.Inventory, heard inventory. if err != nil { return inventory.Plan{}, "", err } + // **A group is one batch, whatever kinds its members are** (novox/hq ADR 0418 clause 6): a merge joins the batch + // of the other kind that holds a merge from its head. + if group, ok, err := batchOfGroup(ctx, inv, announcedOf(heard).Head, heard.Repository); err != nil { + return inventory.Plan{}, "", err + } else if ok && group.OwnPath() != own && group.Delivery != nil && group.Delivery.Batch != nil { + group.Delivery.Batch.CutAtOnce = false + return group, "its group " + announcedOf(heard).Head + " has a merge in " + group.ID + ", a batch of the other kind: " + + "a group is one batch", nil + } joins, whys, _, err := relatedBatches(ctx, inv, []inventory.BatchedMerge{heard}, own, "", entries, read, edges) if err != nil { return inventory.Plan{}, "", err @@ -573,18 +619,25 @@ func keepBatchBecause(ctx context.Context, inv *inventory.Inventory, b *inventor if b.Delivery == nil { b.Delivery = &inventory.PlanDelivery{} } - closes, latest := windowOf(merges, b.Created, window, atMost) + anchor := b.Created + if was := b.Delivery.Batch; was != nil && was.HeldAt.After(anchor) { + anchor = was.HeldAt // the maximum counts from the last hold's end (ADR 0418 clause 3) + } + closes, latest := windowOf(merges, anchor, window, atMost) b.Delivery.Merges = named var closed, cutAtOnce bool var withheld []string var waits string + var holds map[string]string + var heldAt time.Time if was := b.Delivery.Batch; was != nil { + holds, heldAt = was.Holds, was.HeldAt closed, withheld = was.Closed, was.Withheld // a stop's, which no look takes back (novox/hq ADR 0299) cutAtOnce = was.CutAtOnce // the merge's, until another joins it (ADR 0304 decision 1) waits = was.Waits // its group's, as said when it was made (ADR 0306 decision 2) } b.Delivery.Batch = &inventory.PlanBatch{ClosesAt: closes, AtMost: latest, Behind: behind, BehindWhy: why, Own: b.OwnPath(), - Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits} + Closed: closed, Withheld: withheld, CutAtOnce: cutAtOnce, Waits: waits, Holds: holds, HeldAt: heldAt} b.State = inventory.PlanAssembling if behind != "" && windowClosed(*b, now) { b.State = inventory.PlanQueued @@ -620,13 +673,20 @@ func windowOf(merges []inventory.BatchedMerge, created time.Time, window, atMost } // windowClosed says a batch's window is closed: closed by a stop, closed at once for a merge that shares -// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached. +// nothing (ADR 0304 decision 1), no merge for the window's length, or its maximum reached. **A batch holding a group +// with a member still to merge is closed by none of the last three** (novox/hq ADR 0418 clause 3). func windowClosed(b inventory.Plan, now time.Time) bool { if b.Delivery == nil || b.Delivery.Batch == nil { return true } w := b.Delivery.Batch - return w.Closed || w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) + if w.Closed { + return true + } + if len(w.Holds) > 0 { + return false // a group's last member is still to merge: neither the window nor its maximum closes the batch + } + return w.CutAtOnce || !now.Before(w.ClosesAt) || !now.Before(w.AtMost) } // carriedOf is a set of merges as a walk carries them: the latest merge of each repository's branch, and @@ -790,6 +850,11 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error { if err != nil { return err } + // **A hold is looked at again at every cut** (ADR 0418 clause 3): a group whose last member merged or closed + // since is released, and its window runs again from now. + if batches, err = releaseHolds(ctx, inv, batches, now); err != nil { + return err + } keepAll := func(behind string) error { for i := range batches { if err := keepBatch(ctx, inv, &batches[i], now, behind); err != nil { @@ -813,6 +878,41 @@ func cutBatchesHeld(ctx context.Context, open *stores, now time.Time) error { return cutUnrelated(ctx, open, batches, started, waiting, now) } +// releaseHolds asks mesh-delivery again about each group a batch holds, and clears the holds of those with no member +// left to merge. A group mesh-delivery cannot be asked about stays held: the safe error is to wait. It answers the +// batches as kept. +func releaseHolds(ctx context.Context, inv *inventory.Inventory, batches []inventory.Plan, now time.Time) ([]inventory.Plan, error) { + var entries []inventory.Entry + for i := range batches { + b := &batches[i] + if b.Delivery == nil || b.Delivery.Batch == nil || len(b.Delivery.Batch.Holds) == 0 { + continue + } + if entries == nil { + var err error + if entries, err = inv.Catalogued(ctx); err != nil { + return nil, err + } + } + changed := false + for head := range b.Delivery.Batch.Holds { + open, isGroup, _, err := openMembersOf(ctx, inv, deliverySeatHeld(entries), head, "") + if err != nil || (isGroup && len(open) > 0) { + continue + } + holdFor(b, head, "", false, false, now) + changed = true + fmt.Printf("%s: group %s has no member left to merge; its window runs again from now\n", b.ID, head) + } + if changed { + if err := keepBatch(ctx, inv, b, now, ""); err != nil { + return nil, err + } + } + } + return batches, nil +} + // cutUnrelated cuts the closed batches that share nothing with an open walk, each into a walk beside the open // ones, and says the others queued behind the first walk they are related to (novox/hq ADR 0407 decision 4, // issue 973). A batch cut in this look counts as open for the batches after it. @@ -1039,9 +1139,13 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ } } sort.Strings(also) - ordered := groupOrderEdges(members, entries, read, edges) + ordered, err := changeOrderEdges(ctx, merges, members, entries, moved, edges) + if err != nil { + return err + } var withheld []string plan := planOfMoves(moved, append(append([]inventory.Edge{}, edges...), ordered...)) + placedBy(&plan, ordered) // **Withheld, a stopped merge's modules** (novox/hq ADR 0299): a module is built at its branch, which holds the // merge a stop left out of the walk these merges were walked again from — built here only as a dependent, it // would deliver that merge. One a merge of this batch moves is built: that merge carries the stopped change, @@ -1063,6 +1167,7 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ plan.Delivery.Merges = named plan.Delivery.Alone = batch.Delivery != nil && batch.Delivery.Alone plan.Delivery.Withheld = withheld + plan.Delivery.Order = ordered // **Left where it is, a module whose program did not change** (novox/hq issue 721): its check found it the same as // the build the mesh holds. for n, why := range leaveSameAsRunning(ctx, inv, &plan, moved, append(append([]inventory.Edge{}, edges...), ordered...)) { @@ -1094,7 +1199,14 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ } var tiers []string for i, t := range plan.Tiers { - tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(t, ", "))) + var names []string + for _, n := range t { + if m := plan.Modules[n]; m != nil && m.Placed != "" { + n += " " + m.Placed + } + names = append(names, n) + } + tiers = append(tiers, fmt.Sprintf("%d: %s", i, strings.Join(names, ", "))) } var answers []string for _, n := range named { @@ -1122,6 +1234,23 @@ func planBatch(ctx context.Context, open *stores, batch *inventory.Plan, carry [ return nil } +// placedBy writes on each module of a plan the edges of its order that put it after another module of the plan, with the +// rule that drew each (novox/hq ADR 0419 clause 5). +func placedBy(plan *inventory.Plan, ordered []inventory.Edge) { + for _, e := range ordered { + m, in := plan.Modules[e.From] + if _, before := plan.Modules[e.To]; !in || !before { + continue + } + said := "after " + e.To + " (" + e.Why + ")" + if m.Placed == "" { + m.Placed = said + } else if !strings.Contains(m.Placed, said) { + m.Placed += "; " + said + } + } +} + // unchangedSaid names the modules a walk says matched what runs, for its note. func unchangedSaid(unchanged map[string]string) string { names := make([]string, 0, len(unchanged)) @@ -1310,73 +1439,46 @@ type batchMember struct { moved []string } -// afterLines are the `after: ` lines of a pull request's description, as mesh-delivery reads them. -var afterLines = regexp.MustCompile(`(?im)^\s*after:\s*([A-Za-z0-9_.\-/]+)\s*$`) - -// groupOrderEdges is the order of every delivery group among a batch's merges as edges of its walk's tiers -// (ADR 0276 decision 5, ADR 0249 unchanged): merges of two repositories or more from one head branch name are -// a group; each pair its order rules give puts the modules of the member after in a later tier than the -// member before's. A pair in a contradiction orders nothing: its group was refused before it merged. -func groupOrderEdges(members []batchMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository, - edges []inventory.Edge) []inventory.Edge { - byHead := map[string][]batchMember{} +// changeOrderEdges is the order of a batch's moved modules as edges of its plan's tiers (novox/hq ADR 0419): drawn +// per module from the module graph and from what the merges say of the manifests they touch (delivery_edges.go), +// whatever repository the merges are in and whether or not they form a delivery group. No `after:` line is read. +// A cycle is said, never lost: the plan puts the modules left in its last tier, and the composed check fails the +// group before it merges. +func changeOrderEdges(ctx context.Context, merges []inventory.BatchedMerge, members []batchMember, + entries []inventory.Entry, moved []string, edges []inventory.Edge) ([]inventory.Edge, error) { + // What each repository's branch held before the batch: the first parent of its earliest merge in it. + earliest := map[string]inventory.BatchedMerge{} + for _, k := range merges { + key := strings.ToLower(k.Repository) + "\x00" + k.Branch + if e, ok := earliest[key]; !ok || laterMerge(e, k) { + earliest[key] = k + } + } + var sources []changeSource for _, m := range members { - if m.merge.Head != "" { - byHead[m.merge.Head] = append(byHead[m.merge.Head], m) + s := changeSource{Owner: m.merge.Owner, Repo: m.merge.Repo, New: m.merge.Commit, + Paths: m.merge.Paths, Truncated: m.merge.PathsTruncated, Removed: m.merge.Removed} + if e, ok := earliest[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)+"\x00"+m.merge.Base]; ok { + s.Since = e.Commit } + if s.Truncated { + s.Paths, s.Truncated = truncatedPaths(entries, s.Owner, s.Repo, moved), false + } + sources = append(sources, s) } - var out []inventory.Edge - heads := make([]string, 0, len(byHead)) - for h := range byHead { - heads = append(heads, h) + changes, err := changesOf(ctx, sources) + if err != nil { + return nil, fmt.Errorf("what the merges say of their manifests cannot be read, so the order of their modules "+ + "cannot be drawn: %w", err) } - sort.Strings(heads) - for _, h := range heads { - group := byHead[h] - repos := map[string]bool{} - for _, m := range group { - repos[strings.ToLower(m.merge.Owner+"/"+m.merge.Repo)] = true - } - if len(repos) < 2 { - continue - } - var om []orderMember - moved := map[string][]string{} - for _, m := range group { - repository := m.merge.Owner + "/" + m.merge.Repo - id := repository + "@" + m.merge.Commit - var after []string - for _, l := range afterLines.FindAllStringSubmatch(m.merge.Body, -1) { - after = append(after, strings.TrimSuffix(l[1], ".git")) - } - om = append(om, orderMember{ID: id, Repository: repository, Base: m.merge.Base, Head: m.merge.Commit, - Number: m.merge.Number, Paths: m.merge.Paths, PathsTruncated: m.merge.PathsTruncated, - Removed: m.merge.Removed, ModuleDirs: m.merge.ModuleDirs, ModuleDirsSaid: m.merge.ModuleDirsSaid, - CloneURL: m.merge.CloneURL, After: after}) - moved[id] = m.moved - } - order := orderOf(om, reachOfMembers(om, entries, read, edges), edges) - contradicted := map[string]bool{} - for _, c := range order.Contradictions { - for _, id := range c.Members { - contradicted[id] = true - } - } - for _, p := range order.Pairs { - if contradicted[p.Before] && contradicted[p.After] { - continue - } - for _, before := range moved[p.Before] { - for _, after := range moved[p.After] { - if before != after { - out = append(out, inventory.Edge{From: after, To: before, Kind: edgeGroupOrder}) - } - } - } - fmt.Printf(" group %s: %s before %s (%s) inside the walk\n", h, p.Before, p.After, p.Why) - } + order := orderEdges(moved, changes, edges) + for _, e := range order.Edges { + fmt.Printf(" order: %s\n", edgeSaid(e)) } - return out + if said := order.CycleSaid(); said != "" { + fmt.Printf(" %s\n", said) + } + return order.Edges, nil } // searchAfterFailures starts the search a failed walk leaves (ADR 0276 decision 3): every merge it answered diff --git a/cmd/mesh-controller/batches_integration_test.go b/cmd/mesh-controller/batches_integration_test.go new file mode 100644 index 00000000..880e9940 --- /dev/null +++ b/cmd/mesh-controller/batches_integration_test.go @@ -0,0 +1,698 @@ +//go:build integration + +package main + +import ( + "bytes" + "io" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func windowed(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + settings := t.TempDir() + "/merge-window.json" + if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv(mergeWindowFileVar, settings) + noGroups(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, + inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", + Head: "c0"}); err != nil { + t.Fatal(err) + } + for _, m := range []struct{ module, repository, path string }{ + {"app", "novox/mesh-catalog", "modules/app"}, + {"notes", "novox/mesh-catalog", "modules/notes"}, + {"one", "novox/one", ""}, + {"two", "novox/two", ""}, + {"three", "novox/three", ""}, + } { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"}, + inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0", + Head: "c0"}); err != nil { + t.Fatal(err) + } + } + return open +} + +func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) { + t.Helper() + if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil { + t.Fatal(err) + } +} + +func cutAt(t *testing.T, open *stores, now time.Time) { + t.Helper() + if err := cutBatchesHeld(t.Context(), open, now); err != nil { + t.Fatal(err) + } +} + +func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksWithPaths(t) + claude := catalogueMerge("553b7191claude", "app", t0) + dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) + hear(t, open, claude, t0.Add(time.Second)) + hear(t, open, dunst, t0.Add(18*time.Second)) + if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { + t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + cutAt(t, open, t0.Add(18*time.Second+89*time.Second)) + if ws, _ := walks(t, open); len(ws) != 0 { + t.Fatalf("cut before the window closed: %+v", ws) + } + cutAt(t, open, t0.Add(18*time.Second+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 { + t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + w := ws[0] + if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit { + t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits) + } + for _, m := range []string{"app", "notes"} { + if _, in := w.Modules[m]; !in { + t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules) + } + } + want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit}, + {Repository: "novox/mesh-catalog", Commit: dunst.Commit}} + if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { + t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) + } + if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" { + t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked) + } + // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. + hear(t, open, claude, t0.Add(5*time.Minute)) + if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { + t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs)) + } +} + +func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "feat/three") + asksRecorded(t) + for i, r := range []string{"one", "two", "three"} { + hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"), + t0.Add(time.Duration(i)*20*time.Second+time.Second)) + } + cutAt(t, open, t0.Add(41*time.Second+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 { + t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) + } + w := ws[0] + if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 { + t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges) + } + for _, r := range []string{"one", "two", "three"} { + if w.CommitOf("novox/"+r) != "c-"+r { + t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r)) + } + if _, in := w.Modules[r]; !in { + t.Fatalf("the walk does not build %s: %v", r, w.Modules) + } + } + // Each module's ask is recorded at its own repository's commit. + for _, r := range []string{"one", "two", "three"} { + q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build) + if err != nil || !found || q.Commit != "c-"+r { + t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err) + } + } +} + +func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "feat/both", "feat/busy") + asksWithPaths(t) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) + hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second)) + cutAt(t, open, t0.Add(100*time.Second)) + if ws, _ := walks(t, open); len(ws) != 0 { + t.Fatal("the window closed 90 seconds after its first merge, not after its last") + } + cutAt(t, open, t0.Add(170*time.Second)) + if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 { + t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws) + } + + busy := windowed(t) + groupsWaiting(t, "feat/both", "feat/busy") + asksWithPaths(t) + var last time.Time + for i := 0; i < 12; i++ { + last = t0.Add(time.Duration(i) * time.Minute) + repo := []string{"one", "two", "three"}[i%3] + hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last) + if ws, _ := walks(t, busy); len(ws) > 0 { + if i != 10 { + t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i) + } + break + } + } + ws, _ := walks(t, busy) + if len(ws) != 1 { + t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws)) + } +} + +func TestTheAssemblingBatchIsShown(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/553b7191claude") + asksWithPaths(t) + var said []inventory.Plan + was := inventory.PlanSaved + inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) } + t.Cleanup(func() { inventory.PlanSaved = was }) + now := time.Now().UTC() + claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)) + claude.Number, claude.Title = 174, "claude-code: an agent proposes a section" + dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)) + dunst.Number, dunst.Title = 175, "dunst: the font the operator chose" + hear(t, open, claude, now.Add(-19*time.Second)) + hear(t, open, dunst, now.Add(-time.Second)) + hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now) + out := captured(t, func() error { return plansCommand(t.Context(), nil) }) + lines := strings.Split(out, "\n") + first := lines[0] + for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " + + "(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} { + if !strings.Contains(first, want) { + t.Fatalf("plans' first line %q does not say %q", first, want) + } + } + // Under it, one line per repository: the pull request, what it answers, what it moves. + if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+ + "#174 claude-code: an agent proposes a section) · app, notes" || + strings.TrimSpace(lines[2]) != "one a6bc0931 · one" { + t.Fatalf("the grouped list reads %q", lines[1:4]) + } + if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil || + len(said[len(said)-1].Delivery.Merges) != 3 { + t.Fatalf("the batch was not said as assembling with its merges: %+v", said) + } +} + +func captured(t *testing.T, run func() error) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + was := os.Stdout + os.Stdout = w + runErr := run() + os.Stdout = was + _ = w.Close() + var b bytes.Buffer + _, _ = io.Copy(&b, r) + if runErr != nil { + t.Fatal(runErr) + } + return b.String() +} + +func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + hear(t, open, repoMerge("one", "c1", t0), t0) + cutAt(t, open, t0.Add(90*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Open() { + t.Fatalf("the first batch was not cut: %+v", ws) + } + first := ws[0] + hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) + cutAt(t, open, t0.Add(5*time.Minute)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID { + t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs) + } + if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() { + t.Fatalf("the started walk was %s by the next merge", got.State) + } + if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) { + t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute))) + } + first.State = inventory.PlanDone + if err := open.inventory.SavePlan(t.Context(), &first); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(6*time.Minute)) + ws, bs = walks(t, open) + if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { + t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs) + } +} + +func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + ctx := t.Context() + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + hear(t, open, repoMerge("one", "c1", t0), t0) + cutAt(t, open, t0.Add(90*time.Second)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Waiting() { + t.Fatalf("the walk does not wait for its word: %+v", ws) + } + waiting := ws[0] + hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) + cutAt(t, open, t0.Add(2*time.Minute+90*time.Second)) + ws, bs := walks(t, open) + if len(bs) != 0 || len(ws) != 2 { + t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) + } + folded, err := open.inventory.PlanByID(ctx, waiting.ID) + if err != nil { + t.Fatal(err) + } + newer := ws[0] + if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID { + t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy) + } + if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" { + t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges) + } + if _, in := newer.Modules["one"]; !in { + t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules) + } +} + +func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) + hear(t, open, later, t0.Add(18*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + w := ws[0] + w.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + // Moves only notes, which the done walk built from the branch after it: answered there, at once. + hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, w.ID) + if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 || + !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early", + Carried: later.Commit}) { + t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges) + } + // Moves app, which that walk never built: the next batch walks it. + hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute)) + _, bs := walks(t, open) + if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" { + t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs) + } +} + +func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksWithPaths(t) + ctx := t.Context() + // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. + oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) + middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second)) + newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second)) + for i, m := range []link.SourceMoved{oldest, middle, newest} { + hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second)) + } + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + failed := ws[0] + failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0" + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(3*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, failed.ID) + if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 { + t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges) + } + ws, _ = walks(t, open) + alone := ws[0] + if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 || + !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { + t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) + } + if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app", + middle.Commit} { + t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked) + } + // Retried, the failed walk would name merges the search answers now. + if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { + t.Fatalf("a failed walk whose merges are searched was retried: %v", err) + } + alone.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &alone); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(4*time.Minute)) + ws, bs := walks(t, open) + if ws[0].ID != alone.ID || len(bs) != 0 { + t.Fatalf("the search went on after a merge was delivered: %+v", ws[0]) + } + done, _ := open.inventory.PlanByID(ctx, alone.ID) + if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", + Commit: oldest.Commit, Carried: middle.Commit}) { + t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges) + } + // A stopped walk starts no search: a person ended it. + hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute)) + hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second)) + cutAt(t, open, t0.Add(8*time.Minute)) + ws, _ = walks(t, open) + if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(9*time.Minute)) + if again, _ := walks(t, open); again[0].ID != ws[0].ID { + t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0]) + } +} + +func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) { + for _, grouped := range []bool{true, false} { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + for _, m := range []string{"mesh-host"} { + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, + inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + host := repoMerge("mesh-host", "h1", t0) + ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second)) + host.Head, ctl.Head = "feat/together", "feat/together" + if !grouped { + ctl.Head = "feat/alone" + } + hear(t, open, ctl, t0.Add(2*time.Second)) + hear(t, open, host, t0.Add(3*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("%d walks", len(ws)) + } + tiers := ws[0].Tiers + if grouped { + if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) || + !slices.Equal(tiers[1], []string{"mesh-controller"}) { + t.Fatalf("the group's order is not the walk's tiers: %v", tiers) + } + } else if len(tiers) != 1 { + t.Fatalf("two merges of no group were ordered: %v", tiers) + } + } +} + +func TestABatchSurvivesARestart(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "feat/both") + asksWithPaths(t) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) + hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second)) + + // A new controller: nothing of the first one's but the store. + again, err := openStores(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(again.Close) + hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again + cutAt(t, again, t0.Add(119*time.Second)) + if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { + t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs) + } + cutAt(t, again, t0.Add(120*time.Second)) + ws, bs := walks(t, again) + if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { + t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs) + } +} + +func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + c2 := repoMerge("one", "c2", t0.Add(-50*time.Second)) + hear(t, open, c2, t0) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + w := ws[0] + w.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, w.ID) + if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one", + Commit: "c1", Carried: "c2"}) { + t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges) + } + + // A walk of two that built only two: a late merge of one is not its to name. + hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute)) + cutAt(t, open, t0.Add(18*time.Minute)) + ws, _ = walks(t, open) + w = ws[0] + delete(w.Modules, "two") + w.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute)) + if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 { + t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges) + } + if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" { + t.Fatalf("the late merge did not join the next batch: %+v", bs) + } +} + +func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { + open := windowed(t) + asked := asksWithPaths(t) + ctx := t.Context() + hear(t, open, repoMerge("one", "c1", t0), t0) + cutAt(t, open, t0.Add(2*time.Minute)) + started, _ := walks(t, open) + waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0, + State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil { + t.Fatalf("the word was refused beside %s: %v", started[0].ID, err) + } + before := len(*asked) + advanceHeld(ctx, open) + got, _ := open.inventory.PlanByID(ctx, waiting.ID) + if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) { + t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note) + } + // Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3. + later := time.Now().Add(time.Hour) + if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") { + t.Fatalf("a deferred walk reads %q", line) + } + facts, _, err := gatherPlans(ctx, open.inventory, later, nil) + if err != nil { + t.Fatal(err) + } + for _, f := range facts { + if f.id == got.ID { + t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) + } + } + done := started[0] + done.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &done); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + if len(*asked) != before+1 { + t.Fatalf("the walk did not start once the started one ended: asked %v", *asked) + } +} + +func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asked := asksWithPaths(t) + ctx := t.Context() + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0) + hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second)) + hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second)) + _, bs := walks(t, open) + var ownBatch, catalogueBatch inventory.Plan + for _, b := range bs { + if b.OwnPath() { + ownBatch = b + } else { + catalogueBatch = b + } + } + if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 || + len(ownBatch.Delivery.Merges) != 1 { + t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs) + } + if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") { + t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second))) + } + // Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it, + // is cut when the controller's walk ended, and waits for its word with both merges. + cutAt(t, open, t0.Add(2*time.Minute)) + ws, bs := walks(t, open) + if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" { + t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws) + } + ownWalk := ws[0] + for _, m := range []string{"app", "notes"} { + if _, in := ownWalk.Modules[m]; in { + t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m) + } + } + if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() { + t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs) + } + ownWalk.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil { + t.Fatal(err) + } + cutAt(t, open, t0.Add(3*time.Minute)) + ws, bs = walks(t, open) + if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 { + t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs) + } + for _, m := range []string{"app", "notes"} { + if _, in := ws[0].Modules[m]; !in { + t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules) + } + } + for _, a := range *asked { + if a[1] == "modules/app" || a[1] == "modules/notes" { + t.Fatalf("a catalogue module was asked without the word: %v", *asked) + } + } + + // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path + // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. + catalogueWalk := ws[0] + hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) + cutAt(t, open, t0.Add(6*time.Minute)) + ws, _ = walks(t, open) + kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID) + if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() { + t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws) + } + if _, in := ws[0].Modules["app"]; in { + t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules) + } + if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil { + t.Fatal(err) + } + before := len(*asked) + advanceHeld(ctx, open) + if len(*asked) != before { + t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) + } + own2 := ws[0] + own2.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &own2); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" { + t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:]) + } +} + +func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + ctx := t.Context() + for _, m := range []struct { + module string + claims []catalogue.Claim + }{ + {"nats", nil}, + {"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + } { + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano), + Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"}, + ModuleDirsSaid: true} + hear(t, open, mixed, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 || ws[0].Waiting() { + t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws) + } + done := ws[0] + done.State = inventory.PlanDone + if err := open.inventory.SavePlan(ctx, &done); err != nil { + t.Fatal(err) + } + hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute)) + got, _ := open.inventory.PlanByID(ctx, done.ID) + _, bs := walks(t, open) + if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" { + t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)", + got.Delivery.Merges, bs) + } +} diff --git a/cmd/mesh-controller/batches_test.go b/cmd/mesh-controller/batches_test.go index 4c7a4252..da7ad0e9 100644 --- a/cmd/mesh-controller/batches_test.go +++ b/cmd/mesh-controller/batches_test.go @@ -1,10 +1,9 @@ package main import ( - "bytes" "context" "encoding/json" - "io" + "errors" "os" "slices" "strings" @@ -19,42 +18,19 @@ import ( // The suite's tests of a merge, but for the merge window's own (this file's), plan the merge as it is heard: a // window of nothing closes at once, so a merge with no walk open is cut into its walk at once, as every merge // was planned before novox/hq ADR 0276. The window's tests set it through the controller's settings. -func init() { mergeWindowDefault = 0 } +func init() { + mergeWindowDefault = 0 + // No forge in a test: no manifest is found at any commit, so a merge draws no edge of the manifests' change + // (delivery_edges.go). A test that wants some replaces these (forgeHolding). + manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) { + return catalogue.Manifest{}, false, nil + } + parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } +} // windowed is a mesh whose controller's settings give a merge window of 90 seconds and at most 10 minutes, as its // settings file says them, and // modules built from the catalogue (app, notes) and from three repositories of their own. -func windowed(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - settings := t.TempDir() + "/merge-window.json" - if err := os.WriteFile(settings, []byte(`{"merge-window": "90s", "merge-window-at-most": "10m"}`), 0o600); err != nil { - t.Fatal(err) - } - t.Setenv(mergeWindowFileVar, settings) - noGroups(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - for _, m := range []struct{ module, repository, path string }{ - {"app", "novox/mesh-catalog", "modules/app"}, - {"notes", "novox/mesh-catalog", "modules/notes"}, - {"one", "novox/one", ""}, - {"two", "novox/two", ""}, - {"three", "novox/three", ""}, - } { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1"}, - inventory.Source{Repository: m.repository, Seat: "git", Path: m.path, Ref: "main", BuiltFrom: "c0", - Head: "c0"}); err != nil { - t.Fatal(err) - } - } - return open -} // t0 is the moment a test's clock starts: merges are dated by it, and heard after it. var t0 = time.Now().UTC().Truncate(time.Second) @@ -83,15 +59,16 @@ func noGroups(t *testing.T) { t.Cleanup(func() { deliveryGroupOf = was }) } -// groupsWaiting stubs mesh-delivery's word on groups for a test: each head named is a group with a member of another -// repository whose pull request is open (novox/elsewhere, ready), so a merge from it waits for the window (ADR 0306 -// decision 2); every other head names no group. +// groupsWaiting stubs mesh-delivery's word on groups for a test of the merge window's own timing: each head named +// is one whose group mesh-delivery cannot say, so a merge from it waits for the window (ADR 0306 decision 2) and +// nothing holds the batch open; every other head names no group. A group with a member still to merge, which holds +// its batch open however long it takes (ADR 0418 clause 3), is groupStates'. func groupsWaiting(t *testing.T, heads ...string) { t.Helper() was := deliveryGroupOf deliveryGroupOf = func(_ context.Context, _ bool, head string) ([]groupMember, bool, error) { if slices.Contains(heads, head) { - return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil + return nil, false, errors.New("the test's mesh-delivery does not say") } return nil, false, nil } @@ -99,12 +76,37 @@ func groupsWaiting(t *testing.T, heads ...string) { } // everyHeadWaits stubs mesh-delivery's word on groups for a test whose merges each wait for the window as they did -// before novox/hq ADR 0304: every head is a group with a member of another repository still to merge. +// before novox/hq ADR 0304: mesh-delivery cannot say any head's group. func everyHeadWaits(t *testing.T) { t.Helper() was := deliveryGroupOf deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { - return []groupMember{{Repository: "novox/elsewhere", State: "ready"}}, true, nil + return nil, false, errors.New("the test's mesh-delivery does not say") + } + t.Cleanup(func() { deliveryGroupOf = was }) +} + +// groupStates stubs mesh-delivery's word on one group: its members and their states, which a test changes as the +// members merge. +func groupStates(t *testing.T, head string, states map[string]string) { + t.Helper() + groupStatesFrom(t, head, states, true) +} + +// groupStatesFrom is groupStates for a mesh-delivery that does (composed) or does not yet say the group's gate is the +// composed check's: only the first holds the group's batch open for its last member. +func groupStatesFrom(t *testing.T, head string, states map[string]string, composed bool) { + t.Helper() + was := deliveryGroupOf + deliveryGroupOf = func(_ context.Context, _ bool, asked string) ([]groupMember, bool, error) { + if asked != head { + return nil, false, nil + } + var members []groupMember + for repository, state := range states { + members = append(members, groupMember{Repository: repository, State: state, Composed: composed}) + } + return members, true, nil } t.Cleanup(func() { deliveryGroupOf = was }) } @@ -124,20 +126,8 @@ func lone(m link.SourceMoved) link.SourceMoved { } // hear is a merge heard at a moment. -func hear(t *testing.T, open *stores, m link.SourceMoved, now time.Time) { - t.Helper() - if err := (following{open: open}).hearMerge(t.Context(), m, now); err != nil { - t.Fatal(err) - } -} // cutAt is the cutter looking at a moment. -func cutAt(t *testing.T, open *stores, now time.Time) { - t.Helper() - if err := cutBatchesHeld(t.Context(), open, now); err != nil { - t.Fatal(err) - } -} // walks is every plan record that is a walk, newest first, and the batches not yet cut. func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) { @@ -158,426 +148,34 @@ func walks(t *testing.T, open *stores) (walks, batches []inventory.Plan) { // The replay of issue 362: two catalogue merges 18 seconds apart are one batch, cut once into one walk at the // later commit, which names both merges — the earlier carried by the later. -func TestTwoMergesSecondsApartAreOneWalkAtTheLaterCommit(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - claude := catalogueMerge("553b7191claude", "app", t0) - dunst := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, claude, t0.Add(time.Second)) - hear(t, open, dunst, t0.Add(18*time.Second)) - if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { - t.Fatalf("within the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - cutAt(t, open, t0.Add(18*time.Second+89*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatalf("cut before the window closed: %+v", ws) - } - cutAt(t, open, t0.Add(18*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("after the window: %d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if w.Commit != dunst.Commit || len(w.Commits) != 1 || w.Commits[0].Commit != dunst.Commit { - t.Fatalf("the walk is at %s %+v, not the later commit", w.Commit, w.Commits) - } - for _, m := range []string{"app", "notes"} { - if _, in := w.Modules[m]; !in { - t.Fatalf("the walk does not build %s, which one of its merges moved: %v", m, w.Modules) - } - } - want := []inventory.PlanMerge{{Repository: "novox/mesh-catalog", Commit: claude.Commit, Carried: dunst.Commit}, - {Repository: "novox/mesh-catalog", Commit: dunst.Commit}} - if w.Delivery == nil || !slices.EqualFunc(w.Delivery.Merges, want, sameMerge) { - t.Fatalf("the walk answers %+v, want %+v", w.Delivery, want) - } - if len(*asked) != 2 || (*asked)[0][2] != "main" || (*asked)[1][2] != "main" { - t.Fatalf("the walk did not ask its modules at the branch, which holds the commit it carries: %v", *asked) - } - // Heard again, as the bus may hand it over twice: never a second merge, nor a second walk. - hear(t, open, claude, t0.Add(5*time.Minute)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("a merge heard twice made %d walk(s) and %d batch(es)", len(ws), len(bs)) - } -} // Merges of three repositories in one window are one walk, one commit for each. -func TestThreeRepositoriesInOneWindowAreOneWalk(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/three") - asksRecorded(t) - for i, r := range []string{"one", "two", "three"} { - hear(t, open, fromHead(repoMerge(r, "c-"+r, t0.Add(time.Duration(i)*20*time.Second)), "feat/three"), - t0.Add(time.Duration(i)*20*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(41*time.Second+90*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - w := ws[0] - if len(w.Commits) != 3 || len(w.Delivery.Merges) != 3 { - t.Fatalf("the walk carries %+v and answers %+v", w.Commits, w.Delivery.Merges) - } - for _, r := range []string{"one", "two", "three"} { - if w.CommitOf("novox/"+r) != "c-"+r { - t.Fatalf("the walk carries %s at %q", r, w.CommitOf("novox/"+r)) - } - if _, in := w.Modules[r]; !in { - t.Fatalf("the walk does not build %s: %v", r, w.Modules) - } - } - // Each module's ask is recorded at its own repository's commit. - for _, r := range []string{"one", "two", "three"} { - q, found, err := open.inventory.BuildRequestByID(t.Context(), w.Modules[r].Build) - if err != nil || !found || q.Commit != "c-"+r { - t.Fatalf("%s's build was not recorded at its own commit: %+v %v %v", r, q, found, err) - } - } -} // Each merge restarts the window: a merge at second 80 keeps the batch open until second 170; the window // closes at most ten minutes after its first merge however busy. -func TestAMergeRestartsTheWindowAndTheMaximumClosesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(80*time.Second)), "feat/both"), t0.Add(80*time.Second)) - cutAt(t, open, t0.Add(100*time.Second)) - if ws, _ := walks(t, open); len(ws) != 0 { - t.Fatal("the window closed 90 seconds after its first merge, not after its last") - } - cutAt(t, open, t0.Add(170*time.Second)) - if ws, _ := walks(t, open); len(ws) != 1 || len(ws[0].Commits) != 2 { - t.Fatalf("the window did not close 90 seconds after its last merge: %+v", ws) - } - - busy := windowed(t) - groupsWaiting(t, "feat/both", "feat/busy") - asksWithPaths(t) - var last time.Time - for i := 0; i < 12; i++ { - last = t0.Add(time.Duration(i) * time.Minute) - repo := []string{"one", "two", "three"}[i%3] - hear(t, busy, fromHead(repoMerge(repo, "c"+string(rune('a'+i)), last), "feat/busy"), last) - if ws, _ := walks(t, busy); len(ws) > 0 { - if i != 10 { - t.Fatalf("a busy window closed at merge %d (minute %d), not at its maximum", i, i) - } - break - } - } - ws, _ := walks(t, busy) - if len(ws) != 1 { - t.Fatalf("ten minutes of merges a minute apart were never cut: %d walk(s)", len(ws)) - } -} // `plans` lists the batch being assembled first, in the operator's words: how long is left, when at the latest, // what is grouped, and that the plan is not yet calculated. It says so on the bus as plan-moved, too. -func TestTheAssemblingBatchIsShown(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/553b7191claude") - asksWithPaths(t) - var said []inventory.Plan - was := inventory.PlanSaved - inventory.PlanSaved = func(_ *inventory.Inventory, p inventory.Plan) { said = append(said, p) } - t.Cleanup(func() { inventory.PlanSaved = was }) - now := time.Now().UTC() - claude := catalogueMerge("553b7191claude", "app", now.Add(-20*time.Second)) - claude.Number, claude.Title = 174, "claude-code: an agent proposes a section" - dunst := catalogueMerge("48bda475dunst", "notes", now.Add(-2*time.Second)) - dunst.Number, dunst.Title = 175, "dunst: the font the operator chose" - hear(t, open, claude, now.Add(-19*time.Second)) - hear(t, open, dunst, now.Add(-time.Second)) - hear(t, open, fromHead(repoMerge("one", "a6bc0931one", now), dunst.Head), now) - out := captured(t, func() error { return plansCommand(t.Context(), nil) }) - lines := strings.Split(out, "\n") - first := lines[0] - for _, want := range []string{"assembling: ", " s left (at the latest ", "grouped: novox/mesh-catalog@48bda475 " + - "(answers 553b7191), novox/one@a6bc0931; plan not yet calculated"} { - if !strings.Contains(first, want) { - t.Fatalf("plans' first line %q does not say %q", first, want) - } - } - // Under it, one line per repository: the pull request, what it answers, what it moves. - if len(lines) < 3 || strings.TrimSpace(lines[1]) != "mesh-catalog #175 dunst: the font the operator chose (answers "+ - "#174 claude-code: an agent proposes a section) · app, notes" || - strings.TrimSpace(lines[2]) != "one a6bc0931 · one" { - t.Fatalf("the grouped list reads %q", lines[1:4]) - } - if len(said) == 0 || said[len(said)-1].State != inventory.PlanAssembling || said[len(said)-1].Delivery.Batch == nil || - len(said[len(said)-1].Delivery.Merges) != 3 { - t.Fatalf("the batch was not said as assembling with its merges: %+v", said) - } -} // captured is what a command printed. -func captured(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - was := os.Stdout - os.Stdout = w - runErr := run() - os.Stdout = was - _ = w.Close() - var b bytes.Buffer - _, _ = io.Copy(&b, r) - if runErr != nil { - t.Fatal(runErr) - } - return b.String() -} // A merge heard during a walk it is related to (the same repository, issue 973) joins the next batch, which is // cut when the walk ends; the walk that started is never superseded (ADR 0276 decision 4, ADR 0407 decision 4). -func TestAMergeDuringAWalkJoinsTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("the first batch was not cut: %+v", ws) - } - first := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(5*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != first.ID { - t.Fatalf("the merge during the walk: %d walk(s), batches %+v", len(ws), bs) - } - if got, _ := open.inventory.PlanByID(t.Context(), first.ID); !got.Open() { - t.Fatalf("the started walk was %s by the next merge", got.State) - } - if !strings.HasPrefix(batchWords(bs[0], t0.Add(5*time.Minute)), "queued behind "+first.ID) { - t.Fatalf("a queued batch reads %q", batchWords(bs[0], t0.Add(5*time.Minute))) - } - first.State = inventory.PlanDone - if err := open.inventory.SavePlan(t.Context(), &first); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(6*time.Minute)) - ws, bs = walks(t, open) - if len(ws) != 2 || len(bs) != 0 || ws[0].Commit != "c2" { - t.Fatalf("the queued batch was not cut when the walk ended: %+v %+v", ws, bs) - } -} // A walk waiting for its delivery's word is folded into the next batch's walk, which answers its merges; it // names the walk that took it over. -func TestAWaitingWalkIsFoldedIntoTheNextBatch(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(90*time.Second)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Waiting() { - t.Fatalf("the walk does not wait for its word: %+v", ws) - } - waiting := ws[0] - hear(t, open, repoMerge("one", "c2", t0.Add(2*time.Minute)), t0.Add(2*time.Minute)) - cutAt(t, open, t0.Add(2*time.Minute+90*time.Second)) - ws, bs := walks(t, open) - if len(bs) != 0 || len(ws) != 2 { - t.Fatalf("%d walk(s), %d batch(es)", len(ws), len(bs)) - } - folded, err := open.inventory.PlanByID(ctx, waiting.ID) - if err != nil { - t.Fatal(err) - } - newer := ws[0] - if folded.State != inventory.PlanSuperseded || folded.Delivery.TakenOverBy != newer.ID { - t.Fatalf("the waiting walk is %s, taken over by %q", folded.State, folded.Delivery.TakenOverBy) - } - if len(newer.Delivery.Merges) != 2 || newer.CommitOf("novox/one") != "c2" { - t.Fatalf("the newer walk answers %+v", newer.Delivery.Merges) - } - if _, in := newer.Modules["one"]; !in { - t.Fatalf("what the folded walk was to build is not built: %v", newer.Modules) - } -} // A merge heard after a later merge of its branch was walked is answered by that walk when it builds all it // moves — named at once on a walk done — and joins the next batch when it does not. -func TestALateMergeIsAnsweredByTheWalkOfTheLaterOne(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - later := catalogueMerge("48bda475dunst", "notes", t0.Add(17*time.Second)) - hear(t, open, later, t0.Add(18*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - // Moves only notes, which the done walk built from the branch after it: answered there, at once. - hear(t, open, catalogueMerge("553b7191early", "notes", t0), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if got.State != inventory.PlanDone || len(got.Delivery.Merges) != 2 || - !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: "553b7191early", - Carried: later.Commit}) { - t.Fatalf("the late merge is not named by the walk that carried it: %+v", got.Delivery.Merges) - } - // Moves app, which that walk never built: the next batch walks it. - hear(t, open, catalogueMerge("1111aaaaapp", "app", t0.Add(time.Second)), t0.Add(16*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "1111aaaaapp" { - t.Fatalf("a late merge moving what the walk never built did not join the next batch: %+v", bs) - } -} // A failed walk marks nothing delivered: its earlier merges are walked alone, on their own commit, the newest // first; the first delivered answers the older ones, and the search stops. -func TestAFailedWalkWalksItsEarlierMergesAlone(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - ctx := t.Context() - // Made before the cut marks their modules seen, as merges are: walked again, they are news all the same. - oldest := catalogueMerge("aaaa0001", "app", t0.Add(-3*time.Minute)) - middle := catalogueMerge("bbbb0002", "app", t0.Add(-3*time.Minute+10*time.Second)) - newest := catalogueMerge("cccc0003", "notes", t0.Add(-3*time.Minute+20*time.Second)) - for i, m := range []link.SourceMoved{oldest, middle, newest} { - hear(t, open, m, t0.Add(time.Duration(i)*10*time.Second+time.Second)) - } - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - failed := ws[0] - failed.State, failed.Note = inventory.PlanFailed, "notes failed to build in tier 0" - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, failed.ID) - if got.State != inventory.PlanFailed || len(got.Delivery.Merges) != 3 { - t.Fatalf("the failed walk's record changed: %s %+v", got.State, got.Delivery.Merges) - } - ws, _ = walks(t, open) - alone := ws[0] - if alone.ID == failed.ID || alone.Commit != middle.Commit || len(alone.Delivery.Merges) != 1 || - !sameMerge(alone.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", Commit: middle.Commit}) { - t.Fatalf("the newest earlier merge was not walked alone on its own commit: %+v", alone) - } - if _, in := alone.Modules["app"]; !in || (*asked)[len(*asked)-1] != [3]string{"novox/mesh-catalog", "modules/app", - middle.Commit} { - t.Fatalf("the merge walked alone does not build app at its own commit: %v, asked %v", alone.Modules, *asked) - } - // Retried, the failed walk would name merges the search answers now. - if _, err := retryPlan(ctx, open, failed.ID); err == nil || !strings.Contains(err.Error(), "walked alone") { - t.Fatalf("a failed walk whose merges are searched was retried: %v", err) - } - alone.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &alone); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(4*time.Minute)) - ws, bs := walks(t, open) - if ws[0].ID != alone.ID || len(bs) != 0 { - t.Fatalf("the search went on after a merge was delivered: %+v", ws[0]) - } - done, _ := open.inventory.PlanByID(ctx, alone.ID) - if len(done.Delivery.Merges) != 2 || !sameMerge(done.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/mesh-catalog", - Commit: oldest.Commit, Carried: middle.Commit}) { - t.Fatalf("the oldest merge is not answered by the walk that delivered the one after it: %+v", done.Delivery.Merges) - } - // A stopped walk starts no search: a person ended it. - hear(t, open, catalogueMerge("dddd0004", "app", t0.Add(5*time.Minute)), t0.Add(5*time.Minute)) - hear(t, open, catalogueMerge("eeee0005", "notes", t0.Add(5*time.Minute+time.Second)), t0.Add(5*time.Minute+time.Second)) - cutAt(t, open, t0.Add(8*time.Minute)) - ws, _ = walks(t, open) - if _, err := stopWalk(ctx, open.inventory, ws[0].ID, "mesh-delivery for jochen", "not now"); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(9*time.Minute)) - if again, _ := walks(t, open); again[0].ID != ws[0].ID { - t.Fatalf("a stopped walk's earlier merge was walked alone: %+v", again[0]) - } -} // A delivery group's order holds inside the walk (ADR 0249 unchanged): merges of two repositories from one head // branch name, the node-engine's before the controller's, are tiered so; without the group, one tier. -func TestAGroupsOrderBecomesTiersInsideTheWalk(t *testing.T) { - for _, grouped := range []bool{true, false} { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []string{"mesh-host"} { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: "1"}, - inventory.Source{Repository: "novox/" + m, Seat: "git", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - host := repoMerge("mesh-host", "h1", t0) - ctl := repoMerge("mesh-controller", "k1", t0.Add(time.Second)) - host.Head, ctl.Head = "feat/together", "feat/together" - if !grouped { - ctl.Head = "feat/alone" - } - hear(t, open, ctl, t0.Add(2*time.Second)) - hear(t, open, host, t0.Add(3*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("%d walks", len(ws)) - } - tiers := ws[0].Tiers - if grouped { - if len(tiers) != 2 || !slices.Equal(tiers[0], []string{"mesh-host"}) || - !slices.Equal(tiers[1], []string{"mesh-controller"}) { - t.Fatalf("the group's order is not the walk's tiers: %v", tiers) - } - } else if len(tiers) != 1 { - t.Fatalf("two merges of no group were ordered: %v", tiers) - } - } -} // A restarted controller resumes the window where it stood: the batch, its merges and their times are in the // store, a merge handed over again is not doubled, and the batch is cut when its window closes. -func TestABatchSurvivesARestart(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "feat/both") - asksWithPaths(t) - hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0) - hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(30*time.Second)), "feat/both"), t0.Add(30*time.Second)) - - // A new controller: nothing of the first one's but the store. - again, err := openStores(t.Context()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(again.Close) - hear(t, again, fromHead(repoMerge("one", "c1", t0), "feat/both"), t0.Add(60*time.Second)) // the bus hands it over again - cutAt(t, again, t0.Add(119*time.Second)) - if ws, bs := walks(t, again); len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller's batch: %d walk(s), %+v", len(ws), bs) - } - cutAt(t, again, t0.Add(120*time.Second)) - ws, bs := walks(t, again) - if len(ws) != 1 || len(bs) != 0 || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the restarted controller did not cut the batch at its window: %+v %+v", ws, bs) - } -} // The window's settings reach the controller in its settings file, read at every look; one that says no // duration, or no file, is the default. @@ -622,94 +220,9 @@ func TestAWaitingWalkNamesTheMergesItAnswers(t *testing.T) { // A merge heard after a later merge of its branch was cut is named by that walk however its modules read since // the cut, in a repository of one module (review of this change); a walk that never built what it moves names // nothing, and the merge joins the next batch. -func TestALateMergeOfAOneModuleRepositoryIsNamed(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - c2 := repoMerge("one", "c2", t0.Add(-50*time.Second)) - hear(t, open, c2, t0) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - w := ws[0] - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("one", "c1", t0.Add(-60*time.Second)), t0.Add(15*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, w.ID) - if len(got.Delivery.Merges) != 2 || !sameMerge(got.Delivery.Merges[0], inventory.PlanMerge{Repository: "novox/one", - Commit: "c1", Carried: "c2"}) { - t.Fatalf("the late merge was not named by the walk that carried it: %+v", got.Delivery.Merges) - } - - // A walk of two that built only two: a late merge of one is not its to name. - hear(t, open, repoMerge("two", "d2", t0.Add(16*time.Minute)), t0.Add(16*time.Minute)) - cutAt(t, open, t0.Add(18*time.Minute)) - ws, _ = walks(t, open) - w = ws[0] - delete(w.Modules, "two") - w.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - hear(t, open, repoMerge("two", "d1", t0.Add(15*time.Minute)), t0.Add(20*time.Minute)) - if got, _ := open.inventory.PlanByID(ctx, w.ID); len(got.Delivery.Merges) != 1 { - t.Fatalf("a walk that never built what the late merge moves named it: %+v", got.Delivery.Merges) - } - if _, bs := walks(t, open); len(bs) != 1 || bs[0].Delivery.Merges[0].Commit != "d1" { - t.Fatalf("the late merge did not join the next batch: %+v", bs) - } -} // A related walk waits for the delivery's word too: a waiting walk let go beside a started one it is related to // (the same repository, issue 973) takes the word and starts once that one ended, asking nothing before. -func TestAWalkLetGoBesideAStartedOneStartsOnceItEnded(t *testing.T) { - open := windowed(t) - asked := asksWithPaths(t) - ctx := t.Context() - hear(t, open, repoMerge("one", "c1", t0), t0) - cutAt(t, open, t0.Add(2*time.Minute)) - started, _ := walks(t, open) - waiting := inventory.Plan{ID: "plan-waiting", Repository: "novox/one", Branch: "main", Commit: "d1", Created: t0, - State: inventory.PlanBuilding, Tiers: [][]string{{"one"}}, Modules: map[string]*inventory.PlanModule{"one": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if _, err := letGo(ctx, open.inventory, waiting.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatalf("the word was refused beside %s: %v", started[0].ID, err) - } - before := len(*asked) - advanceHeld(ctx, open) - got, _ := open.inventory.PlanByID(ctx, waiting.ID) - if len(*asked) != before || !strings.Contains(got.Note, "starts once "+started[0].ID) { - t.Fatalf("a walk let go beside a started one asked (%d → %d) or does not say it waits: %q", before, len(*asked), got.Note) - } - // Read as a wait, an hour on: its note on the line, never LATE, and no tier of it late for S3. - later := time.Now().Add(time.Hour) - if line := planLine(got, later); !strings.Contains(line, "starts once "+started[0].ID) || strings.Contains(line, "LATE") { - t.Fatalf("a deferred walk reads %q", line) - } - facts, _, err := gatherPlans(ctx, open.inventory, later, nil) - if err != nil { - t.Fatal(err) - } - for _, f := range facts { - if f.id == got.ID { - t.Fatalf("a deferred walk is watched as a tier running late: %+v", f) - } - } - done := started[0] - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) != before+1 { - t.Fatalf("the walk did not start once the started one ended: asked %v", *asked) - } -} // A file a merge of the batch removed and a later one brought back is not removed; one removed last is. func TestARemovedFileIsTheLastMergesWord(t *testing.T) { @@ -733,106 +246,6 @@ func TestARemovedFileIsTheLastMergesWord(t *testing.T) { // (decided during the build, 2026-10-10): the two are batches of their own, cut one after the other; a walk on // the own path folds no waiting catalogue walk but batches its merges again behind it; the catalogue's walk // still waits for the word, so no catalogue delivery skips its turn. -func TestAMergeOnTheControllersPathNeverSharesABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asked := asksWithPaths(t) - ctx := t.Context() - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("aaaa0001", "app", t0), t0) - hear(t, open, repoMerge("mesh-controller", "k1", t0.Add(10*time.Second)), t0.Add(10*time.Second)) - hear(t, open, catalogueMerge("bbbb0002", "notes", t0.Add(20*time.Second)), t0.Add(20*time.Second)) - _, bs := walks(t, open) - var ownBatch, catalogueBatch inventory.Plan - for _, b := range bs { - if b.OwnPath() { - ownBatch = b - } else { - catalogueBatch = b - } - } - if len(bs) != 2 || ownBatch.ID == "" || catalogueBatch.ID == "" || len(catalogueBatch.Delivery.Merges) != 2 || - len(ownBatch.Delivery.Merges) != 1 { - t.Fatalf("a controller merge and two catalogue merges in one window: %+v", bs) - } - if !strings.Contains(batchWords(ownBatch, t0.Add(30*time.Second)), "own path") { - t.Fatalf("the own-path batch does not say so: %q", batchWords(ownBatch, t0.Add(30*time.Second))) - } - // Both windows closed, the controller's batch is cut first and starts; the catalogue batch queues behind it, - // is cut when the controller's walk ended, and waits for its word with both merges. - cutAt(t, open, t0.Add(2*time.Minute)) - ws, bs := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k1" { - t.Fatalf("the controller's batch was not cut first into a started walk: %+v", ws) - } - ownWalk := ws[0] - for _, m := range []string{"app", "notes"} { - if _, in := ownWalk.Modules[m]; in { - t.Fatalf("the controller's walk builds %s, a catalogue module: it skipped its turn", m) - } - } - if len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != ownWalk.ID || bs[0].OwnPath() { - t.Fatalf("the catalogue batch does not queue behind the controller's walk: %+v", bs) - } - ownWalk.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &ownWalk); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(3*time.Minute)) - ws, bs = walks(t, open) - if len(bs) != 0 || ws[0].ID != catalogueBatch.ID || !ws[0].Waiting() || len(ws[0].Delivery.Merges) != 2 { - t.Fatalf("the catalogue batch was not cut into a waiting walk once the controller's ended: %+v %+v", ws, bs) - } - for _, m := range []string{"app", "notes"} { - if _, in := ws[0].Modules[m]; !in { - t.Fatalf("the catalogue walk does not build %s: %v", m, ws[0].Modules) - } - } - for _, a := range *asked { - if a[1] == "modules/app" || a[1] == "modules/notes" { - t.Fatalf("a catalogue module was asked without the word: %v", *asked) - } - } - - // A catalogue walk waiting for its word when an own-path batch is cut keeps waiting beside the own-path - // walk, is not folded into it, and its word is taken meanwhile: it starts once the own-path walk ended. - catalogueWalk := ws[0] - hear(t, open, repoMerge("mesh-controller", "k2", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - kept, _ := open.inventory.PlanByID(ctx, catalogueWalk.ID) - if !kept.Waiting() || ws[0].CommitOf("novox/mesh-controller") != "k2" || ws[0].Waiting() { - t.Fatalf("the waiting catalogue walk was not kept waiting beside the controller's walk: %s %+v", kept.State, ws) - } - if _, in := ws[0].Modules["app"]; in { - t.Fatalf("the controller's walk folded the waiting catalogue walk in: %v", ws[0].Modules) - } - if _, err := letGo(ctx, open.inventory, catalogueWalk.ID, catalogue.DeliverySeat, "its turn"); err != nil { - t.Fatal(err) - } - before := len(*asked) - advanceHeld(ctx, open) - if len(*asked) != before { - t.Fatalf("the catalogue walk started beside the controller's: asked %v", (*asked)[before:]) - } - own2 := ws[0] - own2.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &own2); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - if len(*asked) < before+1 || (*asked)[before][1] != "modules/app" { - t.Fatalf("the catalogue walk did not start once the controller's ended: %v", (*asked)[before:]) - } -} // `plans` names a walk by what it moves (asked by the operator, 2026-10-10): the repository's pull request and // title, the modules it moves and the machines running them, then the state words; a record naming no pull @@ -878,46 +291,3 @@ func sameMerge(a, b inventory.PlanMerge) bool { // A catalogue merge heard after a later merge of its branch was walked without the word (a merge that touched // the bus too, on the controller's own path) is not answered by that walk while the delivery seat has a holder: // its delivery would skip its turn. It joins the next catalogue batch. -func TestALateCatalogueMergeIsNotAnsweredByAnOwnPathWalk(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - ctx := t.Context() - for _, m := range []struct { - module string - claims []catalogue.Claim - }{ - {"nats", nil}, - {"mesh-delivery", []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, - } { - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: m.module, Version: "1", Claims: m.claims}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m.module, Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - mixed := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "m1xed", MergedAt: t0.Format(time.RFC3339Nano), - Paths: []string{"modules/nats/module.json", "modules/app/module.json"}, ModuleDirs: []string{"modules/nats", "modules/app"}, - ModuleDirsSaid: true} - hear(t, open, mixed, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || ws[0].Waiting() { - t.Fatalf("the mixed merge's walk waited, or was not cut: %+v", ws) - } - done := ws[0] - done.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - hear(t, open, catalogueMerge("ear1ier", "app", t0.Add(-30*time.Second)), t0.Add(3*time.Minute)) - got, _ := open.inventory.PlanByID(ctx, done.ID) - _, bs := walks(t, open) - if len(got.Delivery.Merges) != 1 || len(bs) != 1 || bs[0].OwnPath() || bs[0].Delivery.Merges[0].Commit != "ear1ier" { - t.Fatalf("the late catalogue merge was answered by the own-path walk (%+v) rather than the next catalogue batch (%+v)", - got.Delivery.Merges, bs) - } -} diff --git a/cmd/mesh-controller/batches_watch.go b/cmd/mesh-controller/batches_watch.go index 9881c19e..ae933763 100644 --- a/cmd/mesh-controller/batches_watch.go +++ b/cmd/mesh-controller/batches_watch.go @@ -3,6 +3,8 @@ package main import ( "context" "fmt" + "sort" + "strings" "time" "github.com/novox/mesh-controller/internal/conditions" @@ -17,6 +19,9 @@ import ( const ( kindBatchNotCut = "batch-not-cut" kindBatchBehindWalk = "batch-behind-walk" + // kindGroupNotMerged is raised for a group whose last member has not merged by the merge window's maximum after + // its first (novox/hq ADR 0418 clause 3). + kindGroupNotMerged = "group-not-merged" ) // batchFacts is one batch not yet cut, as the watchdogs read it. @@ -28,6 +33,11 @@ type batchFacts struct { // tiers. behind string walkBound time.Duration + // holds are the groups the batch is held for, by head, with why; created is when the batch was made, the first + // merge of the group heard; atMostWindow is the maximum a window may take (the controller's setting). + holds map[string]string + created time.Time + atMostWindow time.Duration } // gatherBatches is every batch not yet cut, with the bound of the walk it waits behind. @@ -42,8 +52,10 @@ func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) } var out []batchFacts for _, b := range batches { - f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b)} + _, atMostWindow := mergeWindowOf() + f := batchFacts{id: b.ID, state: b.State, grouped: groupedWords(b), created: b.Created, atMostWindow: atMostWindow} if w := b.Delivery; w != nil && w.Batch != nil { + f.holds = w.Batch.Holds f.atMost, f.closed, f.behind = w.Batch.AtMost, w.Batch.ClosesAt, w.Batch.Behind if f.atMost.Before(f.closed) { f.closed = f.atMost @@ -67,7 +79,8 @@ func gatherBatches(ctx context.Context, inv *inventory.Inventory, now time.Time) func watchBatchesNotCut(f *signalFacts) []conditions.Observation { var out []conditions.Observation for _, b := range f.batches { - if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter { + if b.state != inventory.PlanAssembling || b.atMost.IsZero() || f.now.Sub(b.atMost) <= batchLateAfter || + len(b.holds) > 0 { // a batch held for a group waits on a member, which has its own condition continue } late := f.now.Sub(b.atMost) @@ -105,3 +118,36 @@ func watchBatchesBehind(f *signalFacts) []conditions.Observation { } return out } + +// watchGroupsNotMerged is the condition of a batch held for a group whose last member has not merged by the merge +// window's maximum after the first member merged, naming the member and why (novox/hq ADR 0418 clause 3): nothing of +// the group is delivered while it waits. +func watchGroupsNotMerged(f *signalFacts) []conditions.Observation { + var out []conditions.Observation + for _, b := range f.batches { + if len(b.holds) == 0 || b.created.IsZero() || f.now.Sub(b.created) <= b.atMostWindow { + continue + } + heads := make([]string, 0, len(b.holds)) + for h := range b.holds { + heads = append(heads, h) + } + sort.Strings(heads) + var why []string + for _, h := range heads { + why = append(why, b.holds[h]) + } + waited := f.now.Sub(b.created) + out = append(out, conditions.Observation{Scope: conditions.ScopePlan, ID: b.id, + Kind: kindGroupNotMerged, Severity: conditions.Warning, + Summary: fmt.Sprintf("%s; its first member merged %s ago, past the merge window's maximum (%s), and nothing "+ + "of the group is delivered while it waits (batch %s)", strings.Join(why, "; "), ago(waited), ago(b.atMostWindow), b.id), + Said: "a group waits for a member that has not merged", + Headline: "A change waits for the rest of its group to merge", + Explanation: fmt.Sprintf("The changes of a group are delivered together, once the last one is merged. "+ + "One has not merged in %s. Nothing is lost; the merged ones are kept and delivered with it.", + humanDuration(waited)), + Resolved: "The group's last change merged"}) + } + return out +} diff --git a/cmd/mesh-controller/beside_test.go b/cmd/mesh-controller/beside_test.go index a4c0237d..0d417830 100644 --- a/cmd/mesh-controller/beside_test.go +++ b/cmd/mesh-controller/beside_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/bindings_integration_test.go b/cmd/mesh-controller/bindings_integration_test.go new file mode 100644 index 00000000..580c1fde --- /dev/null +++ b/cmd/mesh-controller/bindings_integration_test.go @@ -0,0 +1,102 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" +) + +func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + for _, m := range storeManifests() { + register(t, open, m) + } + assignAll := func(pairs ...[2]string) { + for _, a := range pairs { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + } + // The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each. + assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) + for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { + if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { + t.Fatal(err) + } + } + assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, + [2]string{"laptop", "board"}) + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData { + t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData) + } + if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData { + t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData) + } + + // Sent, and recorded: only the binding to data. + bindings := boundToData(plan, nil) + if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) { + t.Fatalf("recorded %+v", bindings) + } + if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil { + t.Fatal(err) + } + if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { + t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err) + } + + // The laptop's store taken away: refused, not moved to the anchor's empty one. + if err := inv.Unassign(ctx, "laptop", "store"); err != nil { + t.Fatal(err) + } + if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) || + !strings.Contains(err.Error(), "board on laptop is bound to laptop/store") || + !strings.Contains(err.Error(), "pin laptop postgres-database anchor store") { + t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err) + } + + // A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was. + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { + t.Fatal(err) + } + plan, _, err = planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { + t.Fatalf("pinned to the anchor and bound to %s", n.From) + } + found, err := probeBindings(ctx, &doctor{open: open}) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning || + !strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") { + t.Fatalf("a pinned move is not said before it is sent: %+v", found) + } + if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { + t.Fatal(err) + } + all, err := inv.Bindings(ctx) + if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" { + t.Fatalf("%+v, %v", all, err) + } + if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { + t.Fatalf("a move sent is still said: %+v, %v", found, err) + } +} diff --git a/cmd/mesh-controller/bindings_test.go b/cmd/mesh-controller/bindings_test.go index 0a6ed5b5..98cb6b69 100644 --- a/cmd/mesh-controller/bindings_test.go +++ b/cmd/mesh-controller/bindings_test.go @@ -39,96 +39,6 @@ func need(t *testing.T, plan catalogue.Resolution, consumer, provision string) c // The incident through the stores: the laptop runs its own store and a consumer of it, the anchor's // store holds the mesh's seat. The consumer stays beside its data, the resolver follows its seat, the // binding is recorded as sent, and a pin — only a pin — moves it, said before the push that carries it. -func TestTheIncidentAConsumerStaysBesideItsDataUntilAPersonPinsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range storeManifests() { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The anchor's store and resolver hold the mesh's seats, on record; the laptop runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" || n.Module != "store" || !n.KeepsData { - t.Fatalf("the consumer was bound to %s/%s (keeps data: %v); its data is beside it", n.From, n.Module, n.KeepsData) - } - if n := need(t, plan, "network", "wildcard-resolution"); n.From != "anchor" || n.KeepsData { - t.Fatalf("the resolver was bound to %s (keeps data: %v); its seat is held on anchor (issue 258)", n.From, n.KeepsData) - } - - // Sent, and recorded: only the binding to data. - bindings := boundToData(plan, nil) - if len(bindings) != 1 || bindings[0].Provider != (catalogue.Chosen{Node: "laptop", Module: "store"}) { - t.Fatalf("recorded %+v", bindings) - } - if err := inv.RecordBindings(ctx, "laptop", bindings); err != nil { - t.Fatal(err) - } - if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { - t.Fatalf("a mesh bound where it was sent: %+v, %v", found, err) - } - - // The laptop's store taken away: refused, not moved to the anchor's empty one. - if err := inv.Unassign(ctx, "laptop", "store"); err != nil { - t.Fatal(err) - } - if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) || - !strings.Contains(err.Error(), "board on laptop is bound to laptop/store") || - !strings.Contains(err.Error(), "pin laptop postgres-database anchor store") { - t.Fatalf("the consumer's store went and it was answered elsewhere: %v", err) - } - - // A person pins the anchor's: it moves, said before it is sent, and the record keeps where it was. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { - t.Fatal(err) - } - plan, _, err = planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { - t.Fatalf("pinned to the anchor and bound to %s", n.From) - } - found, err := probeBindings(ctx, &doctor{open: open}) - if err != nil { - t.Fatal(err) - } - if len(found) != 1 || found[0].Kind != kindBindingMoving || found[0].Severity != conditions.Warning || - !strings.Contains(found[0].Summary, "board's postgres-database moves from laptop/store to anchor/store") { - t.Fatalf("a pinned move is not said before it is sent: %+v", found) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - all, err := inv.Bindings(ctx) - if err != nil || len(all) != 1 || all[0].MovedFrom != "laptop/store" { - t.Fatalf("%+v, %v", all, err) - } - if found, err := probeBindings(ctx, &doctor{open: open}); err != nil || len(found) != 0 { - t.Fatalf("a move sent is still said: %+v, %v", found, err) - } -} // What one machine's resolution says against its record. func TestBindingFindingsSayAKeptMoveAndAMoveNothingAskedFor(t *testing.T) { diff --git a/cmd/mesh-controller/build_integration_test.go b/cmd/mesh-controller/build_integration_test.go new file mode 100644 index 00000000..c2afe6e7 --- /dev/null +++ b/cmd/mesh-controller/build_integration_test.go @@ -0,0 +1,131 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestABuildHeardIsRecordedAndRegistered(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"}) + m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{ + ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop", + Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"}, + })) + if err != nil { + t.Fatal(err) + } + if m.Module != "shop" { + t.Fatalf("registered %q", m.Module) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if _, held := shelf["shop"]; !held { + t.Fatal("the module a heard build produced is not in the catalogue") + } + src, err := open.inventory.SourceOf(ctx, "shop") + if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" { + t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err) + } + builds, err := open.inventory.Builds(ctx, "shop", 5) + if err != nil || len(builds) != 1 || builds[0].ID != "b-1" { + t.Fatalf("the build is not recorded once: %v %v", builds, err) + } + + named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{ + map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa", + "env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}}) + _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ + ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named}) + if err == nil || !strings.Contains(err.Error(), "does not register it") { + t.Fatalf("a definition naming an installation was taken in: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" { + t.Fatal("the refused module was registered anyway") + } + if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 { + t.Fatalf("the refused build was not recorded: %v", builds) + } + + _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"}) + if err == nil || !strings.Contains(err.Error(), "no compiler") { + t.Fatalf("a failure is said in the builder's words: %v", err) + } +} + +func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"}) + result := func(id, ref, commit string) link.BuildResult { + return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + } + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil { + t.Fatal(err) + } + if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil { + t.Fatal(err) + } + src, err := open.inventory.SourceOf(ctx, "unifi") + if err != nil { + t.Fatal(err) + } + if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" { + t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom) + } + + // One new to the catalogue, first built at a commit, follows the default branch. + other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"}) + r := result("b-3", "deadbeef", "deadbeefcafe") + r.Manifest, r.Path = other, "modules/letta" + if _, _, err := takeIn(ctx, open.inventory, r); err != nil { + t.Fatal(err) + } + if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" { + t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref) + } +} + +func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) + newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) + result := func(asked time.Time, image string) link.BuildResult { + manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image}) + return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git", + Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} + } + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil { + t.Fatal(err) + } + _, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9")) + if !errors.Is(err, inventory.ErrSuperseded) { + t.Fatalf("the older request's outcome was taken in as current: %v", err) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if got := shelf["postgres"].Version; got != "4bcd5f73" { + t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got) + } + if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 { + t.Errorf("the late build was not recorded: %v", builds) + } +} diff --git a/cmd/mesh-controller/build_source_integration_test.go b/cmd/mesh-controller/build_source_integration_test.go new file mode 100644 index 00000000..8bc6ded9 --- /dev/null +++ b/cmd/mesh-controller/build_source_integration_test.go @@ -0,0 +1,353 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "slices" + "strings" + "testing" + "time" + + "git.novox.be/novox/mesh-sdk/go/asks" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func theCatalogue(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + for _, b := range []link.BuildResult{ + onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}), + onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}), + } { + keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true) + if _, _, err := takeIn(ctx, open.inventory, b); err != nil { + t.Fatal(err) + } + } + return open +} + +func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + for _, c := range []struct { + name, repository, path, module string + }{ + {"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"}, + {"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"}, + {"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"}, + } { + t.Run(c.name, func(t *testing.T) { + id := "build-" + strings.ReplaceAll(c.repository, "/", "-") + keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb + evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"}) + _, _, err := takeIn(ctx, open.inventory, evil) + if !errors.Is(err, errNotItsSource) { + t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err) + } + shelf, err := open.inventory.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if got := shelf[c.module].Version; got != "1" { + t.Fatalf("%s is now %q, from %s", c.module, got, c.repository) + } + if _, found, _ := open.inventory.BuildByID(ctx, id); !found { + t.Errorf("the refused build %s is not recorded", id) + } + }) + } +} + +func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + keptAsked(t, open.inventory, "build-new", "agent/tools", "", false) + _, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "", + map[string]any{"module": "agent-tools", "version": "1"})) + if !errors.Is(err, errNotItsSource) { + t.Fatalf("a new module from an agent's repository was taken in: %v", err) + } + // And one asked of nobody here — an outcome on the bus no request was kept for — the same. + _, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "", + map[string]any{"module": "agent-tools", "version": "1"})) + if !errors.Is(err, errNotItsSource) { + t.Fatalf("an outcome nobody asked for was taken in: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" { + t.Fatal("the refused module is in the catalogue") + } +} + +func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true) + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "", + map[string]any{"module": "sudo", "version": "2"})); err != nil { + t.Fatalf("a move the operator asked for at the terminal was refused: %v", err) + } + if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" { + t.Fatalf("sudo is built from %q", src.Repository) + } + if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external", + Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil { + t.Fatal(err) + } + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "", + map[string]any{"module": "app", "version": "1"})); err != nil { + t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err) + } +} + +func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) + if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID, + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil { + t.Fatal(err) + } + if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil { + t.Fatalf("a plan's build of the module's own repository was refused: %v", err) + } + added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"}) + if _, _, err := takeIn(ctx, open.inventory, added); err != nil { + t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err) + } + shelf, _ := open.inventory.Catalogue(ctx) + if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" { + t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module) + } +} + +func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) { + theCatalogue(t) + ctx := t.Context() + t.Setenv(verbVar, "build") + t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat") + err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git") + var policy *heldAtTheTerminal + if !errors.As(err, &policy) { + t.Fatalf("a verb's build of an agent's repository was asked: %v", err) + } + if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, + "http://forge.internal:20000/novox/mesh-catalog.git"); err != nil { + t.Fatalf("a verb's build of the catalogue repository was refused: %v", err) + } + t.Setenv(verbVar, "") + t.Setenv(link.CallerVar, "") + if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil { + t.Fatalf("the terminal was refused: %v", err) + } +} + +func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo", + map[string]any{"module": "sudo", "version": "evil"}) + fork.Commit = "c0ffee0123456789" // the commit sudo was registered at + keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false) + if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) { + t.Fatalf("the fork's build was taken in: %v", err) + } + // Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older + // than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main. + failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo", + map[string]any{"module": "sudo", "version": "2"}) + failed.Commit = "badbadbad0123456" + if _, _, err := takeIn(ctx, inv, failed); err != nil { + t.Fatal(err) + } + record, _, err := inv.BuildByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record) + if err != nil { + t.Fatal(err) + } + if found && previous.ID == fork.ID { + t.Fatalf("a rollback would put back the fork's build %s", previous.ID) + } + if !found || previous.ID != "build-sudo" { + t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found) + } +} + +func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"}) + t.Cleanup(func() { theForge.Delete("novox/unguarded") }) + first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"}) + keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true) + if _, _, err := takeIn(ctx, open.inventory, first); err != nil { + t.Fatalf("at the terminal: %v", err) + } + again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"}) + if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) || + !strings.Contains(err.Error(), "push to main directly") { + t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err) + } + theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api")) + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "", + map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) { + t.Fatalf("a forge that could not say was read as a protected trunk: %v", err) + } + if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" { + t.Fatalf("unguarded is %q", shelf["unguarded"].Version) + } +} + +func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true}) + t.Cleanup(func() { theForge.Delete("novox/remade") }) + first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"}) + keptAsked(t, open.inventory, first.ID, "novox/remade", "", true) + if _, _, err := takeIn(ctx, open.inventory, first); err != nil { + t.Fatal(err) + } + if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 { + t.Fatalf("the forge's id was not recorded: %d", id) + } + theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "", + map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) || + !strings.Contains(err.Error(), "made again") { + t.Fatalf("a repository made again under the name was taken in: %v", err) + } + // And a new module from it, beside the one registered from the first, the same. + if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other", + map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) { + t.Fatalf("a new module from a repository made again was taken in: %v", err) + } +} + +func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app", + Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil { + t.Fatal(err) + } + other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"}) + if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) { + t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err) + } + elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"}) + if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) { + t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err) + } +} + +func TestTheServingControllerIsNeverTheTerminal(t *testing.T) { + t.Setenv(verbVar, "") + t.Setenv(link.CallerVar, "") + t.Setenv(servedVar, "") + if !startedAtTheTerminal() { + t.Fatal("a process started by hand is not the terminal") + } + markServed() + if startedAtTheTerminal() { + t.Fatal("the serving controller reads as the terminal") + } + child := exec.Command(os.Args[0], "-test.run=^$") + child.Env = os.Environ() + if !slices.Contains(child.Env, servedVar+"=1") { + t.Fatal("what the serving controller starts does not carry its mark") + } +} + +func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) { + open := theCatalogue(t) + ctx := t.Context() + if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil { + t.Fatal(err) + } + if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 { + t.Fatalf("the recorded id became %d", id) + } +} + +func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) { + const commit = "c0ffee0123456789" + for _, c := range []struct { + what string + change func(*anApprovedMerge) + unkept bool + request string // who asked: "merge", or "build" through a verb + moved bool + }{ + {what: "the operator's Approve of the merged head", request: "merge", moved: true}, + {what: "no merge mesh-delivery announced", unkept: true, request: "merge"}, + {what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) { + a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"} + }}, + {what: "a Decline", request: "merge", change: func(a *anApprovedMerge) { + a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline" + }}, + {what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }}, + {what: "a build asked through a verb", request: "build"}, + } { + open := theCatalogue(t) + ctx := t.Context() + inv := open.inventory + a := approvedMerge(t) + a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" + if c.change != nil { + c.change(&a) + } + wasKept, wasAsk := keptDeliveryMerge, routerAskOf + keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) { + return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil + } + routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) { + if id != a.merge.AskID { + t.Errorf("%s: the router was asked for %q", c.what, id) + } + return &a.ask, &a.warrant, nil + } + t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) + event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, + "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) + if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, + Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { + t.Fatal(err) + } + id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git", + Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil { + t.Fatal(err) + } + moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) + moved.Commit = commit + _, _, err := takeIn(ctx, inv, moved) + src, _ := inv.SourceOf(ctx, "sudo") + switch { + case c.moved && err != nil: + t.Errorf("%s: the move was refused: %v", c.what, err) + case c.moved && src.Repository != "novox/mesh-host": + t.Errorf("%s: sudo is built from %q", c.what, src.Repository) + case !c.moved && !errors.Is(err, errNotItsSource): + t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err) + case !c.moved && src.Repository != "novox/mesh-catalog": + t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository) + } + } +} diff --git a/cmd/mesh-controller/build_source_test.go b/cmd/mesh-controller/build_source_test.go index 03ce2bd0..1f692d8b 100644 --- a/cmd/mesh-controller/build_source_test.go +++ b/cmd/mesh-controller/build_source_test.go @@ -3,18 +3,10 @@ package main import ( "context" "encoding/json" - "errors" - "fmt" "hash/fnv" - "os" - "os/exec" - "slices" "strings" "sync" "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/asks" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -47,141 +39,14 @@ func keptAsked(t *testing.T, inv *inventory.Inventory, id, repository, path stri } // theCatalogue is a mesh whose sudo is built from the catalogue repository and whose node-engine from its own. -func theCatalogue(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - for _, b := range []link.BuildResult{ - onTrunk("build-sudo", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "1"}), - onTrunk("build-host", "novox/mesh-host", "git", "", map[string]any{"module": "mesh-host", "version": "1"}), - } { - keptAsked(t, open.inventory, b.ID, b.Source.Repository, b.Path, true) - if _, _, err := takeIn(ctx, open.inventory, b); err != nil { - t.Fatal(err) - } - } - return open -} - -func TestABuildFromAnAgentsRepositoryIsNotRegisteredAsAModuleTheMeshHolds(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - for _, c := range []struct { - name, repository, path, module string - }{ - {"its own repository naming sudo", "agent/sudo", "modules/sudo", "sudo"}, - {"a fork of the catalogue", "agent/mesh-catalog", "modules/sudo", "sudo"}, - {"a fork of the node-engine", "agent/mesh-host", "", "mesh-host"}, - } { - t.Run(c.name, func(t *testing.T) { - id := "build-" + strings.ReplaceAll(c.repository, "/", "-") - keptAsked(t, open.inventory, id, c.repository, c.path, false) // through the build verb - evil := onTrunk(id, c.repository, "git", c.path, map[string]any{"module": c.module, "version": "evil"}) - _, _, err := takeIn(ctx, open.inventory, evil) - if !errors.Is(err, errNotItsSource) { - t.Fatalf("a build of %s was taken in as %s: %v", c.repository, c.module, err) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if got := shelf[c.module].Version; got != "1" { - t.Fatalf("%s is now %q, from %s", c.module, got, c.repository) - } - if _, found, _ := open.inventory.BuildByID(ctx, id); !found { - t.Errorf("the refused build %s is not recorded", id) - } - }) - } -} - -func TestANewModuleFromARepositoryTheCatalogueDoesNotBuildFromIsNotRegistered(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - keptAsked(t, open.inventory, "build-new", "agent/tools", "", false) - _, _, err := takeIn(ctx, open.inventory, onTrunk("build-new", "agent/tools", "git", "", - map[string]any{"module": "agent-tools", "version": "1"})) - if !errors.Is(err, errNotItsSource) { - t.Fatalf("a new module from an agent's repository was taken in: %v", err) - } - // And one asked of nobody here — an outcome on the bus no request was kept for — the same. - _, _, err = takeIn(ctx, open.inventory, onTrunk("build-unasked", "agent/tools", "git", "", - map[string]any{"module": "agent-tools", "version": "1"})) - if !errors.Is(err, errNotItsSource) { - t.Fatalf("an outcome nobody asked for was taken in: %v", err) - } - if shelf, _ := open.inventory.Catalogue(ctx); shelf["agent-tools"].Module != "" { - t.Fatal("the refused module is in the catalogue") - } -} // The operator at the terminal may still move a module, or add one from a new repository. -func TestAtTheTerminalAnotherRepositoryIsRegistered(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - keptAsked(t, open.inventory, "build-moved", "novox/sudo", "", true) - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-moved", "novox/sudo", "git", "", - map[string]any{"module": "sudo", "version": "2"})); err != nil { - t.Fatalf("a move the operator asked for at the terminal was refused: %v", err) - } - if src, _ := open.inventory.SourceOf(ctx, "sudo"); src.Repository != "novox/sudo" { - t.Fatalf("sudo is built from %q", src.Repository) - } - if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-external", - Repository: "http://forge.internal:20000/someone/app.git", For: "build", AtTerminal: true}); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-external", "someone/app", "", "", - map[string]any{"module": "app", "version": "1"})); err != nil { - t.Fatalf("a new module the operator asked for at the terminal was refused: %v", err) - } -} // The delivery's flow is untouched: a merge's rebuild of a module from its own repository, and a merge adding // a module to a repository the catalogue builds from (novox/hq issue 300), are registered with no terminal. -func TestADeliveryFromTheRegisteredRepositoryIsRegistered(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - rebuilt := onTrunk("build-plan", "novox/mesh-catalog", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) - if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: rebuilt.ID, - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/sudo", For: "plan"}); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, open.inventory, rebuilt); err != nil { - t.Fatalf("a plan's build of the module's own repository was refused: %v", err) - } - added := onTrunk("build-merge", "novox/mesh-catalog", "git", "modules/zram", map[string]any{"module": "zram", "version": "1"}) - if _, _, err := takeIn(ctx, open.inventory, added); err != nil { - t.Fatalf("a module a merge added to the catalogue repository was refused: %v", err) - } - shelf, _ := open.inventory.Catalogue(ctx) - if shelf["sudo"].Version != "2" || shelf["zram"].Module == "" { - t.Fatalf("not registered: sudo %q, zram %q", shelf["sudo"].Version, shelf["zram"].Module) - } -} // Through a verb, a build of a repository the catalogue builds nothing from is not even asked: the build node // would run what the agent wrote. -func TestAVerbAsksNoBuildOfARepositoryTheCatalogueDoesNotBuildFrom(t *testing.T) { - theCatalogue(t) - ctx := t.Context() - t.Setenv(verbVar, "build") - t.Setenv(link.CallerVar, "node-tools.anchor, through the mesh-controller seat") - err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, "http://forge.internal:20000/agent/sudo.git") - var policy *heldAtTheTerminal - if !errors.As(err, &policy) { - t.Fatalf("a verb's build of an agent's repository was asked: %v", err) - } - if err := verbMayAsk(ctx, buildSource{Repository: "novox/mesh-catalog", Seat: "git"}, - "http://forge.internal:20000/novox/mesh-catalog.git"); err != nil { - t.Fatalf("a verb's build of the catalogue repository was refused: %v", err) - } - t.Setenv(verbVar, "") - t.Setenv(link.CallerVar, "") - if err := verbMayAsk(ctx, buildSource{Repository: "agent/sudo", Seat: "git"}, ""); err != nil { - t.Fatalf("the terminal was refused: %v", err) - } -} // asTheOperator keeps a build as asked at the controller's terminal, as the operator's first build of a module // from a repository the catalogue does not yet build from is (novox/hq ADR 0266), and hands it back. @@ -201,40 +66,6 @@ func asTheOperator(t *testing.T, inv *inventory.Inventory, b link.BuildResult) l // A rollback puts back only a build of the module's own repository: an agent's build of the module's name, // recorded and refused, at the very commit the machine ran before (a fork carries it), is never registered by // the back door of a failed gate. -func TestARollbackNeverPutsBackABuildFromAnotherRepository(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - inv := open.inventory - fork := onTrunk("build-1791500000000000000", "agent/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "evil"}) - fork.Commit = "c0ffee0123456789" // the commit sudo was registered at - keptAsked(t, inv, fork.ID, "agent/mesh-catalog", "modules/sudo", false) - if _, _, err := takeIn(ctx, inv, fork); !errors.Is(err, errNotItsSource) { - t.Fatalf("the fork's build was taken in: %v", err) - } - // Asked after the registered build, whenever the test runs: an id naming a fixed moment read as older - // than the registered build once the clock passed it (2026-10-10 02:40 UTC), and the test failed on main. - failed := onTrunk(fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()), "novox/mesh-catalog", "git", "modules/sudo", - map[string]any{"module": "sudo", "version": "2"}) - failed.Commit = "badbadbad0123456" - if _, _, err := takeIn(ctx, inv, failed); err != nil { - t.Fatal(err) - } - record, _, err := inv.BuildByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - previous, found, err := inv.PreviousBuild(ctx, "sudo", "c0ffee0123456789", record) - if err != nil { - t.Fatal(err) - } - if found && previous.ID == fork.ID { - t.Fatalf("a rollback would put back the fork's build %s", previous.ID) - } - if !found || previous.ID != "build-sudo" { - t.Fatalf("a rollback puts back %q (found %v), want the registered build-sudo", previous.ID, found) - } -} // theForge is what the forge says in a test, by owner/name: a repository not named here is protected as a // trunk must be, with an id of its own. @@ -258,97 +89,15 @@ func init() { // A module's trunk the forge does not protect — direct pushes, no required status — or a forge that cannot say, // registers nothing through a verb: the trunk rule means nothing on a branch anyone pushes to. -func TestATrunkTheForgeDoesNotProtectRegistersNothing(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - theForge.Store("novox/unguarded", forgeFacts{ID: 7, Why: "the rule main lets a person push to main directly"}) - t.Cleanup(func() { theForge.Delete("novox/unguarded") }) - first := onTrunk("build-unguarded-1", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "1"}) - keptAsked(t, open.inventory, first.ID, "novox/unguarded", "", true) - if _, _, err := takeIn(ctx, open.inventory, first); err != nil { - t.Fatalf("at the terminal: %v", err) - } - again := onTrunk("build-unguarded-2", "novox/unguarded", "git", "", map[string]any{"module": "unguarded", "version": "2"}) - if _, _, err := takeIn(ctx, open.inventory, again); !errors.Is(err, errNotItsSource) || - !strings.Contains(err.Error(), "push to main directly") { - t.Fatalf("a rebuild from an unprotected trunk was taken in: %v", err) - } - theForge.Store("novox/unguarded", errors.New("nothing serves gitea.gitea_api")) - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-unguarded-3", "novox/unguarded", "git", "", - map[string]any{"module": "unguarded", "version": "3"})); !errors.Is(err, errNotItsSource) { - t.Fatalf("a forge that could not say was read as a protected trunk: %v", err) - } - if shelf, _ := open.inventory.Catalogue(ctx); shelf["unguarded"].Version != "1" { - t.Fatalf("unguarded is %q", shelf["unguarded"].Version) - } -} // A repository deleted and made again under the module's repository's name is another repository: the forge's // id, recorded at registration, tells them apart. -func TestARepositoryMadeAgainUnderItsNameIsNotTheModulesSource(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - theForge.Store("novox/remade", forgeFacts{ID: 100, Guarded: true}) - t.Cleanup(func() { theForge.Delete("novox/remade") }) - first := onTrunk("build-remade-1", "novox/remade", "git", "", map[string]any{"module": "remade", "version": "1"}) - keptAsked(t, open.inventory, first.ID, "novox/remade", "", true) - if _, _, err := takeIn(ctx, open.inventory, first); err != nil { - t.Fatal(err) - } - if id, _ := open.inventory.SourceIdentity(ctx, "remade"); id != 100 { - t.Fatalf("the forge's id was not recorded: %d", id) - } - theForge.Store("novox/remade", forgeFacts{ID: 101, Guarded: true}) // deleted, and made again by an agent - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-2", "novox/remade", "git", "", - map[string]any{"module": "remade", "version": "evil"})); !errors.Is(err, errNotItsSource) || - !strings.Contains(err.Error(), "made again") { - t.Fatalf("a repository made again under the name was taken in: %v", err) - } - // And a new module from it, beside the one registered from the first, the same. - if _, _, err := takeIn(ctx, open.inventory, onTrunk("build-remade-3", "novox/remade", "git", "modules/other", - map[string]any{"module": "other", "version": "1"})); !errors.Is(err, errNotItsSource) { - t.Fatalf("a new module from a repository made again was taken in: %v", err) - } -} // The terminal's mark is the operator's for the repository and path they asked: an outcome of another, under that // build's id, is not theirs. -func TestATerminalRequestCoversOnlyWhatItAsked(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - if err := open.inventory.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-asked", Repository: "novox/app", - Seat: "git", Path: "modules/app", For: "build", AtTerminal: true}); err != nil { - t.Fatal(err) - } - other := onTrunk("build-asked", "agent/sudo", "git", "modules/app", map[string]any{"module": "sudo", "version": "evil"}) - if _, _, err := takeIn(ctx, open.inventory, other); !errors.Is(err, errNotItsSource) { - t.Fatalf("an outcome of another repository under a terminal request's id was taken in: %v", err) - } - elsewhere := onTrunk("build-asked", "novox/app", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "evil"}) - if _, _, err := takeIn(ctx, open.inventory, elsewhere); !errors.Is(err, errNotItsSource) { - t.Fatalf("an outcome of another path under a terminal request's id was taken in: %v", err) - } -} // The serving controller, and everything it starts, is never the terminal: a build asked in it reads as asked // through the mesh even when no verb and no caller is named. -func TestTheServingControllerIsNeverTheTerminal(t *testing.T) { - t.Setenv(verbVar, "") - t.Setenv(link.CallerVar, "") - t.Setenv(servedVar, "") - if !startedAtTheTerminal() { - t.Fatal("a process started by hand is not the terminal") - } - markServed() - if startedAtTheTerminal() { - t.Fatal("the serving controller reads as the terminal") - } - child := exec.Command(os.Args[0], "-test.run=^$") - child.Env = os.Environ() - if !slices.Contains(child.Env, servedVar+"=1") { - t.Fatal("what the serving controller starts does not carry its mark") - } -} // The forge applies one rule to a branch: the rule named for it, else the first glob that covers it. A stronger // rule later in the list is not what guards the branch, and is not read as if it were (the confirmation review). @@ -371,88 +120,9 @@ func TestTheRuleJudgedIsTheOneTheForgeApplies(t *testing.T) { } // An id the forge could not give keeps the id already recorded. -func TestAnUnknownIdentityKeepsTheRecordedOne(t *testing.T) { - open := theCatalogue(t) - ctx := t.Context() - if err := open.inventory.SetSourceIdentity(ctx, "sudo", 100); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSourceIdentity(ctx, "sudo", 0); err != nil { - t.Fatal(err) - } - if id, _ := open.inventory.SourceIdentity(ctx, "sudo"); id != 100 { - t.Fatalf("the recorded id became %d", id) - } -} // **The operator's Approve of a reviewed group registers a moved module from its new repository** (novox/hq ADR 0420 §7, // issue 1025). sudo is built from the catalogue; a merge into the repository of node-engine's module, announced by // mesh-delivery with the ask it merged on, brings its build from there. Accepted with the operator's Approve of the // merged head; refused with no kept merge, a warrant for other heads, a Decline, a person who is not the operator, and // through a verb. -func TestAModuleMovesOnTheOperatorsApproveOfTheMergeThatMovedIt(t *testing.T) { - const commit = "c0ffee0123456789" - for _, c := range []struct { - what string - change func(*anApprovedMerge) - unkept bool - request string // who asked: "merge", or "build" through a verb - moved bool - }{ - {what: "the operator's Approve of the merged head", request: "merge", moved: true}, - {what: "no merge mesh-delivery announced", unkept: true, request: "merge"}, - {what: "a warrant for other heads", request: "merge", change: func(a *anApprovedMerge) { - a.merge.Heads = map[string]string{"novox/mesh-host@aaaaaaaaaaaa": "e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5e5"} - }}, - {what: "a Decline", request: "merge", change: func(a *anApprovedMerge) { - a.warrant.Option, a.warrant.Label, a.merge.Option = "decline", "Decline", "decline" - }}, - {what: "a person who is not the operator", request: "merge", change: func(a *anApprovedMerge) { a.warrant.By.Who = "agent" }}, - {what: "a build asked through a verb", request: "build"}, - } { - open := theCatalogue(t) - ctx := t.Context() - inv := open.inventory - a := approvedMerge(t) - a.merge.Owner, a.merge.Repo, a.merge.Number, a.merge.HeadSHA = "novox", "mesh-host", 7, "a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1" - if c.change != nil { - c.change(&a) - } - wasKept, wasAsk := keptDeliveryMerge, routerAskOf - keptDeliveryMerge = func(_ context.Context, owner, repo string, number int, head string) (deliveryMerge, bool, error) { - return a.merge, !c.unkept && owner == "novox" && repo == "mesh-host" && number == 7 && head == a.merge.HeadSHA, nil - } - routerAskOf = func(_ context.Context, _ *inventory.Inventory, id string) (*asks.Ask, *asks.Warrant, error) { - if id != a.merge.AskID { - t.Errorf("%s: the router was asked for %q", c.what, id) - } - return &a.ask, &a.warrant, nil - } - t.Cleanup(func() { keptDeliveryMerge, routerAskOf = wasKept, wasAsk }) - event, _ := json.Marshal(map[string]any{"owner": "novox", "repo": "mesh-host", "base": "main", "number": 7, - "head_sha": a.merge.HeadSHA, "merge_commit_sha": commit}) - if _, err := inv.AddMerge(ctx, inventory.BatchedMerge{Repository: "novox/mesh-host", Branch: "main", Commit: commit, - Merged: time.Now(), Heard: time.Now(), Event: event}); err != nil { - t.Fatal(err) - } - id := fmt.Sprintf("build-%d", time.Now().Add(time.Hour).UnixNano()) - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-host", Seat: "git", - Path: "modules/sudo", Ref: "main", Commit: commit, For: c.request}); err != nil { - t.Fatal(err) - } - moved := onTrunk(id, "novox/mesh-host", "git", "modules/sudo", map[string]any{"module": "sudo", "version": "2"}) - moved.Commit = commit - _, _, err := takeIn(ctx, inv, moved) - src, _ := inv.SourceOf(ctx, "sudo") - switch { - case c.moved && err != nil: - t.Errorf("%s: the move was refused: %v", c.what, err) - case c.moved && src.Repository != "novox/mesh-host": - t.Errorf("%s: sudo is built from %q", c.what, src.Repository) - case !c.moved && !errors.Is(err, errNotItsSource): - t.Errorf("%s: the move was not refused as ADR 0266 has it: %v", c.what, err) - case !c.moved && src.Repository != "novox/mesh-catalog": - t.Errorf("%s: a refused move changed sudo's source to %q", c.what, src.Repository) - } - } -} diff --git a/cmd/mesh-controller/build_test.go b/cmd/mesh-controller/build_test.go index aa70ff81..f7fb0490 100644 --- a/cmd/mesh-controller/build_test.go +++ b/cmd/mesh-controller/build_test.go @@ -1,13 +1,9 @@ package main import ( - "encoding/json" - "errors" - "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -15,126 +11,12 @@ import ( // the module registered with its source as the seat and path when the request said so — never the // URL. A definition naming an installation is recorded and not registered; a failure is recorded // and said. -func TestABuildHeardIsRecordedAndRegistered(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - manifest, _ := json.Marshal(map[string]any{"module": "shop", "version": "3"}) - m, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, link.BuildResult{ - ID: "b-1", Repository: "http://forge.internal:20000/novox/shop.git", Path: "modules/shop", - Ref: "main", On: "anchor", Commit: "abcdef0123", Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/shop"}, - })) - if err != nil { - t.Fatal(err) - } - if m.Module != "shop" { - t.Fatalf("registered %q", m.Module) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if _, held := shelf["shop"]; !held { - t.Fatal("the module a heard build produced is not in the catalogue") - } - src, err := open.inventory.SourceOf(ctx, "shop") - if err != nil || src.Seat != "git" || src.Repository != "novox/shop" || src.BuiltFrom != "abcdef0123" { - t.Fatalf("the source is the seat and the path, never the URL: %+v %v", src, err) - } - builds, err := open.inventory.Builds(ctx, "shop", 5) - if err != nil || len(builds) != 1 || builds[0].ID != "b-1" { - t.Fatalf("the build is not recorded once: %v %v", builds, err) - } - - named, _ := json.Marshal(map[string]any{"module": "idp", "version": "1", "resources": []any{ - map[string]any{"id": "server", "type": "container", "image": "x@sha256:aa", - "env": map[string]any{"KC_HOSTNAME": "https://login.mesh-one.be"}}}}) - _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ - ID: "b-2", Repository: "/r", On: "anchor", Commit: "0123456789", Manifest: named}) - if err == nil || !strings.Contains(err.Error(), "does not register it") { - t.Fatalf("a definition naming an installation was taken in: %v", err) - } - if shelf, _ := open.inventory.Catalogue(ctx); shelf["idp"].Module != "" { - t.Fatal("the refused module was registered anyway") - } - if builds, _ := open.inventory.Builds(ctx, "idp", 5); len(builds) != 1 { - t.Fatalf("the refused build was not recorded: %v", builds) - } - - _, _, err = takeIn(ctx, open.inventory, link.BuildResult{ID: "b-3", Repository: "/r", On: "anchor", Failed: "no compiler"}) - if err == nil || !strings.Contains(err.Error(), "no compiler") { - t.Fatalf("a failure is said in the builder's words: %v", err) - } -} // novox/hq 04-ISSUES/215: a build asked at a commit is recorded as built from that commit, and the // module keeps following the branch it followed — a new one, the default branch. -func TestABuildAtACommitKeepsTheBranchTheModuleFollows(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - manifest, _ := json.Marshal(map[string]any{"module": "unifi", "version": "1"}) - result := func(id, ref, commit string) link.BuildResult { - return link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", - Path: "modules/unifi", Ref: ref, On: "anchor", Commit: commit, Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - } - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result("b-1", "main", "1111111aaaa"))); err != nil { - t.Fatal(err) - } - if _, _, err := takeIn(ctx, open.inventory, result("b-2", "9c97a8a", "9c97a8a1d2c3")); err != nil { - t.Fatal(err) - } - src, err := open.inventory.SourceOf(ctx, "unifi") - if err != nil { - t.Fatal(err) - } - if src.Ref != "main" || src.BuiltFrom != "9c97a8a1d2c3" { - t.Errorf("after a build at a commit the module follows %q, built from %q; want main, 9c97a8a1d2c3", src.Ref, src.BuiltFrom) - } - - // One new to the catalogue, first built at a commit, follows the default branch. - other, _ := json.Marshal(map[string]any{"module": "letta", "version": "1"}) - r := result("b-3", "deadbeef", "deadbeefcafe") - r.Manifest, r.Path = other, "modules/letta" - if _, _, err := takeIn(ctx, open.inventory, r); err != nil { - t.Fatal(err) - } - if src, _ := open.inventory.SourceOf(ctx, "letta"); src.Ref != "" { - t.Errorf("a module first built at a commit follows %q, want the default branch", src.Ref) - } -} // novox/hq 04-ISSUES/219: an older request heard after a newer one is recorded and not registered, // so a push sends what the newer request built. -func TestAnOlderBuildHeardLaterDoesNotReplaceTheNewer(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - older := time.Date(2026, 10, 3, 21, 33, 45, 0, time.UTC) - newer := time.Date(2026, 10, 3, 21, 51, 57, 0, time.UTC) - result := func(asked time.Time, image string) link.BuildResult { - manifest, _ := json.Marshal(map[string]any{"module": "postgres", "version": image}) - return link.BuildResult{ID: link.NewBuildID(asked), Repository: "http://forge.internal:20000/novox/mesh-catalog.git", - Path: "modules/postgres", Ref: "main", On: "anchor", Commit: "efff5415", Manifest: manifest, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}} - } - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, result(newer, "4bcd5f73"))); err != nil { - t.Fatal(err) - } - _, _, err := takeIn(ctx, open.inventory, result(older, "0ab07fa9")) - if !errors.Is(err, inventory.ErrSuperseded) { - t.Fatalf("the older request's outcome was taken in as current: %v", err) - } - shelf, err := open.inventory.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if got := shelf["postgres"].Version; got != "4bcd5f73" { - t.Errorf("postgres is %q; want the newer request's 4bcd5f73", got) - } - if builds, _ := open.inventory.Builds(ctx, "postgres", 5); len(builds) != 2 { - t.Errorf("the late build was not recorded: %v", builds) - } -} func TestABuildIDSaysWhenItWasAsked(t *testing.T) { at := time.Date(2026, 10, 3, 21, 51, 57, 392539762, time.UTC) diff --git a/cmd/mesh-controller/bus_waits_integration_test.go b/cmd/mesh-controller/bus_waits_integration_test.go new file mode 100644 index 00000000..6fec986c --- /dev/null +++ b/cmd/mesh-controller/bus_waits_integration_test.go @@ -0,0 +1,70 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) { + now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) + b := aBusOnAnchor() + walk := func(id, repo, note string, updated time.Time) inventory.Plan { + return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0, + Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note, + Updated: updated} + } + for _, c := range []struct { + name string + bus busPending + plans []inventory.Plan + first map[string]time.Time + want []string + since time.Time + }{ + {"a walk refused for the bus", b, + []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil, + []string{"plan-1"}, now.Add(-27 * time.Minute)}, + {"refused earlier than its last save, as this controller saw it", b, + []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))}, + map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)}, + {"a walk refused for another reason", b, + []inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil, + nil, time.Time{}}, + {"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(), + []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, + {"the bus already runs the build held: its step started", func() busPending { + o := aBusOnAnchor() + o.from = map[string]string{"anchor": o.to} + return o + }(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, + {"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan { + p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now) + p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"} + return []inventory.Plan{p} + }(), nil, nil, time.Time{}}, + } { + t.Run(c.name, func(t *testing.T) { + got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] }) + var ids []string + for _, w := range got.waits { + ids = append(ids, w.plan) + } + if strings.Join(ids, ",") != strings.Join(c.want, ",") { + t.Fatalf("held for the bus: %v, want %v", ids, c.want) + } + if len(c.want) > 0 { + if !got.waits[0].since.Equal(c.since) { + t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since) + } + if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" { + t.Errorf("behind %+v", got) + } + } + }) + } +} diff --git a/cmd/mesh-controller/bus_waits_test.go b/cmd/mesh-controller/bus_waits_test.go index 2a462b21..668d7cff 100644 --- a/cmd/mesh-controller/bus_waits_test.go +++ b/cmd/mesh-controller/bus_waits_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" ) // novox/hq issue 336: a send held because it would replace the bus outside its planned step waits for a @@ -27,65 +26,6 @@ func refusedNote(b busPending, tier int) string { return "tier " + string(rune('0'+tier)) + ": " + errBusWaits.Error() + ": " + held + " — tried again" } -func TestASendHeldForTheBusStepIsFoundFromTheWalksItHolds(t *testing.T) { - now := time.Date(2026, 10, 8, 18, 30, 0, 0, time.UTC) - b := aBusOnAnchor() - walk := func(id, repo, note string, updated time.Time) inventory.Plan { - return inventory.Plan{ID: id, Repository: repo, Commit: "c0ffee001122", State: inventory.PlanRolling, Tier: 0, - Tiers: [][]string{{"mesh-host"}}, Modules: map[string]*inventory.PlanModule{"mesh-host": {}}, Note: note, - Updated: updated} - } - for _, c := range []struct { - name string - bus busPending - plans []inventory.Plan - first map[string]time.Time - want []string - since time.Time - }{ - {"a walk refused for the bus", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-27*time.Minute))}, nil, - []string{"plan-1"}, now.Add(-27 * time.Minute)}, - {"refused earlier than its last save, as this controller saw it", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now.Add(-5*time.Minute))}, - map[string]time.Time{"plan-1": now.Add(-28 * time.Minute)}, []string{"plan-1"}, now.Add(-28 * time.Minute)}, - {"a walk refused for another reason", b, - []inventory.Plan{walk("plan-1", "novox/mesh-host", "tier 0: the build seat is paused — tried again", now)}, nil, - nil, time.Time{}}, - {"refused for an older bus build than the one held now", func() busPending { o := b; o.to = "99999999cccc"; return o }(), - []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"the bus already runs the build held: its step started", func() busPending { - o := aBusOnAnchor() - o.from = map[string]string{"anchor": o.to} - return o - }(), []inventory.Plan{walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now)}, nil, nil, time.Time{}}, - {"a walk waiting for its delivery's word asks no send", b, func() []inventory.Plan { - p := walk("plan-1", "novox/mesh-host", refusedNote(b, 0), now) - p.Delivery = &inventory.PlanDelivery{Awaits: "mesh-delivery"} - return []inventory.Plan{p} - }(), nil, nil, time.Time{}}, - } { - t.Run(c.name, func(t *testing.T) { - got := busWaitsOf(c.plans, c.bus, func(id string) time.Time { return c.first[id] }) - var ids []string - for _, w := range got.waits { - ids = append(ids, w.plan) - } - if strings.Join(ids, ",") != strings.Join(c.want, ",") { - t.Fatalf("held for the bus: %v, want %v", ids, c.want) - } - if len(c.want) > 0 { - if !got.waits[0].since.Equal(c.since) { - t.Errorf("waiting since %s, want %s", got.waits[0].since, c.since) - } - if strings.Join(got.machines, ",") != "anchor" || got.to != b.to || got.module != "nats" { - t.Errorf("behind %+v", got) - } - } - }) - } -} - // **Said at once, not as lateness, and cleared when the step starts**: the walk held only for the bus raises // the bus's condition on the first tick, before any tier bound, naming what waits, the bus build from and to, // since when and `bus upgrade`, for the operator; S3 says nothing of it even past its bound; and the diff --git a/cmd/mesh-controller/busobjects_test.go b/cmd/mesh-controller/busobjects_test.go index 962a4659..349ab225 100644 --- a/cmd/mesh-controller/busobjects_test.go +++ b/cmd/mesh-controller/busobjects_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/calls_verb_integration_test.go b/cmd/mesh-controller/calls_verb_integration_test.go new file mode 100644 index 00000000..3efc354a --- /dev/null +++ b/cmd/mesh-controller/calls_verb_integration_test.go @@ -0,0 +1,36 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" +) + +func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) { + handlers, behind, err := seatToolHandlers() + if err != nil || len(behind) != 0 { + t.Fatalf("%v %v", behind, err) + } + calls, ok := handlers["calls"] + if !ok { + t.Fatal("calls is not served") + } + if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil || + !strings.Contains(err.Error(), `"node"`) { + t.Errorf("calls took an argument it does not declare: %v", err) + } + if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil || + !strings.Contains(err.Error(), "not across a restart") { + t.Errorf("an unknown call was not said plainly: %v", err) + } + got, err := calls(context.Background(), json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + if _, listed := got.(map[string]any)["calls"]; !listed { + t.Errorf("calls answered %v", got) + } +} diff --git a/cmd/mesh-controller/calls_verb_test.go b/cmd/mesh-controller/calls_verb_test.go index 606a53f8..4ff06dcb 100644 --- a/cmd/mesh-controller/calls_verb_test.go +++ b/cmd/mesh-controller/calls_verb_test.go @@ -1,9 +1,6 @@ package main import ( - "context" - "encoding/json" - "strings" "testing" "time" @@ -52,28 +49,3 @@ func TestAPushAnswersBeforeItSends(t *testing.T) { } // `calls` is served, takes a call's id and nothing else, and says plainly when it holds no such call. -func TestCallsIsServedAndSaysWhatItKeeps(t *testing.T) { - handlers, behind, err := seatToolHandlers() - if err != nil || len(behind) != 0 { - t.Fatalf("%v %v", behind, err) - } - calls, ok := handlers["calls"] - if !ok { - t.Fatal("calls is not served") - } - if _, err := calls(context.Background(), json.RawMessage(`{"node":"anchor"}`)); err == nil || - !strings.Contains(err.Error(), `"node"`) { - t.Errorf("calls took an argument it does not declare: %v", err) - } - if _, err := calls(context.Background(), json.RawMessage(`{"call":"call-0-0"}`)); err == nil || - !strings.Contains(err.Error(), "not across a restart") { - t.Errorf("an unknown call was not said plainly: %v", err) - } - got, err := calls(context.Background(), json.RawMessage(`{}`)) - if err != nil { - t.Fatal(err) - } - if _, listed := got.(map[string]any)["calls"]; !listed { - t.Errorf("calls answered %v", got) - } -} diff --git a/cmd/mesh-controller/check_builds_integration_test.go b/cmd/mesh-controller/check_builds_integration_test.go new file mode 100644 index 00000000..37f33a6a --- /dev/null +++ b/cmd/mesh-controller/check_builds_integration_test.go @@ -0,0 +1,102 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) { + p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ + {Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3}, + {Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}} + asked := reuseAskOf(p) + if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) { + t.Fatalf("the walk names %+v", asked.Pulls) + } + walk := withReuse(t.Context(), asked) + if reuseAsked(walk, false) != asked { + t.Fatal("a walk's ask does not carry its leave") + } + if reuseAsked(walk, true) != nil { + t.Fatal("a dry run carries a leave to take a check's build") + } + if reuseAsked(t.Context(), false) != nil { + t.Fatal("an ask that is not a walk's carries a leave") + } +} + +func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + gateEvery = 0 + + digest := "sha256:" + strings.Repeat("c", 64) + reused := func(id, reference string) link.BuildResult { + manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{ + {"id": "svc", "type": "container", "name": "app", "image": reference}}}) + return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor", + Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"}, + Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}}, + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}, + Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest, + ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9}, + Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}} + } + if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil || + !strings.Contains(err.Error(), "scratch") { + t.Fatalf("a build in the scratch namespace was registered: %v", err) + } + id := link.NewBuildID(time.Now().Add(time.Second)) + result := reused(id, "store.internal:5000/app/server@"+digest) + if _, _, err := takeIn(ctx, inv, result); err != nil { + t.Fatal(err) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil || current["app"].Commit != "c3" { + t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + raw, _ := json.Marshal(shelf["app"].Resources) + if !strings.Contains(string(raw), "app/server@"+digest) { + t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw) + } + + // The walk asked it: the walk keeps what its build did, and its gate judges the registered build. + asked := time.Now().UTC() + p := g.plan(t) + p.Commit = "c3" + p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords()) + p = g.plan(t) + if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") { + t.Fatalf("the walk keeps %+v", m) + } + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p = g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed { + t.Fatalf("the gate judged %+v, not the reused build", gate) + } + if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" { + t.Fatalf("the first machine was sent %v, not the reused build", sent) + } +} diff --git a/cmd/mesh-controller/check_builds_test.go b/cmd/mesh-controller/check_builds_test.go index 0327b85a..7a339b51 100644 --- a/cmd/mesh-controller/check_builds_test.go +++ b/cmd/mesh-controller/check_builds_test.go @@ -2,12 +2,10 @@ package main import ( "context" - "encoding/json" "errors" "reflect" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/builder" "github.com/novox/mesh-controller/internal/catalogue" @@ -46,25 +44,6 @@ func TestACheckIsAskedToBuildWhatTheMergeWouldMove(t *testing.T) { // A walk's ask carries its leave to take a check's build, naming the pull requests of the merges it answers; a dry // run never does, and neither does an ask that is not a walk's. -func TestOnlyAWalksAskTakesAChecksBuildAndADryRunNever(t *testing.T) { - p := &inventory.Plan{Delivery: &inventory.PlanDelivery{Merges: []inventory.PlanMerge{ - {Repository: "novox/shop", Number: 7}, {Repository: "http://forge.invalid/novox/web.git", Number: 3}, - {Repository: "novox/shop", Number: 7}, {Repository: "novox/shop"}}}} - asked := reuseAskOf(p) - if want := []link.PullRef{{Owner: "novox", Repo: "shop", Number: 7}, {Owner: "novox", Repo: "web", Number: 3}}; !reflect.DeepEqual(asked.Pulls, want) { - t.Fatalf("the walk names %+v", asked.Pulls) - } - walk := withReuse(t.Context(), asked) - if reuseAsked(walk, false) != asked { - t.Fatal("a walk's ask does not carry its leave") - } - if reuseAsked(walk, true) != nil { - t.Fatal("a dry run carries a leave to take a check's build") - } - if reuseAsked(t.Context(), false) != nil { - t.Fatal("an ask that is not a walk's carries a leave") - } -} // A walk ended lets go of its merges' check builds — only once it ended, after it took what it reuses. func TestAWalkEndedLetsGoOfItsMergesCheckBuilds(t *testing.T) { @@ -124,72 +103,6 @@ func TestAWalksBuildSaysWhetherItReusedOrBuilt(t *testing.T) { // module's namespace — and the walk keeps "reused from …" for it; its first-node gate judges that registered // build as it judges one the walk built. A scratch reference is refused. Needs the store (MESH_TEST_POSTGRES), and // is named in testdata/store-tests. -func TestAReusedCheckBuildIsRegisteredByItsDigestAndGatedAsABuiltOne(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - gateEvery = 0 - - digest := "sha256:" + strings.Repeat("c", 64) - reused := func(id, reference string) link.BuildResult { - manifest := mustJSON(t, catalogue.Manifest{Module: "app", Version: "c3", Resources: []map[string]any{ - {"id": "svc", "type": "container", "name": "app", "image": reference}}}) - return link.BuildResult{ID: id, Repository: "novox/mesh-catalog", Path: "modules/app", Ref: "main", On: "anchor", - Commit: "c3", Manifest: manifest, Trunk: "main", OnTrunk: true, Branches: []string{"main"}, - Made: []link.MadeArtifact{{Name: "server", Kind: "image", Reference: reference}}, - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}, - Reused: "reused from the check of novox/mesh-catalog#9 at head abcdefabcdef (merged into trunk 123456123456): server " + digest, - ReusedFrom: &link.ReusedFrom{Pull: link.PullRef{Owner: "novox", Repo: "mesh-catalog", Number: 9}, - Head: "abcdefabcdef", Trunk: "123456123456", Key: "r1:" + strings.Repeat("1", 64)}} - } - if _, _, err := takeIn(ctx, inv, reused(link.NewBuildID(time.Now()), "store.internal:5000/scratch/app/server@"+digest)); err == nil || - !strings.Contains(err.Error(), "scratch") { - t.Fatalf("a build in the scratch namespace was registered: %v", err) - } - id := link.NewBuildID(time.Now().Add(time.Second)) - result := reused(id, "store.internal:5000/app/server@"+digest) - if _, _, err := takeIn(ctx, inv, result); err != nil { - t.Fatal(err) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil || current["app"].Commit != "c3" { - t.Fatalf("the reused build is not what app is registered at: %+v %v", current["app"], err) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - raw, _ := json.Marshal(shelf["app"].Resources) - if !strings.Contains(string(raw), "app/server@"+digest) { - t.Fatalf("app is registered as %s, not by the check's digest in its own namespace", raw) - } - - // The walk asked it: the walk keeps what its build did, and its gate judges the registered build. - asked := time.Now().UTC() - p := g.plan(t) - p.Commit = "c3" - p.Modules["app"] = &inventory.PlanModule{State: "asked", AskedAt: &asked, Build: id} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - planBuiltSaying(ctx, g.open, "app", "c3", "", asked, id, result.MadeWords()) - p = g.plan(t) - if m := p.Modules["app"]; m.State != "built" || !strings.HasPrefix(m.Made, "reused from the check of novox/mesh-catalog#9") { - t.Fatalf("the walk keeps %+v", m) - } - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p = g.plan(t) - gate := p.Modules["app"].Gate - if gate == nil || gate.To != "c3" || gate.Verdict != inventory.GatePassed { - t.Fatalf("the gate judged %+v, not the reused build", gate) - } - if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c3" { - t.Fatalf("the first machine was sent %v, not the reused build", sent) - } -} // A walk's leave names the builds its own pull requests' verdicts made at the head each merge merged — never an // earlier head's — read once per walk, and none when they cannot be read: the builder takes nothing a verdict does diff --git a/cmd/mesh-controller/check_siblings.go b/cmd/mesh-controller/check_siblings.go index b2e5626e..df08f3f5 100644 --- a/cmd/mesh-controller/check_siblings.go +++ b/cmd/mesh-controller/check_siblings.go @@ -127,10 +127,15 @@ func siblingsFor(p link.PullUpdated, entries []inventory.Entry, beside map[strin // could not be learned (ADR 0382 §1): a check never waits on mesh-delivery past groupAskWithin and never fails for it, // so a fix to mesh-delivery is still checked without it. func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry) { + lines, err := groupsAsked(ctx, deliverySeatHeld(entries)) + siblingsFrom(request, p, entries, lines, err) +} + +// siblingsFrom is withSiblings from mesh-delivery's answer already had. +func siblingsFrom(request *link.BuildRequest, p link.PullUpdated, entries []inventory.Entry, lines []groupsLine, err error) { if request.Check == nil { return } - lines, err := groupsAsked(ctx, deliverySeatHeld(entries)) if err != nil { request.Check.SiblingsUnknown = fmt.Sprintf("mesh-delivery did not say the group's pull requests (%v): "+ "the check reads no siblings", err) @@ -138,3 +143,23 @@ func withSiblings(ctx context.Context, request *link.BuildRequest, p link.PullUp } request.Check.Siblings, request.Check.SiblingsUnknown = siblingsFor(p, entries, request.Check.Beside, lines) } + +// composedMember says the pull request is a member of a delivery group whose gate is the composed check's (novox/hq +// ADR 0403 clauses 1 and 7): the group has two members or more and mesh-delivery says so on its line. The group is +// found as siblingsFor finds it, by its head branch name, or, for a recheck that carries none, by the pull request's +// repository and number — not by siblings, which name only what the check reads beside it. +func composedMember(p link.PullUpdated, lines []groupsLine) bool { + own := strings.ToLower(p.Owner + "/" + p.Repo) + for _, l := range lines { + found := p.Head != "" && l.ID == p.Head + for _, m := range l.Details { + if p.Head == "" && p.Number != 0 && m.Number == p.Number && strings.ToLower(m.Repository) == own { + found = true + } + } + if found { + return l.ComposedGate && max(len(l.Details), len(l.Members)) >= 2 + } + } + return false +} diff --git a/cmd/mesh-controller/checks.go b/cmd/mesh-controller/checks.go index e0104e34..b564f0c7 100644 --- a/cmd/mesh-controller/checks.go +++ b/cmd/mesh-controller/checks.go @@ -3,7 +3,6 @@ package main import ( "context" "encoding/json" - "errors" "fmt" "path" "regexp" @@ -155,6 +154,24 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { return err } scope := pullScope(p, entries, read, edges) + // **A cycle of edges fails the check, naming every module and edge** (novox/hq ADR 0419 clause 6), a lone pull + // request's as a group's composed check is: no order exists for what it changes. + alone := orderMember{ID: p.Owner + "/" + p.Repo + "@" + twelve(p.Commit), Repository: p.Owner + "/" + p.Repo, Base: p.Base, + Head: p.Commit, Number: p.Number, Paths: p.Paths, PathsTruncated: p.PathsTruncated, Removed: p.Removed, BaseSHA: p.BaseSHA} + if failing, cyclic, err := cycleVerdict(ctx, "", []orderMember{alone}, + map[string]orderReach{alone.ID: {Moved: scope.Modules}}, edges, entries); err != nil { + return err + } else if cyclic { + failing.Commit, failing.Trunk = p.Commit, twelve(p.BaseSHA) + fmt.Printf("%s/%s#%d (%.8s): %s\n", p.Owner, p.Repo, p.Number, p.Commit, failing.Summary) + sayChecked(ctx, failing) + return nil + } + // **A member of a group whose gate is the composed check's runs its repository's check alone** (novox/hq ADR 0403 + // clause 1): mesh-delivery is asked once, and says so on the group's line. Told nothing, the pull request is + // checked alone, gate included (clause 6). + lines, groupsErr := groupsAsked(ctx, deliverySeatHeld(entries)) + composed := groupsErr == nil && composedMember(p, lines) // The change plan of the commit at hand (ADR 0238): what a merge of it would build and send, posted // with the verdict whatever the verdict is. plan := changePlanOf(p.Owner+"/"+p.Repo, p.Base, p.Commit, scope.Reach, entries, func(module string) (inventory.Upgrade, bool) { @@ -163,6 +180,10 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { }) fmt.Print(planText(plan)) direct := directVerdict(p, plan) + if composed { + scope.Modules, scope.Dependents, scope.New, scope.Manifests = nil, nil, nil, nil + composedGate(&direct) + } switch { case !scope.gated() && !scope.Mesh: fmt.Printf("%s/%s#%d (%.8s): %s, and the repository is not the mesh's: said, nothing run\n", @@ -180,8 +201,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { return err } request.Check.Plan = &plan + request.Check.Composed = composed // **The group's other open pull requests are read beside it** (novox/hq ADR 0382 §1). - withSiblings(ctx, &request, p, entries) + siblingsFrom(&request, p, entries, lines, groupsErr) seat := buildSeatHeld(ctx) ask, err := askOverOn(seat) if err != nil { @@ -195,6 +217,9 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { // issue 974): the check queue keeps one request a pull request, and a running check watches its key. Nothing is asked // of the forge here (issue 726). what := "its own merge-check.sh alone: " + noModuleTouched + if composed { + what = "its own merge-check.sh alone: the gate is its group's composed check's" + } if scope.gated() { what = fmt.Sprintf("the gate over %s (a merge would build %d module(s) in %d tier(s))", strings.Join(append(append([]string{}, scope.Modules...), prefixedAll("new:", scope.New)...), ", "), @@ -205,6 +230,20 @@ func (f following) PullUpdated(ctx context.Context, p link.PullUpdated) error { return nil } +// composedWords is what a group member's own verdict says of its gate layer (novox/hq ADR 0403 clause 1). +const composedWords = "judged by its group's composed check" + +// gateComposed is a gate layer's verdict when no gate was run for the member alone: never a pass. Mesh-delivery reads it +// as the composed check's to judge, and the forge's holder sets no mesh/merge-gate from it. +const gateComposed = "composed" + +// composedGate says a group member's own verdict has no gate of its own: its gate layer, and the verdict that is +// the gate's, read "composed" in place of a pass. +func composedGate(c *link.Checked) { + c.Verdict, c.Summary = gateComposed, composedWords + c.Gate = &link.CheckLayer{Verdict: gateComposed, Summary: composedWords} +} + // directVerdict is the verdict of a change no build judges — it touches no module, and either the repository is not // the mesh's or the head holds no merge-check.sh. **It names the trunk commit it holds for** (novox/hq ADR 0323 §3): // the base branch's head the forge announced with the head, so the merge tool can weigh it against the trunk's head @@ -254,12 +293,7 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop if err != nil { return link.BuildRequest{}, err } - // **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358): asked of the forge once. - release, err := releaseBranchOf(ctx, p, scope, entries) - if err != nil { - return link.BuildRequest{}, err - } - beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone, release) + beside, err := besideOf(entries, func(module string) string { return current[module].Commit }, clone) if err != nil { return link.BuildRequest{}, err } @@ -274,17 +308,14 @@ func checkRequestFor(ctx context.Context, open *stores, p link.PullUpdated, scop Check: &link.CheckRequest{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Base: p.Base, Trunk: p.BaseSHA, Ready: p.Ready, Paths: p.Paths, Beside: beside, Modules: scope.Modules, Dependents: scope.Dependents, New: scope.New, Manifests: scope.Manifests, Judge: scope.Judge, Builds: builds, Named: namedForTheCheck(ctx, p, builds), - Release: release, Running: runningBy(ctx)}, + Running: runningBy(ctx)}, }, nil } // besideOf is what a check reads beside its own repository, by the directory it is found under: each core repository -// by the module the mesh builds from it, at besideRefs — or, for a pull request into a release branch (novox/hq ADR -// 0358 §3), at its branch of that name where it has one and at its trunk otherwise, the controller's main kept for -// its judge, and mesh-lab at its trunk whatever the base: its replays run with the container runtime's socket. running -// is the commit the mesh runs of a module. +// by the module the mesh builds from it, at besideRefs. running is the commit the mesh runs of a module. func besideOf(entries []inventory.Entry, running func(module string) string, - clone func(inventory.Source) (string, error), release string) (map[string]link.CheckedOut, error) { + clone func(inventory.Source) (string, error)) (map[string]link.CheckedOut, error) { beside := map[string]link.CheckedOut{} for _, e := range entries { dir, core := coreModules[e.Manifest.Module] @@ -296,34 +327,15 @@ func besideOf(entries []inventory.Entry, running func(module string) string, return nil, err } refs := besideRefs(dir, running(e.Manifest.Module)) - at := func(dir, trunk string) link.CheckedOut { - if release == "" { - return link.CheckedOut{Repository: url, Ref: refs[dir]} - } - if trunk == "" { - trunk = "main" - } - return link.CheckedOut{Repository: url, Ref: release, Else: trunk} - } - beside[dir] = at(dir, followedBranch(e.Source.Ref)) + beside[dir] = link.CheckedOut{Repository: url, Ref: refs[dir]} if dir == "mesh-controller" { // The controller's main stays its main: it judges only a check the running controller predates. beside["mesh-controller-main"] = link.CheckedOut{Repository: url, Ref: refs["mesh-controller-main"]} - if release != "" { - // mesh-controller has the release branch: that is what makes it one (ADR 0358 §1). - beside[dir] = link.CheckedOut{Repository: url, Ref: release} - } if e.Source.Seat != "" { for _, sibling := range []string{"mesh-lab", "mesh-sdk"} { if url, err := clone(inventory.Source{Seat: e.Source.Seat, Repository: siblingOf(e.Source.Repository, sibling)}); err == nil { - b := link.CheckedOut{Repository: url, Ref: refs[sibling]} - // **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container - // runtime's socket, so they are only ever reviewed code — never a branch anyone may create. - if release != "" && sibling != "mesh-lab" { - b = link.CheckedOut{Repository: url, Ref: release, Else: "main"} - } - beside[sibling] = b + beside[sibling] = link.CheckedOut{Repository: url, Ref: refs[sibling]} } } } @@ -332,47 +344,6 @@ func besideOf(entries []inventory.Entry, running func(module string) string, return beside, nil } -// releaseBranchOf is the release branch a pull request merges into (novox/hq ADR 0358 §1): its base, when that is not -// its repository's trunk and mesh-controller has a branch of that name; empty otherwise, and the pull request is -// checked as before it. **A base that is a trunk asks nothing**: the branch a module built from the repository follows -// (main when it records none), or mesh-controller's own trunk. Any other base is asked of the forge once, and an -// answer it cannot give is an error, never read as "no release branch". -func releaseBranchOf(ctx context.Context, p link.PullUpdated, scope checkScope, entries []inventory.Entry) (string, error) { - base := p.Base - if base == "" { - return "", nil - } - if scope.From != nil && base == trunkOf(scope.From.Source) { - return "", nil - } - var controller *inventory.Entry - for i, e := range entries { - if e.Manifest.Module == "mesh-controller" && !e.Provided && e.Source.Repository != "" { - controller = &entries[i] - break - } - } - if controller == nil || base == trunkOf(controller.Source) { - return "", nil - } - if !tryRef.MatchString(base) || strings.Contains(base, "..") { - return "", nil // no branch the forge could hold: checked as before - } - owner, repo := sourceOwner(controller.Source.Repository), sourceName(controller.Source.Repository) - if owner == "" || repo == "" { - return "", nil - } - has, err := forgeHasBranch(ctx, owner, repo, base) - if err != nil { - return "", fmt.Errorf("whether %s/%s#%d merges into a release branch cannot be told — the forge did not say "+ - "whether %s/%s has a branch %s (novox/hq ADR 0358): %w", p.Owner, p.Repo, p.Number, owner, repo, base, err) - } - if !has { - return "", nil - } - return base, nil -} - // trunkOf is the branch a module's source follows: main when it records none, or records a commit. func trunkOf(s inventory.Source) string { if t := strings.TrimSpace(followedBranch(s.Ref)); t != "" { @@ -390,46 +361,6 @@ func sourceName(repository string) string { return parts[len(parts)-1] } -// forgeHasBranch asks the forge, through its module's tools on the bus, whether a repository has a branch. A -// variable so a test needs no forge. -var forgeHasBranch = func(ctx context.Context, owner, repo, branch string) (bool, error) { - js, err := aBus() - if err != nil { - return false, err - } - defer js.Close() - raw, _ := json.Marshal(map[string]any{"path": "/repos/" + owner + "/" + repo + "/branches/" + branch}) - answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second) - if err != nil { - return false, err - } - return branchAnswered(answer, branch) -} - -// branchAnswered reads the forge's answer about a branch: a 404 is "no such branch"; an answer naming the branch is -// "it has it"; anything else — another error, an unreadable answer, one naming no branch or another — is an error, -// never read as "no release branch". -func branchAnswered(answer link.Answer, branch string) (bool, error) { - if answer.Error != "" { - if strings.Contains(answer.Error, " 404 ") { - return false, nil - } - return false, errors.New(answer.Error) - } - var found struct { - Result struct { - Name string `json:"name"` - } `json:"result"` - } - if err := json.Unmarshal(answer.Result, &found); err != nil { - return false, fmt.Errorf("the forge's answer about branch %s cannot be read: %w", branch, err) - } - if found.Result.Name != branch { - return false, fmt.Errorf("the forge answered about branch %s naming %q", branch, found.Result.Name) - } - return true, nil -} - // namedForTheCheck are the builds earlier checks of the same pull request at the same head named, read once for the // check (novox/hq issue 703): asked again because only the trunk moved, the check keeps what they built of a key // still the same. Another pull request's build, or one of an earlier head, is never kept. None when they cannot be @@ -608,7 +539,6 @@ func checkedOf(result link.BuildResult) link.Checked { // repositories read beside it. c.Trunk, c.Beside, c.Took = result.Check.Trunk, result.Check.Beside, result.Check.Took c.Built = result.Check.Built - c.Release, c.JudgedBy, c.BesideAt = result.Check.Release, result.Check.JudgedBy, result.Check.BesideAt c.Validator = result.Check.Validator c.Siblings, c.SiblingsUnknown, c.JudgeFrom = result.Check.Siblings, result.Check.SiblingsUnknown, result.Check.JudgeFrom if c.Gate == nil { @@ -628,6 +558,12 @@ func checkedOf(result link.BuildResult) link.Checked { c.Gate = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} c.RepoCheck = &link.CheckLayer{Verdict: c.Verdict, Summary: c.Summary} } + // **A group member's own check has no gate of its own** (novox/hq ADR 0403 clause 1): what ran was the repository's + // check alone, and its gate layer is "composed", never a pass. A failure or an error stays what it was. + if result.Checked != nil && result.Checked.Composed && result.Check != nil && c.Gate != nil && + (c.Gate.Verdict == "pass" || c.Gate.Verdict == "warning") { + composedGate(&c) + } if result.Checked != nil && c.Gate != nil && len(c.Gate.Modules) == 0 { c.Gate.Modules = append(append([]string{}, result.Checked.Modules...), prefixedAll("new:", result.Checked.New)...) } diff --git a/cmd/mesh-controller/checks_integration_test.go b/cmd/mesh-controller/checks_integration_test.go new file mode 100644 index 00000000..447c4e2e --- /dev/null +++ b/cmd/mesh-controller/checks_integration_test.go @@ -0,0 +1,56 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + was := checkEvents + checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} + t.Cleanup(func() { checkEvents = was }) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3} + checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey}) + checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"}) + select { + case m := <-heard: + var c link.Checked + _ = json.Unmarshal(m.Data, &c) + if c.ID != "broken" || c.Verdict != "error" { + t.Fatalf("the first verdict said is %+v; want only the broken check's error", c) + } + case <-time.After(5 * time.Second): + t.Fatal("nothing was said") + } + select { + case m := <-heard: + t.Fatalf("a second verdict was said: %s", m.Data) + case <-time.After(300 * time.Millisecond): + } +} diff --git a/cmd/mesh-controller/checks_queue_test.go b/cmd/mesh-controller/checks_queue_test.go index 98dfb31a..d79c6207 100644 --- a/cmd/mesh-controller/checks_queue_test.go +++ b/cmd/mesh-controller/checks_queue_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/checks_test.go b/cmd/mesh-controller/checks_test.go index dd6a3b8e..24e37f3d 100644 --- a/cmd/mesh-controller/checks_test.go +++ b/cmd/mesh-controller/checks_test.go @@ -1,16 +1,9 @@ package main import ( - "encoding/json" "errors" "strings" "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -298,44 +291,3 @@ func TestARecheckedHeadsDirectVerdictNamesItsTrunk(t *testing.T) { // A check stopped by its pull request's key moving on says no verdict (issue 974, review 2): an error said for it would name the // live head, and the newer request is the one that judges. A check that failed for any other reason still says its error. -func TestACheckStoppedByItsKeyIsAnnouncedAsNoVerdict(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - was := checkEvents - checkEvents = link.OverNATS{Conn: js.Conn(), JS: legacy} - t.Cleanup(func() { checkEvents = was }) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyChecked), heard) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 3} - checked(t.Context(), link.BuildResult{ID: "stopped", Ref: "abc", Checked: asked, Failed: link.SupersededByKey}) - checked(t.Context(), link.BuildResult{ID: "broken", Ref: "abc", Checked: asked, Failed: "the facts snapshot cannot be read"}) - select { - case m := <-heard: - var c link.Checked - _ = json.Unmarshal(m.Data, &c) - if c.ID != "broken" || c.Verdict != "error" { - t.Fatalf("the first verdict said is %+v; want only the broken check's error", c) - } - case <-time.After(5 * time.Second): - t.Fatal("nothing was said") - } - select { - case m := <-heard: - t.Fatalf("a second verdict was said: %s", m.Data) - case <-time.After(300 * time.Millisecond): - } -} diff --git a/cmd/mesh-controller/composed_member_check_test.go b/cmd/mesh-controller/composed_member_check_test.go new file mode 100644 index 00000000..f5bbc15b --- /dev/null +++ b/cmd/mesh-controller/composed_member_check_test.go @@ -0,0 +1,68 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A member of a group whose gate is the composed check's is asked of the build seat with no module: its repository's +// check alone (novox/hq ADR 0403 clause 1). With no word from mesh-delivery, or one that does not say the gate is +// composed, it is checked alone with its gate (clause 6). + +func askedForMember(t *testing.T, answer []groupsLine, answerErr error) link.CheckRequest { + t.Helper() + open := aMesh(t) + ctx := t.Context() + js, _ := aBuildQueue(t) + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(nil) }) + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, + inventory.Source{Repository: "novox/mesh-controller", Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + was := groupsAsked + groupsAsked = func(context.Context, bool) ([]groupsLine, error) { return answer, answerErr } + t.Cleanup(func() { groupsAsked = was }) + p := link.PullUpdated{Owner: "novox", Repo: "mesh-controller", Number: 346, Base: "main", Head: "feat/x", Commit: headController, + Paths: []string{"main.go"}, ModuleDirsSaid: true, MergeCheck: true, MergeCheckSaid: true} + if err := (following{open: open, oneShot: true}).PullUpdated(ctx, p); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil || len(q.Asks) != 1 { + t.Fatalf("asks %d: %v", len(q.Asks), err) + } + return *q.Asks[0].Request.Check +} + +func TestAComposedMembersCheckIsItsRepositoryChecksAlone(t *testing.T) { + line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true)) + line.ComposedGate = true + c := askedForMember(t, []groupsLine{line}, nil) + if len(c.Modules) != 0 || len(c.New) != 0 || len(c.Dependents) != 0 || !c.Composed { + t.Fatalf("the member's check is asked as %+v; want no module and composed", c) + } +} + +func TestAMemberOfAGroupOfAnOlderMeshDeliveryKeepsItsOwnGate(t *testing.T) { + line := group("feat/x", member("novox/mesh-controller", 346, headController, "main", true), member("novox/mesh-catalog", 350, headHost, "main", true)) + c := askedForMember(t, []groupsLine{line}, nil) + if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed { + t.Fatalf("an older mesh-delivery's member is asked as %+v; want its own gate", c) + } +} + +func TestAMemberWhoseGroupCannotBeLearnedIsCheckedAloneWithItsGate(t *testing.T) { + c := askedForMember(t, nil, context.DeadlineExceeded) + if !strings.Contains(strings.Join(c.Modules, ","), "mesh-controller") || c.Composed || + !strings.Contains(c.SiblingsUnknown, "reads no siblings") { + t.Fatalf("the member is asked as %+v; want its gate and why its siblings are unknown", c) + } +} diff --git a/cmd/mesh-controller/composed_verdict_test.go b/cmd/mesh-controller/composed_verdict_test.go new file mode 100644 index 00000000..a8573833 --- /dev/null +++ b/cmd/mesh-controller/composed_verdict_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// A delivery group's gate is the composed check's, and its members' own checks run their repository's check alone +// (novox/hq ADR 0403 clauses 1 and 7). Pure: no store, no bus. + +// A member of a group whose line says composed_gate is a composed member; one whose group says nothing of it (an older +// mesh-delivery), one of a group of a single pull request and a lone pull request are not. +func TestAMemberOfAComposedGroupIsKnownByTheGroupsLine(t *testing.T) { + two := func(composed bool) groupsLine { + l := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true), member("novox/mesh-controller", 346, headController, "main", true)) + l.ComposedGate = composed + return l + } + p := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Head: "feat/x", Commit: headHost} + recheck := link.PullUpdated{Owner: "novox", Repo: "mesh-host", Number: 114, Base: "main", Commit: headHost} + for name, c := range map[string]struct { + p link.PullUpdated + lines []groupsLine + want bool + }{ + "composed": {p, []groupsLine{two(true)}, true}, + "a recheck, by its number": {recheck, []groupsLine{two(true)}, true}, + "an older mesh-delivery": {p, []groupsLine{two(false)}, false}, + "another group": {p, []groupsLine{group("feat/y", member("novox/mesh-host", 1, headHost, "main", true))}, false}, + "no group": {p, nil, false}, + } { + if got := composedMember(c.p, c.lines); got != c.want { + t.Errorf("%s: composed %v, want %v", name, got, c.want) + } + } + alone := group("feat/x", member("novox/mesh-host", 114, headHost, "main", true)) + alone.ComposedGate = true + if composedMember(p, []groupsLine{alone}) { + t.Error("a group of a single pull request is composed") + } +} + +// A composed member's verdict never says its gate passed: the gate layer, and the verdict that is the gate's, read +// "composed"; a failure or an error stays what it was. +func TestAComposedMembersVerdictHasNoPassingGate(t *testing.T) { + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114, Composed: true} + pass := link.BuildResult{ID: "b", Ref: headHost, Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", Summary: "no module touched", + Gate: &link.CheckLayer{Verdict: "pass", Summary: "no module touched"}, RepoCheck: &link.CheckLayer{Verdict: "pass", Summary: "ok"}}} + c := checkedOf(pass) + if c.Gate == nil || c.Gate.Verdict != "composed" || c.Verdict != "composed" || c.RepoCheck == nil || c.RepoCheck.Verdict != "pass" { + t.Fatalf("the member's verdict says %+v gate %+v", c, c.Gate) + } + asked.Composed = false + if c := checkedOf(pass); c.Gate.Verdict != "pass" || c.Verdict != "pass" { + t.Fatalf("a member of no composed group has its own gate: %+v", c.Gate) + } + asked.Composed = true + failing := pass + failing.Check = &link.CheckOutcome{Verdict: "fail", Summary: "conflict", Gate: &link.CheckLayer{Verdict: "fail", Summary: "conflict"}, + RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "conflict"}} + if c := checkedOf(failing); c.Gate.Verdict != "fail" || c.Verdict != "fail" { + t.Fatalf("a failure was read as composed: %+v", c.Gate) + } + broken := link.BuildResult{ID: "b", Ref: headHost, Checked: asked, Failed: "the facts snapshot cannot be read"} + if c := checkedOf(broken); c.Verdict != "error" || c.Gate.Verdict != "error" { + t.Fatalf("an error was read as composed: %+v", c) + } +} + +// The composed verdict on the bus carries every member, each with its trunk commit, for the forge's holder to set each +// member's mesh/merge-gate from it. +func TestTheComposedVerdictCarriesEveryMember(t *testing.T) { + asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-host", Number: 114, Group: "feat/x", + Members: []link.GroupMember{{Owner: "novox", Repo: "mesh-controller", Number: 346, Ref: headController}}} + c := checkedOf(link.BuildResult{ID: "g", Ref: headHost, Checked: asked, Check: &link.CheckOutcome{Verdict: "pass", Summary: "ok", + Trunk: "aaaaaaaaaaaa", Members: []link.CheckedMember{{Owner: "novox", Repo: "mesh-controller", Commit: headController[:12], Trunk: "bbbbbbbbbbbb"}}}}) + body, _ := json.Marshal(c) + for _, want := range []string{`"group":"feat/x"`, `"members":[{"owner":"novox","repo":"mesh-host","number":114,"commit":"` + headHost + `","trunk":"aaaaaaaaaaaa"}`, + `{"owner":"novox","repo":"mesh-controller","number":346,"commit":"` + headController + `","trunk":"bbbbbbbbbbbb"}`} { + if !strings.Contains(string(body), want) { + t.Fatalf("the composed verdict lacks %s:\n%s", want, body) + } + } +} + +// Members whose manifests order their modules in a cycle fail the composed check, naming every module and edge, and no +// ask is made of the build seat. +func TestAGroupInACycleFailsItsComposedCheckWithoutABuild(t *testing.T) { + members := []orderMember{ + {ID: "novox/x@" + headHost[:12], Repository: "novox/x", Base: "main", Head: headHost, Number: 7, Paths: []string{"module.json"}}, + {ID: "novox/y@" + headController[:12], Repository: "novox/y", Base: "main", Head: headController, Number: 9, Paths: []string{"module.json"}}, + } + reach := map[string]orderReach{members[0].ID: {Moved: []string{"mx"}}, members[1].ID: {Moved: []string{"my"}}} + was, wasParent := manifestAt, parentOf + t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) + parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } + manifestAt = func(_ context.Context, _, repo, _, ref string) (catalogue.Manifest, bool, error) { + if ref == "main" { + return catalogue.Manifest{}, false, nil // what the pull requests merge into holds no such manifest + } + if repo == "x" { + return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}}, + Invokes: []string{"seat:s.b"}}, true, nil + } + return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}}, + Invokes: []string{"seat:s.a"}}, true, nil + } + c, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, []inventory.Edge(nil), nil) + if err != nil || !cyclic { + t.Fatalf("no cycle found: %v", err) + } + if c.Verdict != "fail" || c.Group != "feat/cycle" || len(c.Members) != 2 || c.Gate == nil || c.Gate.Verdict != "fail" { + t.Fatalf("the verdict is %+v", c) + } + for _, w := range []string{"mx", "my", "s.a", "s.b"} { + if !strings.Contains(c.Summary, w) { + t.Errorf("the cycle does not name %s: %s", w, c.Summary) + } + } + // Members with no edge between them are no cycle. + manifestsReadMu.Lock() + clear(manifestsRead) + manifestsReadMu.Unlock() + manifestAt = func(context.Context, string, string, string, string) (catalogue.Manifest, bool, error) { + return catalogue.Manifest{}, false, nil + } + if _, cyclic, err := cycleVerdict(context.Background(), "feat/cycle", members, reach, nil, nil); err != nil || cyclic { + t.Fatalf("a cycle where there is none: %v %v", cyclic, err) + } +} diff --git a/cmd/mesh-controller/condition_looks_test.go b/cmd/mesh-controller/condition_looks_test.go index 1c79015a..9295188c 100644 --- a/cmd/mesh-controller/condition_looks_test.go +++ b/cmd/mesh-controller/condition_looks_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/conditions_brief_integration_test.go b/cmd/mesh-controller/conditions_brief_integration_test.go new file mode 100644 index 00000000..5edd1090 --- /dev/null +++ b/cmd/mesh-controller/conditions_brief_integration_test.go @@ -0,0 +1,43 @@ +//go:build integration + +package main + +import ( + "fmt" + "strings" + "testing" + "time" +) + +func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) { + var found []string + for i := 0; i < 689; i++ { + found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i)) + } + run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found}, + {ID: "D1", Verdict: "passed"}}} + answer, err := verdictAnswer(run, time.Now(), "") + if err != nil { + t.Fatal(err) + } + brief := answer["run"].(doctorRun).Probes[0] + if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") { + t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More) + } + if len(run.Probes[0].Found) != 689 { + t.Fatal("the overview cut the run it was made from") + } + whole, err := verdictAnswer(run, time.Now(), "d14") + if err != nil { + t.Fatal(err) + } + if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" { + t.Fatalf("doctor probe=D14 answered %+v", p) + } + if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil { + t.Fatal("a probe the run does not have was answered") + } + if !strings.Contains(doctorText(answer), "probe=D14` gives them all") { + t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer)) + } +} diff --git a/cmd/mesh-controller/conditions_brief_test.go b/cmd/mesh-controller/conditions_brief_test.go index 8fae4898..135c6336 100644 --- a/cmd/mesh-controller/conditions_brief_test.go +++ b/cmd/mesh-controller/conditions_brief_test.go @@ -74,35 +74,3 @@ func TestAListOfConditionsCarriesEachInBrief(t *testing.T) { // The self-check's verdict carries at most overviewFindings of one probe's findings, with their count and // how to read them all; `doctor probe=` gives that probe whole (novox/hq issue 314). -func TestAVerdictCarriesAProbesFindingsInBrief(t *testing.T) { - var found []string - for i := 0; i < 689; i++ { - found = append(found, fmt.Sprintf("delivery d-%012x has been checking for 13h, past its bound of 45m", i)) - } - run := doctorRun{Run: "run-1", Probes: []probeVerdict{{ID: "D14", Verdict: "failed", Found: found}, - {ID: "D1", Verdict: "passed"}}} - answer, err := verdictAnswer(run, time.Now(), "") - if err != nil { - t.Fatal(err) - } - brief := answer["run"].(doctorRun).Probes[0] - if len(brief.Found) != overviewFindings || brief.FoundCount != 689 || !strings.Contains(brief.More, "probe=D14") { - t.Fatalf("the verdict carries %d findings of D14, count %d, more %q", len(brief.Found), brief.FoundCount, brief.More) - } - if len(run.Probes[0].Found) != 689 { - t.Fatal("the overview cut the run it was made from") - } - whole, err := verdictAnswer(run, time.Now(), "d14") - if err != nil { - t.Fatal(err) - } - if p := whole["run"].(doctorRun).Probes; len(p) != 1 || len(p[0].Found) != 689 || p[0].More != "" { - t.Fatalf("doctor probe=D14 answered %+v", p) - } - if _, err := verdictAnswer(run, time.Now(), "D99"); err == nil { - t.Fatal("a probe the run does not have was answered") - } - if !strings.Contains(doctorText(answer), "probe=D14` gives them all") { - t.Fatalf("the text does not say how to read them all:\n%s", doctorText(answer)) - } -} diff --git a/cmd/mesh-controller/conditions_integration_test.go b/cmd/mesh-controller/conditions_integration_test.go new file mode 100644 index 00000000..a2fb25f0 --- /dev/null +++ b/cmd/mesh-controller/conditions_integration_test.go @@ -0,0 +1,32 @@ +//go:build integration + +package main + +import ( + "errors" + "strings" + "testing" +) + +func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) { + open := aMesh(t) + _, store := withConditionsInMemory(t) + store.SetFail(errors.New("the bus is away")) + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + if asked.well() || asked.conditionsUnread == "" { + t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread) + } + said := printed(t, func() error { return printStatus(asked) }) + if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") { + t.Fatalf("%s", said) + } + store.SetFail(nil) + asked, _ = theThreeQuestions(t.Context(), open) + said = printed(t, func() error { return printStatus(asked) }) + if asked.well() && !strings.Contains(said, "no open conditions;") { + t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said) + } +} diff --git a/cmd/mesh-controller/conditions_test.go b/cmd/mesh-controller/conditions_test.go index 35bdefe2..24e1ee7b 100644 --- a/cmd/mesh-controller/conditions_test.go +++ b/cmd/mesh-controller/conditions_test.go @@ -1,7 +1,6 @@ package main import ( - "errors" "strings" "testing" @@ -83,25 +82,3 @@ func TestASilenceThroughTheVerbHoldsAndTheConditionStaysOpen(t *testing.T) { } // **Conditions that cannot be read are not none open**: status says so, and is not well. -func TestUnreadableConditionsAreNotAWellMesh(t *testing.T) { - open := aMesh(t) - _, store := withConditionsInMemory(t) - store.SetFail(errors.New("the bus is away")) - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - if asked.well() || asked.conditionsUnread == "" { - t.Fatalf("well with its conditions unread: %+v", asked.conditionsUnread) - } - said := printed(t, func() error { return printStatus(asked) }) - if !strings.HasPrefix(said, "the open conditions could NOT be read") || strings.Contains(said, "no open conditions") { - t.Fatalf("%s", said) - } - store.SetFail(nil) - asked, _ = theThreeQuestions(t.Context(), open) - said = printed(t, func() error { return printStatus(asked) }) - if asked.well() && !strings.Contains(said, "no open conditions;") { - t.Fatalf("the all-well sentence does not say no conditions are open:\n%s", said) - } -} diff --git a/cmd/mesh-controller/config_moved_integration_test.go b/cmd/mesh-controller/config_moved_integration_test.go new file mode 100644 index 00000000..7c930d12 --- /dev/null +++ b/cmd/mesh-controller/config_moved_integration_test.go @@ -0,0 +1,467 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/overlay" +) + +func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for i, name := range []string{"workstation", "desktop"} { + record, err := inv.AddNode(ctx, name) + if err != nil { + t.Fatal(err) + } + if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { + t.Fatal(err) + } + if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ + map[string]any{"name": "container-runtime", "present": true}, + map[string]any{"name": "systemd", "present": true}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { + t.Fatal(err) + } + } + raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) + if err != nil { + t.Fatal(err) + } + var w configuredMesh + if err := json.Unmarshal(raw, &w); err != nil { + t.Fatal(err) + } + list, err := catalogue.ParsePredefined(w.Predefined) + if err != nil { + t.Fatal(err) + } + catalogue.UsePredefined(list) + served := map[string]map[string]any{} + for _, text := range w.Modules { + var shape map[string]json.RawMessage + if err := json.Unmarshal(text, &shape); err != nil { + t.Fatal(err) + } + var name string + _ = json.Unmarshal(shape["module"], &name) + if s, ok := shape["serves-checked"]; ok { + var serves map[string]any + if err := json.Unmarshal(s, &serves); err != nil { + t.Fatal(err) + } + served[name] = serves + delete(shape, "serves-checked") + } + text, err := json.Marshal(shape) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(text) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + register(t, open, m) + } + for _, l := range w.Layers { + values, err := json.Marshal(l.Values) + if err != nil { + t.Fatal(err) + } + if l.Node == "" { + inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) + continue + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(values)) + } + for _, module := range movedKeys(w.On) { + for _, node := range w.On[module] { + if _, err := assign(ctx, open, node, module); err != nil { + t.Fatalf("assign %s %s: %v", node, module, err) + } + } + } + return open, w, served +} + +func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + before := toldEveryMachine(t, open, served, w.On) + listedBefore := effectiveEverywhere(t, open, w) + factsBefore, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + + asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") + if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { + t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) + } + // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). + if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || + !strings.HasSuffix(s[0].From[0], ": timeZone") { + t.Errorf("nodered's time zone moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Key == "timeZone" { + t.Errorf("a key is set under the file's name: %+v", s) + } + } + // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's + // consequences), and its whole-mesh layer sets no module value. + if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { + t.Errorf("claude-code's managed settings move as %+v", s) + } + for _, node := range fourMachines { + if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { + t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) + } + } + // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that + // machine's layer: power's on the workstation is the workstation's, its own elsewhere. + if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { + t.Errorf("power's key on the workstation moves as %+v", s) + } + if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { + t.Errorf("power's key on the anchor moves as %+v", s) + } + for _, s := range asked.Sets { + if s.Level == "node" { + t.Errorf("a machine's layer became node config: %+v", s) + } + } + var stays []string + for _, s := range asked.Stays { + stays = append(stays, strings.Fields(s)[0]) + } + sort.Strings(stays) + if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", + "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { + t.Errorf("the layers that stay are %v, want those of %v", stays, want) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("a dry run changed what a machine is told") + } + + moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") + if err != nil || moved.DryRun || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if len(config.sets) != len(asked.Sets) { + t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) + } + if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' + and module in (select name from module where manifest ? 'config')`); len(held) != 0 { + t.Fatalf("a layer of a module in the config form still holds values: %v", held) + } + after := toldEveryMachine(t, open, served, w.On) + for _, node := range fourMachines { + if after[node] != before[node] { + t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) + } + } + if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { + t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) + } + // And config lists what settings listed: each module's resolved value on each machine is the value settings + // said before the move. + resolved, err := config.Resolved(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range resolved { + said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] + if r.Source == "unset" { + if listed && said != nil { + t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) + } + continue + } + var v any + _ = json.Unmarshal(r.Value, &v) + if !listed || !reflect.DeepEqual(v, said) { + t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) + } + } + + // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and + // each machine's declaration composes to the digest it did before the move. + factsAfter, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. + digests := func(f snapshotFacts) ([]string, string) { + var out []string + zone := "" + for _, m := range f.Machines { + out = append(out, m.Declaration.Digest) + for _, l := range m.Settings { + if l.Module == "nodered" { + zone = compactJSON(l.Values["time-zone"]) + } + } + } + sort.Strings(out) + return out, zone + } + was, _ := digests(factsBefore) + now, zone := digests(factsAfter) + if zone != `"Europe/Brussels"` { + t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) + } + if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { + t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) + } + for _, s := range factsAfter.Settings { + if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && + len(s.Values) > 0 { + t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) + } + } + + config.sets = nil + again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") + if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { + t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) + } + if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { + t.Fatal("the move asked again changed what a machine is told") + } +} + +func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + + inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' + where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) + answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || answer.Switched || len(config.sets) > 0 { + t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) + } + for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", + `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) + inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) + inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) + + // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. + config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { + shelf, _ := inv.Catalogue(ctx) + return shelf["grafana"].Settings + }())[0]] = "set before the move" + answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") + if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { + t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) + } + if moved, _ := inv.ConfigMoved(ctx); moved.Moved { + t.Fatal("the switch is on after a refused move") + } + if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { + t.Fatal("a refused move emptied the layers") + } +} + +func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { + open, w, served := aConfiguredMesh(t) + ctx := t.Context() + inv := open.inventory + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + savedRead, savedAsk := readConfigValues, configAsk + readConfigValues, configAsk = config.laid, config.ask + t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) + if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { + t.Fatal(err) + } + + shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) + for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, + "mesh-config.config", `"mqtt": {`} { + if !strings.Contains(shown, want) { + t.Errorf("settings show does not say %q:\n%s", want, shown) + } + } + if strings.Contains(shown, "timeZone") { + t.Errorf("settings show still says the file's name:\n%s", shown) + } + + config.asked = nil + said := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || + containsString(config.asked, "set module.workstation.dunst.width") { + t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) + } + told := toldEveryMachine(t, open, served, w.On)["workstation"] + if !strings.Contains(told, "12") { + t.Error("the workstation is not told the font size set through settings") + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || + !strings.Contains(err.Error(), "would no longer set width") { + t.Fatalf("a set dropping a value without --replace: %v", err) + } + if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || + !strings.Contains(err.Error(), "no own value for the whole mesh") { + t.Fatalf("a module value for the whole mesh: %v", err) + } + + t.Setenv(verbVar, "settings") + for _, c := range []struct{ module, node, values, key, held, why string }{ + // A file root reads: refused as before the move (novox/hq issue 339). + {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, + {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, + "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, + } { + err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) + if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { + t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) + } + } + // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's + // word for it (ADR 0348 §8). + if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || + !strings.Contains(err.Error(), "ADR 0348 §8") { + t.Fatalf("a trusted value through a verb: %v", err) + } + if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { + t.Fatalf("a value nobody trusts through a verb: %v", err) + } + stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) + }) + if config.values["module.workstation.dunst.width"] != float64(600) { + t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) + } + t.Setenv(verbVar, "") + + stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) + for _, k := range []string{"font-size", "width"} { + if _, held := config.values["module.workstation.dunst."+k]; held { + t.Errorf("settings clear left dunst's %s on the workstation", k) + } + } + + config.asked = nil + proposed := stdoutOf(t, func() error { + return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) + }) + if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { + t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) + } + + config.asked = nil + kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) + if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { + t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) + } +} + +func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] + values := map[string]any{key: "raised before the move"} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Fatalf("grafana does not read config after the move: %v %v", reads, err) + } + _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") + if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || + !strings.Contains(err.Error(), "propose it again") { + t.Fatalf("the stale approval: %v", err) + } + if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { + t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) + } +} + +func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { + open, _, _ := aConfiguredMesh(t) + ctx := t.Context() + readBackTries = 0 + t.Cleanup(func() { readBackTries = 15 }) + inv := open.inventory + config := &standInMeshConfig{open: open, values: map[string]any{}} + inv.ReadsConfigFrom(config.laid) + values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} + p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { + t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) + } + if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { + t.Fatalf("the move says %+v (%v)", moved, err) + } + if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { + t.Fatalf("the placement-only approval after the move: %v", err) + } + if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { + t.Errorf("the approval did not set the layer: %v %v", has, err) + } + if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { + t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) + } +} diff --git a/cmd/mesh-controller/config_moved_test.go b/cmd/mesh-controller/config_moved_test.go index 75657f8b..390a51e7 100644 --- a/cmd/mesh-controller/config_moved_test.go +++ b/cmd/mesh-controller/config_moved_test.go @@ -4,18 +4,12 @@ import ( "context" "encoding/json" "fmt" - "os" - "path/filepath" - "reflect" - "sort" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" snapshot "github.com/novox/mesh-controller/internal/facts" "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/overlay" ) // configuredMesh is every live settings layer (issue 664) — read with the settings verb on 2026-10-15, on the four @@ -42,94 +36,6 @@ var fourMachines = []string{"anchor", "laptop", "workstation", "desktop"} // aConfiguredMesh is aMesh with two machines more, every module of the configured mesh registered and assigned, and // its layers held as the store holds them once move-placement has run. -func aConfiguredMesh(t *testing.T) (*stores, configuredMesh, map[string]map[string]any) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for i, name := range []string{"workstation", "desktop"} { - record, err := inv.AddNode(ctx, name) - if err != nil { - t.Fatal(err) - } - if err := inv.SetPlace(ctx, name, name+".example:51820", "here", false, fmt.Sprintf("10.77.0.%d", i+3)); err != nil { - t.Fatal(err) - } - if err := inv.RecordProfile(ctx, record.ID, map[string]any{"capabilities": []any{ - map[string]any{"name": "container-runtime", "present": true}, - map[string]any{"name": "systemd", "present": true}}}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSealingKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if err := inv.RecordOverlayKey(ctx, record.ID, aPublicKey(t)); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, name, overlay.Name); err != nil { - t.Fatal(err) - } - } - raw, err := os.ReadFile(filepath.Join("testdata", "config-moved", "mesh.json")) - if err != nil { - t.Fatal(err) - } - var w configuredMesh - if err := json.Unmarshal(raw, &w); err != nil { - t.Fatal(err) - } - list, err := catalogue.ParsePredefined(w.Predefined) - if err != nil { - t.Fatal(err) - } - catalogue.UsePredefined(list) - served := map[string]map[string]any{} - for _, text := range w.Modules { - var shape map[string]json.RawMessage - if err := json.Unmarshal(text, &shape); err != nil { - t.Fatal(err) - } - var name string - _ = json.Unmarshal(shape["module"], &name) - if s, ok := shape["serves-checked"]; ok { - var serves map[string]any - if err := json.Unmarshal(s, &serves); err != nil { - t.Fatal(err) - } - served[name] = serves - delete(shape, "serves-checked") - } - text, err := json.Marshal(shape) - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(text) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - register(t, open, m) - } - for _, l := range w.Layers { - values, err := json.Marshal(l.Values) - if err != nil { - t.Fatal(err) - } - if l.Node == "" { - inv.ExecForTest(t, `insert into settings (node, module, values) values (null, $1, $2)`, l.Module, string(values)) - continue - } - inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, - l.Node, l.Module, string(values)) - } - for _, module := range movedKeys(w.On) { - for _, node := range w.On[module] { - if _, err := assign(ctx, open, node, module); err != nil { - t.Fatalf("assign %s %s: %v", node, module, err) - } - } - } - return open, w, served -} // standInMeshConfig is mesh-config as this test holds it: each value at its level, and resolved from the catalogue // the store holds as mesh-config resolves (novox/hq ADR 0348 §2): the module's own value on the node, else node @@ -330,205 +236,11 @@ func setAt(a configMoveAnswer, key string) []configSet { // and where from, and sets nothing; confirmed, it sets them in mesh-config, reads them back, empties the layers it // moved and switches. Each machine is then told byte for byte what it was told before, every module serves the same, // and `settings` lists the same values `config` resolves. Asked again, it sets nothing. -func TestEveryMachineIsToldTheSameAfterTheMoveOfConfig(t *testing.T) { - open, w, served := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - before := toldEveryMachine(t, open, served, w.On) - listedBefore := effectiveEverywhere(t, open, w) - factsBefore, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - - asked, err := runConfigMove(ctx, inv, config, false, "the move of config, run by a test", "") - if err != nil || !asked.DryRun || asked.Switched || len(asked.Problems) > 0 || len(config.sets) > 0 { - t.Fatalf("the move, not confirmed, says %+v (%v) and set %v", asked, err, config.sets) - } - // nodered's live layer says the file's name for its time zone: set under the config key (issue 751). - if s := setAt(asked, "module.anchor.nodered.time-zone"); len(s) != 1 || s[0].Value != "Europe/Brussels" || - !strings.HasSuffix(s[0].From[0], ": timeZone") { - t.Errorf("nodered's time zone moves as %+v", s) - } - for _, s := range asked.Sets { - if s.Key == "timeZone" { - t.Errorf("a key is set under the file's name: %+v", s) - } - } - // claude-code's whole-mesh managed settings become mesh config for agent-managed-settings (ADR 0348's - // consequences), and its whole-mesh layer sets no module value. - if s := setAt(asked, "mesh.agent-managed-settings"); len(s) != 1 || s[0].From[0] != "claude-code on the whole mesh: managed_settings" { - t.Errorf("claude-code's managed settings move as %+v", s) - } - for _, node := range fourMachines { - if s := setAt(asked, "module."+node+".claude-code.managed_settings"); len(s) > 0 { - t.Errorf("claude-code's managed settings are also set on %s: %+v", node, s) - } - } - // A whole-mesh layer of a value no predefined key holds goes to each machine it is assigned to, under that - // machine's layer: power's on the workstation is the workstation's, its own elsewhere. - if s := setAt(asked, "module.workstation.power.handle-power-key"); len(s) != 1 || s[0].Value != "suspend" { - t.Errorf("power's key on the workstation moves as %+v", s) - } - if s := setAt(asked, "module.anchor.power.handle-power-key"); len(s) != 1 || s[0].Value != "poweroff" { - t.Errorf("power's key on the anchor moves as %+v", s) - } - for _, s := range asked.Sets { - if s.Level == "node" { - t.Errorf("a machine's layer became node config: %+v", s) - } - } - var stays []string - for _, s := range asked.Stays { - stays = append(stays, strings.Fields(s)[0]) - } - sort.Strings(stays) - if want := []string{"bazarr", "bazarr", "bookshelf", "jackett", "kernel", "kometa", "lidarr", "nzbget", "ombi", - "plex", "qbittorrent", "radarr", "sonarr", "tautulli"}; len(stays) < 10 || !containsString(stays, "kernel") { - t.Errorf("the layers that stay are %v, want those of %v", stays, want) - } - if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { - t.Fatal("a dry run changed what a machine is told") - } - - moved, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "release 57 is delivered") - if err != nil || moved.DryRun || !moved.Switched { - t.Fatalf("the move says %+v (%v)", moved, err) - } - if len(config.sets) != len(asked.Sets) { - t.Fatalf("the move set %d values, and said it would set %d", len(config.sets), len(asked.Sets)) - } - if held := inv.QueryForTest(t, `select module from settings s where values <> '{}' - and module in (select name from module where manifest ? 'config')`); len(held) != 0 { - t.Fatalf("a layer of a module in the config form still holds values: %v", held) - } - after := toldEveryMachine(t, open, served, w.On) - for _, node := range fourMachines { - if after[node] != before[node] { - t.Errorf("%s is told something else after the move:\n%s\nwas:\n%s", node, after[node], before[node]) - } - } - if listedAfter := effectiveEverywhere(t, open, w); !reflect.DeepEqual(listedAfter, listedBefore) { - t.Errorf("settings lists other values after the move:\n%v\nwas:\n%v", listedAfter, listedBefore) - } - // And config lists what settings listed: each module's resolved value on each machine is the value settings - // said before the move. - resolved, err := config.Resolved(ctx) - if err != nil { - t.Fatal(err) - } - for _, r := range resolved { - said, listed := listedBefore[r.Node+"."+r.Module+"."+r.Key] - if r.Source == "unset" { - if listed && said != nil { - t.Errorf("%s of %s on %s is unset in config, and settings said %v", r.Key, r.Module, r.Node, said) - } - continue - } - var v any - _ = json.Unmarshal(r.Value, &v) - if !listed || !reflect.DeepEqual(v, said) { - t.Errorf("config resolves %s of %s on %s as %v, and settings said %v", r.Key, r.Module, r.Node, v, said) - } - } - - // The facts snapshot carries mesh-config's values for the merge gate, as each machine's layer of each module, and - // each machine's declaration composes to the digest it did before the move. - factsAfter, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - // Machines are named in a snapshot by stand-ins of its own; each is found by what it composes. - digests := func(f snapshotFacts) ([]string, string) { - var out []string - zone := "" - for _, m := range f.Machines { - out = append(out, m.Declaration.Digest) - for _, l := range m.Settings { - if l.Module == "nodered" { - zone = compactJSON(l.Values["time-zone"]) - } - } - } - sort.Strings(out) - return out, zone - } - was, _ := digests(factsBefore) - now, zone := digests(factsAfter) - if zone != `"Europe/Brussels"` { - t.Errorf("the facts snapshot does not carry nodered's time zone from mesh-config: %s", zone) - } - if len(now) != len(fourMachines) || now[0] == "" || !reflect.DeepEqual(now, was) { - t.Errorf("the declarations in the facts snapshot are %v after the move, %v before", now, was) - } - for _, s := range factsAfter.Settings { - if _, isConfig := map[string]bool{"betterbird": true, "claude-code": true, "power": true, "xdg": true}[s.Module]; isConfig && - len(s.Values) > 0 { - t.Errorf("the facts snapshot still carries the whole mesh's layer of %s: %v", s.Module, s.Values) - } - } - - config.sets = nil - again, err := runConfigMove(ctx, inv, config, true, "the move of config, run by a test", "again") - if err != nil || len(config.sets) != 0 || len(again.Moved) != 0 { - t.Fatalf("asked again, the move set %v and says %+v (%v)", config.sets, again, err) - } - if after := toldEveryMachine(t, open, served, w.On); !reflect.DeepEqual(after, before) { - t.Fatal("the move asked again changed what a machine is told") - } -} // **The move refuses what it cannot move as it composes, and sets nothing** (issue 664): a layer that still holds // placement (move-placement has not run), a key its module does not declare, and a value its declaration refuses — // each said by name; and a value mesh-config resolves from a level no layer says (it would be laid once the switch // is on) leaves every layer and the switch as they were. -func TestTheMoveOfConfigRefusesWhatWouldChangeAMachine(t *testing.T) { - open, _, _ := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - - inv.ExecForTest(t, `update settings set values = values || '{"endpoints": {"web": {"reach": "public"}}}' - where module = 'nodered'`) - inv.ExecForTest(t, `update settings set values = values || '{"colour": "blue"}' where module = 'unifi'`) - inv.ExecForTest(t, `update settings set values = '{"font-size": 900}' where module = 'dunst'`) - answer, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") - if err == nil || answer.Switched || len(config.sets) > 0 { - t.Fatalf("the move went ahead: %+v (%v), set %v", answer, err, config.sets) - } - for _, want := range []string{"nodered on anchor still holds placement (endpoints): run move-placement first", - `unifi on anchor says "colour", which unifi does not declare`, "font-size at dunst's own value on workstation"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q: %v", want, err) - } - } - inv.ExecForTest(t, `update settings set values = values - 'endpoints' where module = 'nodered'`) - inv.ExecForTest(t, `update settings set values = values - 'colour' where module = 'unifi'`) - inv.ExecForTest(t, `update settings set values = '{"font-size": 13, "width": 500}' where module = 'dunst'`) - - // Set in mesh-config before the move, on a module whose layers say nothing of it: the switch would lay it. - config.values["module.anchor.grafana."+movedKeys(func() map[string]catalogue.SettingDeclaration { - shelf, _ := inv.Catalogue(ctx) - return shelf["grafana"].Settings - }())[0]] = "set before the move" - answer, err = runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered") - if err == nil || !strings.Contains(err.Error(), "grafana on anchor resolves") || answer.Switched { - t.Fatalf("a value no layer says was laid by the switch: %+v (%v)", answer, err) - } - if moved, _ := inv.ConfigMoved(ctx); moved.Moved { - t.Fatal("the switch is on after a refused move") - } - if held := inv.QueryForTest(t, `select count(*) from settings where values <> '{}'`); held[0][0] == "0" { - t.Fatal("a refused move emptied the layers") - } -} // The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step of // the release, never a repair (issue 664). @@ -614,172 +326,11 @@ func (c *standInMeshConfig) ask(ctx context.Context, args map[string]any) (json. // has run, a module in the config form is shown from mesh-config, its values set and cleared there as its own on the // machine (placement still beside the assignment), and proposed through it; a trusted value is refused through a // verb, a module value for the whole mesh is refused, and a module still in the settings form keeps today's verb. -func TestTheSettingsVerbForwardsToConfigOnceTheValuesMoved(t *testing.T) { - open, w, served := aConfiguredMesh(t) - ctx := t.Context() - inv := open.inventory - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - savedRead, savedAsk := readConfigValues, configAsk - readConfigValues, configAsk = config.laid, config.ask - t.Cleanup(func() { readConfigValues, configAsk = savedRead, savedAsk }) - if _, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil { - t.Fatal(err) - } - - shown := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "nodered", "--node", "anchor"}) }) - for _, want := range []string{`"time-zone": "Europe/Brussels"`, `time-zone = "Europe/Brussels" (module)`, - "mesh-config.config", `"mqtt": {`} { - if !strings.Contains(shown, want) { - t.Errorf("settings show does not say %q:\n%s", want, shown) - } - } - if strings.Contains(shown, "timeZone") { - t.Errorf("settings show still says the file's name:\n%s", shown) - } - - config.asked = nil - said := stdoutOf(t, func() error { - return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 500}`, "--node", "workstation"}) - }) - if config.values["module.workstation.dunst.font-size"] != float64(12) || !strings.Contains(said, "~ font-size") || - containsString(config.asked, "set module.workstation.dunst.width") { - t.Fatalf("settings set reached mesh-config as %v (%v):\n%s", config.values, config.asked, said) - } - told := toldEveryMachine(t, open, served, w.On)["workstation"] - if !strings.Contains(told, "12") { - t.Error("the workstation is not told the font size set through settings") - } - if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`, "--node", "workstation"}); err == nil || - !strings.Contains(err.Error(), "would no longer set width") { - t.Fatalf("a set dropping a value without --replace: %v", err) - } - if err := settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12}`}); err == nil || - !strings.Contains(err.Error(), "no own value for the whole mesh") { - t.Fatalf("a module value for the whole mesh: %v", err) - } - - t.Setenv(verbVar, "settings") - for _, c := range []struct{ module, node, values, key, held, why string }{ - // A file root reads: refused as before the move (novox/hq issue 339). - {"hostname", "anchor", `{"hostname": "elsewhere"}`, "module.anchor.hostname.hostname", "Ace", "issue 339"}, - {"messenger", "laptop", `{"desktop-machines": ["g14", "shanks"], "time-zone": "UTC"}`, - "module.laptop.messenger.time-zone", "Europe/Brussels", "issue 339"}, - } { - err := settingsCommand(ctx, []string{"set", c.module, c.values, "--node", c.node}) - if err == nil || !strings.Contains(err.Error(), c.why) || config.values[c.key] != c.held { - t.Fatalf("a trusted value of %s through a verb: %v, holds %v", c.module, err, config.values[c.key]) - } - } - // And a trusted value no file makes the terminal's is refused as trusted, since mesh-config takes the controller's - // word for it (ADR 0348 §8). - if err := refuseTrustedThroughAVerb(ctx, inv, "messenger", "laptop", []string{"time-zone"}); err == nil || - !strings.Contains(err.Error(), "ADR 0348 §8") { - t.Fatalf("a trusted value through a verb: %v", err) - } - if err := refuseTrustedThroughAVerb(ctx, inv, "dunst", "workstation", []string{"width"}); err != nil { - t.Fatalf("a value nobody trusts through a verb: %v", err) - } - stdoutOf(t, func() error { - return settingsCommand(ctx, []string{"set", "dunst", `{"font-size": 12, "width": 600}`, "--node", "workstation"}) - }) - if config.values["module.workstation.dunst.width"] != float64(600) { - t.Fatalf("a value nobody trusts was not set through a verb: %v", config.values["module.workstation.dunst.width"]) - } - t.Setenv(verbVar, "") - - stdoutOf(t, func() error { return settingsCommand(ctx, []string{"clear", "dunst", "--node", "workstation"}) }) - for _, k := range []string{"font-size", "width"} { - if _, held := config.values["module.workstation.dunst."+k]; held { - t.Errorf("settings clear left dunst's %s on the workstation", k) - } - } - - config.asked = nil - proposed := stdoutOf(t, func() error { - return settingsCommand(ctx, []string{"propose", "hostname", `{"hostname": "elsewhere"}`, "--node", "anchor"}) - }) - if !containsString(config.asked, "propose module.anchor.hostname.hostname") || !strings.Contains(proposed, "p-1") { - t.Fatalf("settings propose reached mesh-config as %v:\n%s", config.asked, proposed) - } - - config.asked = nil - kept := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"show", "kernel", "--node", "laptop"}) }) - if !strings.Contains(kept, "transparent-huge-pages") || len(config.asked) > 0 { - t.Fatalf("a module in the settings form is shown as %q, asking mesh-config %v", kept, config.asked) - } -} // **An approval for a module whose config has moved is refused, and writes nothing** (issue 830, novox/hq ADR 0348): // a proposal raised while grafana read its layer (it had none, so the layer is as the operator was shown it) and // approved after the move would put its values back into the layer, ReadsConfig would turn false, and the module // would compose from emptied layers everywhere. -func TestAnApprovalRaisedBeforeTheMoveOfConfigIsRefusedAfterIt(t *testing.T) { - open, _, _ := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - key := movedKeys(func() map[string]catalogue.SettingDeclaration { return shelf["grafana"].Settings }())[0] - values := map[string]any{key: "raised before the move"} - p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { - t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) - } - if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { - t.Fatalf("the move says %+v (%v)", moved, err) - } - if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { - t.Fatalf("grafana does not read config after the move: %v %v", reads, err) - } - _, err = setLayerIn(open)(ctx, p, "s30", "approved by the operator") - if err == nil || !strings.Contains(err.Error(), "moved to mesh-config since the operator was shown the change") || - !strings.Contains(err.Error(), "propose it again") { - t.Fatalf("the stale approval: %v", err) - } - if layer, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has && len(layer) > 0 { - t.Errorf("the refused approval wrote the layer: %v %v %v", layer, has, err) - } - if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { - t.Errorf("the refused approval turned ReadsConfig off: %v %v", reads, err) - } -} // **A proposal of placement only is set after the move, as before** (issue 830): forwardPropose sends it down the // old path on purpose, so refusing it would loop the operator (refused, proposed again, refused). -func TestAPlacementOnlyApprovalRaisedBeforeTheMoveIsSetAfterIt(t *testing.T) { - open, _, _ := aConfiguredMesh(t) - ctx := t.Context() - readBackTries = 0 - t.Cleanup(func() { readBackTries = 15 }) - inv := open.inventory - config := &standInMeshConfig{open: open, values: map[string]any{}} - inv.ReadsConfigFrom(config.laid) - values := map[string]any{catalogue.PlacesSetting: map[string]any{"data": "/srv/grafana-data"}} - p := settingsProposal{Module: "grafana", Node: "anchor", Values: values, From: "g14/claude-code", At: time.Now(), - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || has { - t.Fatalf("grafana has a layer on anchor before the test: %v %v", has, err) - } - if moved, err := runConfigMove(ctx, inv, config, true, "a test", "release 57 is delivered"); err != nil || !moved.Switched { - t.Fatalf("the move says %+v (%v)", moved, err) - } - if _, err := setLayerIn(open)(ctx, p, "s31", "approved by the operator"); err != nil { - t.Fatalf("the placement-only approval after the move: %v", err) - } - if _, has, err := inv.Layer(ctx, "anchor", "grafana"); err != nil || !has { - t.Errorf("the approval did not set the layer: %v %v", has, err) - } - if reads, err := inv.ReadsConfig(ctx, "grafana"); err != nil || !reads { - t.Errorf("the approval turned ReadsConfig off: %v %v", reads, err) - } -} diff --git a/cmd/mesh-controller/core_box_integration_test.go b/cmd/mesh-controller/core_box_integration_test.go new file mode 100644 index 00000000..55660821 --- /dev/null +++ b/cmd/mesh-controller/core_box_integration_test.go @@ -0,0 +1,129 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + client := aClientModule() + register(t, open, client) + if _, err := assign(ctx, open, "laptop", client.Module); err != nil { + t.Fatal(err) + } + if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"}, + "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + v := &coreStandIn{generation: 1} + withVault(t, v) + return open, v +} + +func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + key, err := open.inventory.SealingKeyOf(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil { + t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f) + } + if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) { + t.Fatalf("the vault was asked %v", vault.asked) + } + if want, _ := sealingKeyPEM(key); vault.keys[0] != want { + t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0]) + } + first, _ := json.Marshal(f[catalogue.VaultSealedField]) + + // The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable. + again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(again) != string(first) { + t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again) + } + // A rotation in the vault (a newer generation) is carried. + vault.generation = 2 + rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + if string(rotated) == string(first) { + t.Fatal("a newer generation was not carried") + } + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("a send the vault answered raised its silence") + } +} + +func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) + vault.silent = errors.New("no responders") + held := clientsBroker(sentTo(t, open, "laptop")) + got, _ := json.Marshal(held[catalogue.VaultSealedField]) + if held == nil || held["sealed"] != nil || string(got) != string(first) { + t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held) + } + c, found := vaultSilence(t, "laptop") + if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") { + t.Fatalf("no condition names the vault and the machine: %+v", c) + } + // The vault answers again: the condition clears. + vault.silent = nil + sentTo(t, open, "laptop") + if _, found := vaultSilence(t, "laptop"); found { + t.Fatal("the condition stayed after the vault answered") + } +} + +func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) { + open, vault := aCoreMesh(t, link.VaultBoxContract) + vault.silent = errors.New("no responders") + plan, settings, err := planFor(t.Context(), open, "laptop") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(t.Context(), open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + f := clientsBroker(declared.Resources) + if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { + t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f) + } + if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left { + t.Fatalf("the core module was left out: %v", why) + } + c, found := vaultSilence(t, "laptop") + if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") { + t.Fatalf("the condition does not say why: %+v", c) + } +} + +func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) { + open, vault := aCoreMesh(t, link.HandedContract) + f := clientsBroker(sentTo(t, open, "laptop")) + if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { + t.Fatalf("an older engine is sent %v", f) + } + if len(vault.asked) != 0 { + t.Fatalf("the vault was asked %v for an older engine", vault.asked) + } +} diff --git a/cmd/mesh-controller/core_box_test.go b/cmd/mesh-controller/core_box_test.go index c73264aa..4a6872de 100644 --- a/cmd/mesh-controller/core_box_test.go +++ b/cmd/mesh-controller/core_box_test.go @@ -7,13 +7,11 @@ import ( "reflect" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // The core's own credentials travel sealed by the vault to their machine (novox/hq ADR 0349 §16, issue 755; the lead's @@ -46,28 +44,6 @@ func withVault(t *testing.T, v *coreStandIn) { t.Cleanup(func() { coreVault = before }) } -func aCoreMesh(t *testing.T, contract int) (*stores, *coreStandIn) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - client := aClientModule() - register(t, open, client) - if _, err := assign(ctx, open, "laptop", client.Module); err != nil { - t.Fatal(err) - } - if err := issueProvisionerClient(ctx, open.inventory, client, "laptop", broker.Broker{Fingerprint: "ab"}, - "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: contract, - SaidAt: time.Now()}); err != nil { - t.Fatal(err) - } - v := &coreStandIn{generation: 1} - withVault(t, v) - return open, v -} - func clientsBroker(resources []map[string]any) map[string]any { for _, r := range resources { if r["id"] == "mesh-provisioner-client.needs-"+broker.ProvisionerSecret { @@ -86,85 +62,6 @@ func vaultSilence(t *testing.T, node string) (conditions.Condition, bool) { return c, found } -func TestTheSendCarriesTheVaultsBoxForACoreCredentialAndKeepsIt(t *testing.T) { - open, vault := aCoreMesh(t, link.VaultBoxContract) - key, err := open.inventory.SealingKeyOf(t.Context(), "laptop") - if err != nil { - t.Fatal(err) - } - f := clientsBroker(sentTo(t, open, "laptop")) - if f == nil || f["sealed"] != nil || f[catalogue.VaultSealedField] == nil { - t.Fatalf("the core credential is sent as %v: the vault's box, never the controller's seal", f) - } - if want := []string{"own/laptop/mesh-provisioner-client/" + broker.ProvisionerSecret}; !reflect.DeepEqual(vault.asked, want) { - t.Fatalf("the vault was asked %v", vault.asked) - } - if want, _ := sealingKeyPEM(key); vault.keys[0] != want { - t.Fatalf("the vault was asked to seal to %q, not the machine's key", vault.keys[0]) - } - first, _ := json.Marshal(f[catalogue.VaultSealedField]) - - // The vault answers afresh with the same generation: the kept box is carried again, so the declaration is stable. - again, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) - if string(again) != string(first) { - t.Fatalf("a fresh box of the same generation changed the declaration: %s, then %s", first, again) - } - // A rotation in the vault (a newer generation) is carried. - vault.generation = 2 - rotated, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) - if string(rotated) == string(first) { - t.Fatal("a newer generation was not carried") - } - if _, found := vaultSilence(t, "laptop"); found { - t.Fatal("a send the vault answered raised its silence") - } -} - -func TestASilentVaultHoldsTheKeptBoxAndSaysSo(t *testing.T) { - open, vault := aCoreMesh(t, link.VaultBoxContract) - first, _ := json.Marshal(clientsBroker(sentTo(t, open, "laptop"))[catalogue.VaultSealedField]) - vault.silent = errors.New("no responders") - held := clientsBroker(sentTo(t, open, "laptop")) - got, _ := json.Marshal(held[catalogue.VaultSealedField]) - if held == nil || held["sealed"] != nil || string(got) != string(first) { - t.Fatalf("a silent vault's credential is sent as %v: the box kept, never the controller's seal", held) - } - c, found := vaultSilence(t, "laptop") - if !found || !strings.Contains(c.Summary, "vault") || !strings.Contains(c.Summary, "laptop") { - t.Fatalf("no condition names the vault and the machine: %+v", c) - } - // The vault answers again: the condition clears. - vault.silent = nil - sentTo(t, open, "laptop") - if _, found := vaultSilence(t, "laptop"); found { - t.Fatal("the condition stayed after the vault answered") - } -} - -func TestACoreModuleFirstPlacedWhileTheVaultIsSilentIsSealedByTheController(t *testing.T) { - open, vault := aCoreMesh(t, link.VaultBoxContract) - vault.silent = errors.New("no responders") - plan, settings, err := planFor(t.Context(), open, "laptop") - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(t.Context(), open, "laptop", plan, settings) - if err != nil { - t.Fatal(err) - } - f := clientsBroker(declared.Resources) - if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { - t.Fatalf("the core credential is composed as %v: the controller's own seal, until the vault holds it", f) - } - if why, left := declared.leftOutWhy["mesh-provisioner-client"]; left { - t.Fatalf("the core module was left out: %v", why) - } - c, found := vaultSilence(t, "laptop") - if !found || !strings.Contains(c.Summary, "sealed by the controller until the vault holds it") { - t.Fatalf("the condition does not say why: %+v", c) - } -} - // Issue 908: a machine at the vault-box contract, the vault silent, no kept box: nats is composed with the // controller's own seal, never left out, and the vault-silent condition names it. func TestReplay908(t *testing.T) { @@ -191,16 +88,6 @@ func TestReplay908(t *testing.T) { } // An engine below VaultBoxContract is sent the controller's own seal, as before, and the vault is not asked. -func TestAnEngineBelowTheContractIsSentTheControllersSealAsBefore(t *testing.T) { - open, vault := aCoreMesh(t, link.HandedContract) - f := clientsBroker(sentTo(t, open, "laptop")) - if f == nil || f["sealed"] == nil || f[catalogue.VaultSealedField] != nil { - t.Fatalf("an older engine is sent %v", f) - } - if len(vault.asked) != 0 { - t.Fatalf("the vault was asked %v for an older engine", vault.asked) - } -} // Every question but a send reads the kept box and asks nothing; with none kept, a stand-in the next send asks for. func TestOnlyASendAsksTheVault(t *testing.T) { diff --git a/cmd/mesh-controller/credential_on_assign_test.go b/cmd/mesh-controller/credential_on_assign_test.go index 959ef073..9c8133a0 100644 --- a/cmd/mesh-controller/credential_on_assign_test.go +++ b/cmd/mesh-controller/credential_on_assign_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/data_integration_test.go b/cmd/mesh-controller/data_integration_test.go new file mode 100644 index 00000000..c1142df8 --- /dev/null +++ b/cmd/mesh-controller/data_integration_test.go @@ -0,0 +1,148 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) { + inv := inventory.ForTest(t) + ctx := t.Context() + shelf := shelfFor(t, houseManifest) + declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}} + start := time.Now().Add(-6 * time.Hour) + measure := func(at time.Time, path string, size int64) []conditions.Observation { + t.Helper() + if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": { + "config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at), + LastBackup: when(at)}}}, "", at); err != nil { + t.Fatal(err) + } + records, err := inv.Data(ctx) + if err != nil { + t.Fatal(err) + } + peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow)) + if err != nil { + t.Fatal(err) + } + return dataFindings(records, peaks, shelf, nil, nil, at) + } + measure(start, "/home/operator/.claude", 94_700_000) + if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" { + t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank]) + } + measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20) + got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank] + if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") { + t.Fatalf("a genuine shrink at the new path was not raised: %+v", got) + } +} + +func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) + house, err := catalogue.ParseManifest([]byte(houseManifest)) + if err != nil { + t.Fatal(err) + } + register(t, open, house) + if _, err := assign(ctx, open, "laptop", "house"); err == nil { + t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") + } + for _, node := range []string{"laptop", "anchor"} { + if _, err := assign(ctx, open, node, "keeper"); err != nil { + t.Fatal(err) + } + } + if _, err := assign(ctx, open, "laptop", "house"); err != nil { + t.Fatal(err) + } + // Sent, and applied: a holder is asked only once it has been (novox/hq issue 275). + for _, node := range []string{"laptop", "anchor"} { + pushedAndApplied(t, open, node) + } + + conn := onATestBus(t) + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + laptop, anchor := &fakeHolder{}, &fakeHolder{} + laptop.serve(t, conn, "laptop") + anchor.serve(t, conn, "anchor") + now := time.Now() + heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{ + {name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}} + d := &doctor{open: open, js: js, watchdogs: heard} + var d13 probe + for _, p := range probeRegistry { + if p.ID == probeDataID { + d13 = p + } + } + run := func() []conditions.Observation { + t.Helper() + probing := context.WithValue(ctx, probeAsksKey{}, d13) + obs, err := probeData(probing, d) + if err != nil { + t.Fatal(err) + } + return obs + } + + laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now)) + if obs := run(); len(obs) != 0 { + t.Fatalf("a measured, backed-up item raised %+v", obs) + } + r, err := inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil { + t.Fatalf("what the holder measured was not kept: %+v, %v", r, err) + } + + said, err := unassign(ctx, open, "laptop", "house") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") { + t.Fatalf("the unassignment does not say the data stays:\n%s", said) + } + laptop.set() + run() + r, err = inv.DataOf(ctx, "laptop", "house", "config") + if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" { + t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err) + } + records, _ := inv.Data(ctx) + if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" { + t.Fatalf("cleanup list: %+v", rows) + } + + // Assigned on the anchor, onto an empty directory. + if _, err := assign(ctx, open, "anchor", "house"); err != nil { + t.Fatal(err) + } + anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now())) + obs := findingsByKind(run()) + o, ok := obs[kindEmptyReplacement] + if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" { + t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs) + } +} diff --git a/cmd/mesh-controller/data_test.go b/cmd/mesh-controller/data_test.go index 00324b19..15dcf45e 100644 --- a/cmd/mesh-controller/data_test.go +++ b/cmd/mesh-controller/data_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "encoding/json" "strings" "sync" @@ -10,14 +9,11 @@ import ( "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - - "github.com/novox/mesh-controller/internal/testbus" ) func bytesOf(n int64) *int64 { return &n } @@ -173,39 +169,6 @@ func TestAShrinkOfMoreThanHalfIsUrgent(t *testing.T) { // operator's own home (94.7 MB) to the agent account's fresh one (490 B), and `data-shrank` was raised // for data that was never lost. A moved item is read against its new path only, so nothing is raised — // and a genuine shrink at the new path, a week of history later, still is. -func TestAMovedPathIsNoShrinkAndAShrinkThereStillIs(t *testing.T) { - inv := inventory.ForTest(t) - ctx := t.Context() - shelf := shelfFor(t, houseManifest) - declared := []inventory.DeclaredData{{Module: "house", Item: "config", Class: "irreplaceable", Owned: true}} - start := time.Now().Add(-6 * time.Hour) - measure := func(at time.Time, path string, size int64) []conditions.Observation { - t.Helper() - if _, err := inv.RecordData(ctx, "home", declared, map[string]map[string]inventory.Measurement{"house": { - "config": {Path: path, Size: bytesOf(size), MeasuredAt: when(at), LastWrite: when(at), - LastBackup: when(at)}}}, "", at); err != nil { - t.Fatal(err) - } - records, err := inv.Data(ctx) - if err != nil { - t.Fatal(err) - } - peaks, err := inv.DataPeaks(ctx, at.Add(-shrinkWindow)) - if err != nil { - t.Fatal(err) - } - return dataFindings(records, peaks, shelf, nil, nil, at) - } - measure(start, "/home/operator/.claude", 94_700_000) - if got := findingsByKind(measure(start.Add(10*time.Minute), "/home/agent/.claude", 490)); got[kindDataShrank].Kind != "" { - t.Fatalf("a moved path raised a shrink: %+v", got[kindDataShrank]) - } - measure(start.Add(2*time.Hour), "/home/agent/.claude", 300<<20) - got := findingsByKind(measure(start.Add(4*time.Hour), "/home/agent/.claude", 1<<20))[kindDataShrank] - if got.Severity != conditions.Urgent || !strings.Contains(got.Summary, "shrank") { - t.Fatalf("a genuine shrink at the new path was not raised: %+v", got) - } -} // Data said to be written all the time and not written; data with no backup or an old one — urgent when // irreplaceable, a warning when valuable; and a new item given its bound before it is said. @@ -313,103 +276,6 @@ func measuredHouse(path string, size int64, at time.Time) map[string]any { // directory is an empty replacement — through the real stores and a real bus. The unassignment says the // data stays; the self-check's next run retires it (kept, listed by cleanup), and when the module comes // up on another machine with an empty directory while the full one waits retired, that is urgent. -func TestNatsUnassigningIrreplaceableDataRetiresItAndAnEmptyReplacementIsUrgent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "keeper", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BackupSeat, Scope: catalogue.ScopeNode, Serves: []string{"backed-up", "now", "restore"}}}}) - house, err := catalogue.ParseManifest([]byte(houseManifest)) - if err != nil { - t.Fatal(err) - } - register(t, open, house) - if _, err := assign(ctx, open, "laptop", "house"); err == nil { - t.Fatal("irreplaceable data was assigned to a machine with nothing to back it up") - } - for _, node := range []string{"laptop", "anchor"} { - if _, err := assign(ctx, open, node, "keeper"); err != nil { - t.Fatal(err) - } - } - if _, err := assign(ctx, open, "laptop", "house"); err != nil { - t.Fatal(err) - } - // Sent, and applied: a holder is asked only once it has been (novox/hq issue 275). - for _, node := range []string{"laptop", "anchor"} { - pushedAndApplied(t, open, node) - } - - conn := onATestBus(t) - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - laptop, anchor := &fakeHolder{}, &fakeHolder{} - laptop.serve(t, conn, "laptop") - anchor.serve(t, conn, "anchor") - now := time.Now() - heard := &watchdogs{last: &signalFacts{now: now, machines: []machineFacts{ - {name: "laptop", lastHeard: now}, {name: "anchor", lastHeard: now}}}} - d := &doctor{open: open, js: js, watchdogs: heard} - var d13 probe - for _, p := range probeRegistry { - if p.ID == probeDataID { - d13 = p - } - } - run := func() []conditions.Observation { - t.Helper() - probing := context.WithValue(ctx, probeAsksKey{}, d13) - obs, err := probeData(probing, d) - if err != nil { - t.Fatal(err) - } - return obs - } - - laptop.set(measuredHouse("/var/lib/house/config", 2<<30, now)) - if obs := run(); len(obs) != 0 { - t.Fatalf("a measured, backed-up item raised %+v", obs) - } - r, err := inv.DataOf(ctx, "laptop", "house", "config") - if err != nil || r.Path != "/var/lib/house/config" || r.Size == nil || *r.Size != 2<<30 || r.LastBackup == nil { - t.Fatalf("what the holder measured was not kept: %+v, %v", r, err) - } - - said, err := unassign(ctx, open, "laptop", "house") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(said, "house's config (/var/lib/house/config, 2.0 GB) stays where it is") { - t.Fatalf("the unassignment does not say the data stays:\n%s", said) - } - laptop.set() - run() - r, err = inv.DataOf(ctx, "laptop", "house", "config") - if err != nil || !r.Retired() || r.Path != "/var/lib/house/config" { - t.Fatalf("unassigned, the irreplaceable item is not kept retired: %+v, %v", r, err) - } - records, _ := inv.Data(ctx) - if rows := retiredData(records, time.Now()); len(rows) != 1 || rows[0].Path != "/var/lib/house/config" { - t.Fatalf("cleanup list: %+v", rows) - } - - // Assigned on the anchor, onto an empty directory. - if _, err := assign(ctx, open, "anchor", "house"); err != nil { - t.Fatal(err) - } - anchor.set(measuredHouse("/var/lib/house/config", 300<<10, time.Now())) - obs := findingsByKind(run()) - o, ok := obs[kindEmptyReplacement] - if !ok || o.Severity != conditions.Urgent || o.Machine != "anchor" || o.Also[0] != "laptop" { - t.Fatalf("an empty replacement of a module's data was not urgent: %+v", obs) - } -} // Data on redundant storage: the array it is on is watched, one condition per array as loud as the most // precious item on it; an item said to be on redundancy and found on plain storage is said; an item diff --git a/cmd/mesh-controller/deadletters_integration_test.go b/cmd/mesh-controller/deadletters_integration_test.go new file mode 100644 index 00000000..d550393d --- /dev/null +++ b/cmd/mesh-controller/deadletters_integration_test.go @@ -0,0 +1,88 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func servingDeadLetters(t *testing.T) *broker.JetStream { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + before := servingBus.Load() + servingBus.Store(js) + t.Cleanup(func() { servingBus.Store(before) }) + return js +} + +func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) { + js := servingDeadLetters(t) + if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil { + t.Fatal(err) + } + d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`)) + if err != nil { + t.Fatal(err) + } + + answer, err := askDeadLetters(t, map[string]any{}) + if err != nil { + t.Fatal(err) + } + listed := answer.(map[string]any) + if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 { + t.Fatalf("listed %v", listed) + } + + // Refused before anything is done: an act without why, an argument the shape passes over, both acts. + for _, args := range []map[string]any{ + {"drop": "1"}, + {"drop": "1", "why": "x", "limit": "3"}, + {"drop": "1", "deliver": "1", "why": "x"}, + {"why": "x"}, + {"id": "nought"}, + } { + if _, err := askDeadLetters(t, args); err == nil { + t.Errorf("%v was done", args) + } + } + if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 { + t.Fatalf("a refused call changed what is kept: %d left", total) + } + + done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"}) + if err != nil { + t.Fatal(err) + } + if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") { + t.Fatalf("answered %v", said) + } + acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute)) + if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter || + !strings.Contains(acts[0].Condition, "media_sonarr") { + t.Fatalf("recorded %+v (%v)", acts, err) + } + if !personsDecision(acts[0]) { + t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it") + } + if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil { + t.Errorf("dead letter %d is still answered after it was dropped", d.ID) + } +} diff --git a/cmd/mesh-controller/deadletters_test.go b/cmd/mesh-controller/deadletters_test.go index e714456f..81d7e9e2 100644 --- a/cmd/mesh-controller/deadletters_test.go +++ b/cmd/mesh-controller/deadletters_test.go @@ -6,31 +6,10 @@ import ( "testing" "time" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // The serving controller's bus, with the mesh's streams, for the verb to read and act on. -func servingDeadLetters(t *testing.T) *broker.JetStream { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - before := servingBus.Load() - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(before) }) - return js -} func askDeadLetters(t *testing.T, args map[string]any) (any, error) { t.Helper() @@ -41,61 +20,6 @@ func askDeadLetters(t *testing.T, args map[string]any) (any, error) { return deadLettersAnswer(context.Background(), a) } -func TestDeadLettersListsDropsAndRecordsWhy(t *testing.T) { - js := servingDeadLetters(t) - if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{"n":1}`)); err != nil { - t.Fatal(err) - } - d, err := link.KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":1,"deliveries":5}`)) - if err != nil { - t.Fatal(err) - } - - answer, err := askDeadLetters(t, map[string]any{}) - if err != nil { - t.Fatal(err) - } - listed := answer.(map[string]any) - if listed["held"] != 1 || len(listed["dead_letters"].([]link.DeadLetter)) != 1 { - t.Fatalf("listed %v", listed) - } - - // Refused before anything is done: an act without why, an argument the shape passes over, both acts. - for _, args := range []map[string]any{ - {"drop": "1"}, - {"drop": "1", "why": "x", "limit": "3"}, - {"drop": "1", "deliver": "1", "why": "x"}, - {"why": "x"}, - {"id": "nought"}, - } { - if _, err := askDeadLetters(t, args); err == nil { - t.Errorf("%v was done", args) - } - } - if _, total, _ := link.DeadLetters(js.Context(), "", 0); total != 1 { - t.Fatalf("a refused call changed what is kept: %d left", total) - } - - done, err := askDeadLetters(t, map[string]any{"drop": "1", "why": "the media server took the download in by hand"}) - if err != nil { - t.Fatal(err) - } - if said := done.(map[string]any); said["recorded"] == nil || !strings.Contains(said["done"].(string), "dropped") { - t.Fatalf("answered %v", said) - } - acts, err := link.HandActs(context.Background(), js.Conn(), time.Now().Add(-time.Minute)) - if err != nil || len(acts) != 1 || acts[0].Verb != "dead-letters drop" || acts[0].Cause != causeDeadLetter || - !strings.Contains(acts[0].Condition, "media_sonarr") { - t.Fatalf("recorded %+v (%v)", acts, err) - } - if !personsDecision(acts[0]) { - t.Error("dropping a dead letter is counted as a repair, so S15 would want a healer for it") - } - if _, err := askDeadLetters(t, map[string]any{"id": "1"}); err == nil { - t.Errorf("dead letter %d is still answered after it was dropped", d.ID) - } -} - // Open while DEAD_LETTERS holds a message for the consumer, in words the operator reads in one pass: // what is held, and where to act. func TestAConsumersDeadLettersAreSaidUntilActedOn(t *testing.T) { diff --git a/cmd/mesh-controller/delivery.go b/cmd/mesh-controller/delivery.go index 86dafe7b..e8a8da77 100644 --- a/cmd/mesh-controller/delivery.go +++ b/cmd/mesh-controller/delivery.go @@ -579,13 +579,11 @@ type orderMember struct { ModuleDirs []string `json:"module_dirs,omitempty"` ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` CloneURL string `json:"clone_url,omitempty"` - // After are the repositories its pull request says it goes after (`after: `). - After []string `json:"after,omitempty"` // BaseSHA is the base branch's head the forge last announced the head with (novox/hq ADR 0323): the trunk commit // a verdict no build judges names when the head is checked again through mesh-delivery. BaseSHA string `json:"base_sha,omitempty"` - // Ready says the member's pull request is marked ready (novox/hq ADR 0406 §3, issue 974): its check is queued before - // every other check. Said by mesh-delivery on a ready mark. + // Ready says mesh-delivery wants the member's check queued before every other check (novox/hq ADR 0406 §3, issue + // 974). There is no ready mark any more (issue 1022): mesh-delivery says it when it asks for the Approve. Ready bool `json:"ready,omitempty"` } @@ -611,14 +609,6 @@ type orderPair struct { Over []string `json:"over,omitempty"` } -// orderContradiction is two members that rules of one rank order both ways: no precedence resolves it, so -// the group is refused, the rules named, and Said says how a person declares the order. -type orderContradiction struct { - Members []string `json:"members"` - Rules []orderPair `json:"rules"` - Said string `json:"said"` -} - // orderReach is what a member moves, as the planner says. type orderReach struct { Moved []string `json:"moved,omitempty"` @@ -627,114 +617,76 @@ type orderReach struct { Manifests []string `json:"manifests,omitempty"` } -// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one — -// with each contradiction no precedence resolves. +// groupOrder is a group's order: the members in it, every pair and why, and the cycle when there is one. The +// order is read from the module graph and the manifests' change alone (novox/hq ADR 0419): a cycle is the only +// refusal, and CycleSaid names every module and edge of it. Edges are the module edges the pairs come from. type groupOrder struct { - Order []string `json:"order"` - Pairs []orderPair `json:"pairs,omitempty"` - Cycle []string `json:"cycle,omitempty"` - Contradictions []orderContradiction `json:"contradictions,omitempty"` - Reach map[string]orderReach `json:"reach,omitempty"` + Order []string `json:"order"` + Pairs []orderPair `json:"pairs,omitempty"` + Cycle []string `json:"cycle,omitempty"` + CycleSaid string `json:"cycle_said,omitempty"` + Edges []inventory.Edge `json:"edges,omitempty"` + Reach map[string]orderReach `json:"reach,omitempty"` } -// The reasons a pair is ordered, in the words the delivery plan shows. -const ( - orderDeclared = "declared" - orderBuiltBy = "built by" - orderVersionSkew = "version skew" - orderEngineFirst = "engine before controller" -) +// orderBuiltBy is the reason a member goes before another that moves a module built by, standing on, packaging or +// declaring one of its modules: the module graph's own edge. +const orderBuiltBy = "built by" -// orderRank is a rule's precedence when two rules order one pair both ways (novox/hq ADR 0249), the lower -// winning: what a person declared; then what the graph and the change say (built by, version skew); then -// the rollout default, which infers a direction from the modules' names alone. -func orderRank(why string) int { - switch why { - case orderDeclared: - return 0 - case orderBuiltBy, orderVersionSkew: - return 1 - case orderEngineFirst: - return 2 - } - return 3 -} - -// orderOf is a group's order (novox/hq ADR 0239 decision 4, ADR 0249), pure. A member goes before another -// when: -// -// - its pull request is named in the other's `after:` lines (declared); -// - it moves a module the other's moved modules are built by, stand on, package or declare (built by); -// - it moves the controller and the other changes any module's manifest (version skew: the newer -// controller parses what the newer manifest says) — the node-engine's excepted, which goes first; -// - it moves the node-engine and the other moves the controller (the witness reads nothing the controller -// does not yet grant, and a controller sends nothing an older engine would refuse — ADR 0236's rollout -// order). -// -// When rules order one pair both ways, the one of higher precedence wins (orderRank) and the pair says -// what it won over. Rules of one rank both ways are a contradiction: both are kept as pairs, the members -// are the cycle, and the contradiction says how to declare the order. Otherwise, by repository then id, so -// every reading gives one order. A cycle through three members or more is named by its members, and none -// of them is ordered. -func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge) groupOrder { +// orderOf is a group's order (novox/hq ADR 0419), pure: a member goes before another when a module it moves goes +// before a module the other moves by an edge of the module graph (built by), or by a rule of delivery_edges.go +// (verb before caller, version skew, engine before controller), which changes names the manifests' added verbs. +// No `after:` line is read. Ties break by repository then id, so every reading gives one order. A cycle through +// two members or more is named by its members, and none of them is ordered. +func orderOf(members []orderMember, reach map[string]orderReach, edges []inventory.Edge, changes map[string]moduleChange) groupOrder { out := groupOrder{Reach: reach} - byID := map[string]orderMember{} + movedBy := map[string][]string{} + var all []string for _, m := range members { - byID[m.ID] = m + for _, n := range reach[m.ID].Moved { + movedBy[n] = append(movedBy[n], m.ID) + all = appendNew(all, n) + } } - var claims []orderPair add := func(before, after, why string) { if before == after { return } - for _, p := range claims { + for _, p := range out.Pairs { if p.Before == before && p.After == after && p.Why == why { return } } - claims = append(claims, orderPair{Before: before, After: after, Why: why}) + out.Pairs = append(out.Pairs, orderPair{Before: before, After: after, Why: why}) } - named := func(said, repository string) bool { - said = strings.TrimSuffix(strings.TrimSpace(said), ".git") - if strings.EqualFold(said, repository) { - return true + for _, e := range edges { + if !orders(e) { + continue } - _, repo, _ := strings.Cut(repository, "/") - return strings.EqualFold(said, repo) + for _, before := range movedBy[e.To] { + for _, after := range movedBy[e.From] { + add(before, after, orderBuiltBy) + } + } } - for _, a := range members { - for _, b := range members { - if a.ID == b.ID { - continue - } - ra, rb := reach[a.ID], reach[b.ID] - for _, said := range b.After { - if named(said, a.Repository) { - add(a.ID, b.ID, orderDeclared) - } - } - for _, e := range edges { - if slices.Contains(ra.Moved, e.To) && slices.Contains(rb.Moved, e.From) && e.From != e.To { - add(a.ID, b.ID, orderBuiltBy) - break - } - } - if slices.Contains(ra.Moved, "mesh-controller") && len(rb.Manifests) > 0 && - !slices.Contains(rb.Moved, "mesh-controller") && !slices.Contains(rb.Moved, "mesh-host") { - add(a.ID, b.ID, orderVersionSkew) - } - if slices.Contains(ra.Moved, "mesh-host") && slices.Contains(rb.Moved, "mesh-controller") && - !slices.Contains(ra.Moved, "mesh-controller") { - add(a.ID, b.ID, orderEngineFirst) + order := orderEdges(all, changes, edges) + out.Edges = order.Edges + out.CycleSaid = order.CycleSaid() + for _, e := range order.Edges { + for _, before := range movedBy[e.To] { + for _, after := range movedBy[e.From] { + add(before, after, e.Why) } } } - out.Pairs, out.Contradictions = resolveClaims(claims, byID) sort.Slice(out.Pairs, func(i, j int) bool { if out.Pairs[i].Before != out.Pairs[j].Before { return out.Pairs[i].Before < out.Pairs[j].Before } - return out.Pairs[i].After < out.Pairs[j].After + if out.Pairs[i].After != out.Pairs[j].After { + return out.Pairs[i].After < out.Pairs[j].After + } + return out.Pairs[i].Why < out.Pairs[j].Why }) // Kahn's walk, the next always the first by repository and id among those with nothing before them. waiting := map[string]int{} @@ -779,89 +731,6 @@ func orderOf(members []orderMember, reach map[string]orderReach, edges []invento return out } -// orderRules is the rules in the order a pair names them when more than one says the same. -var orderRules = []string{orderDeclared, orderBuiltBy, orderVersionSkew, orderEngineFirst} - -// resolveClaims is the pairs the rules' claims come to (novox/hq ADR 0249): for each two members, the claims -// of the highest precedence decide, and the claims they overrule are said on the pair. Claims of one rank -// both ways are a contradiction: both pairs are kept, so the walk finds the cycle, and the contradiction -// says how a person declares the order. -func resolveClaims(claims []orderPair, byID map[string]orderMember) ([]orderPair, []orderContradiction) { - byTwo := map[[2]string][]orderPair{} - var keys [][2]string - for _, c := range claims { - k := [2]string{min(c.Before, c.After), max(c.Before, c.After)} - if _, ok := byTwo[k]; !ok { - keys = append(keys, k) - } - byTwo[k] = append(byTwo[k], c) - } - slices.SortFunc(keys, func(a, b [2]string) int { - if c := strings.Compare(a[0], b[0]); c != 0 { - return c - } - return strings.Compare(a[1], b[1]) - }) - ruleIndex := func(why string) int { return slices.Index(orderRules, why) } - var pairs []orderPair - var contradictions []orderContradiction - for _, k := range keys { - cs := byTwo[k] - slices.SortStableFunc(cs, func(a, b orderPair) int { return ruleIndex(a.Why) - ruleIndex(b.Why) }) - top := orderRank(cs[0].Why) - var won []orderPair - ways := map[string]bool{} - for _, c := range cs { - if orderRank(c.Why) == top { - won = append(won, c) - ways[c.Before] = true - } - } - if len(ways) == 1 { - p := orderPair{Before: won[0].Before, After: won[0].After, Why: won[0].Why} - for _, c := range cs { - if c.Before != p.Before && !slices.Contains(p.Over, c.Why) { - p.Over = append(p.Over, c.Why) - } - } - pairs = append(pairs, p) - continue - } - // One rank both ways: no precedence resolves it. - var first, second *orderPair - var said []string - for i := range won { - c := won[i] - if c.Before == k[0] && first == nil { - first = &won[i] - } - if c.Before == k[1] && second == nil { - second = &won[i] - } - said = append(said, fmt.Sprintf("%s puts %s first", c.Why, c.Before)) - } - pairs = append(pairs, orderPair{Before: first.Before, After: first.After, Why: first.Why}, - orderPair{Before: second.Before, After: second.After, Why: second.Why}) - a, b := byID[k[0]], byID[k[1]] - how := fmt.Sprintf("neither outranks the other: declare the order with a line `after: %s` in %s's "+ - "description, or `after: %s` in %s's", a.Repository, pullName(b), b.Repository, pullName(a)) - if top == orderRank(orderDeclared) { - how = "each pull request declares it goes after the other: remove one of the `after:` lines" - } - contradictions = append(contradictions, orderContradiction{Members: []string{k[0], k[1]}, Rules: won, - Said: fmt.Sprintf("%s and %s: %s; %s", k[0], k[1], strings.Join(said, ", "), how)}) - } - return pairs, contradictions -} - -// pullName is a member as a person finds it: its pull request, or its id when no number was said. -func pullName(m orderMember) string { - if m.Number > 0 { - return fmt.Sprintf("%s#%d", m.Repository, m.Number) - } - return m.ID -} - // reachOfMembers is each member's reach, as the planner says it. func reachOfMembers(members []orderMember, entries []inventory.Entry, read map[string][]inventory.ReadRepository, edges []inventory.Edge) map[string]orderReach { @@ -959,7 +828,11 @@ func deliveryCommand(ctx context.Context, args []string) error { if err != nil { return err } - return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges)) + changes, err := changesOf(ctx, changeSourcesOf(members, entries, reachOfMembers(members, entries, read, edges))) + if err != nil { + return fmt.Errorf("what the members say of their manifests cannot be read: %w", err) + } + return answer(orderOf(members, reachOfMembers(members, entries, read, edges), edges, changes)) case "check": if *group == "" && len(members) == 1 { // One head, checked again as the forge's announcement would have it (a recheck): the controller's @@ -1141,6 +1014,46 @@ func theGraph(ctx context.Context, inv *inventory.Inventory) ([]inventory.Entry, return entries, read, edges, nil } +// cycleVerdict is the failing verdict of a group whose modules are ordered in a cycle, and whether there is one: the +// group's composed check, said without a build. The manifests' change is read from the forge; an answer it cannot give +// is an error, never read as no cycle. +func cycleVerdict(ctx context.Context, group string, members []orderMember, reach map[string]orderReach, + edges []inventory.Edge, entries []inventory.Entry) (link.Checked, bool, error) { + changes, err := changesOf(ctx, changeSourcesOf(members, entries, reach)) + if err != nil { + return link.Checked{}, false, fmt.Errorf("what the group's members say of their manifests cannot be read: %w", err) + } + var moved []string + for _, m := range members { + moved = appendNew(moved, reach[m.ID].Moved...) + } + said := orderEdges(moved, changes, edges).CycleSaid() + if said == "" { + return link.Checked{}, false, nil + } + first := members[0].pull() + c := link.Checked{ID: link.NewBuildID(time.Now()), Owner: first.Owner, Repo: first.Repo, Number: first.Number, + Commit: first.Commit, Verdict: "fail", Summary: said, Report: said, Group: group, + Gate: &link.CheckLayer{Verdict: "fail", Summary: said}} + for _, m := range members { + if group == "" { + break // a lone pull request's verdict names no group and no member + } + p := m.pull() + c.Members = append(c.Members, link.CheckedMember{Owner: p.Owner, Repo: p.Repo, Number: p.Number, Commit: p.Commit, + Trunk: twelve(p.BaseSHA)}) + } + return c, true, nil +} + +// twelve is a commit as twelve characters. +func twelve(c string) string { + if len(c) > 12 { + return c[:12] + } + return c +} + // groupPrimary is the head a group's composed check is run from: the one moving the controller, which then // judges the group by itself; else the one moving the node-engine, whose validator judges; else the first. func groupPrimary(members []orderMember, reach map[string]orderReach) int { @@ -1163,6 +1076,14 @@ func askGroupCheck(ctx context.Context, open *stores, group string, members []or return "", err } reach := reachOfMembers(members, entries, read, edges) + // **A cycle of edges fails the group's composed check, naming every module and edge** (novox/hq ADR 0419 clause 6), + // before the build seat is asked: no composition of the heads has an order to be sent in. + if failing, cyclic, err := cycleVerdict(ctx, group, members, reach, edges, entries); err != nil { + return "", err + } else if cyclic { + sayChecked(ctx, failing) + return failing.ID, nil + } primary := groupPrimary(members, reach) p := members[primary].pull() scope := pullScope(p, entries, read, edges) diff --git a/cmd/mesh-controller/delivery_changes.go b/cmd/mesh-controller/delivery_changes.go new file mode 100644 index 00000000..2fb5f82f --- /dev/null +++ b/cmd/mesh-controller/delivery_changes.go @@ -0,0 +1,271 @@ +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "path" + "slices" + "strings" + "sync" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +// What a change says of the manifests it touches, read from the forge (novox/hq ADR 0419 clause 3, the lead's settlement +// of issue 977 question 1): the controller asks the forge for each changed module.json at the commit and at what it +// replaced, once per commit. The composed check's result is not used: the plan is cut after the merge. + +// changeSource is one change to a repository whose manifests are read: the commit after it and, when known, what it +// replaced (a pull request's base commit or branch). An empty Old is the first parent of New: the merge's base. +type changeSource struct { + Owner, Repo string + New, Old string + Paths []string + Truncated bool + Removed []string + // Since is the first commit of the repository in the change, when the change is several merges: what it replaced + // is the first parent of that one, so a verb an earlier merge added is added by the change too. + Since string +} + +// manifestAt reads a repository's module.json at a ref from the forge; the bool says it exists there. A variable so a +// test needs no forge. +var manifestAt = func(ctx context.Context, owner, repo, file, ref string) (catalogue.Manifest, bool, error) { + var found struct { + Result struct { + Content string `json:"content"` + Encoding string `json:"encoding"` + } `json:"result"` + } + missing, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/contents/"+escapedPath(file)+"?ref="+url.QueryEscape(ref), &found) + if err != nil { + return catalogue.Manifest{}, false, err + } + if missing { + return catalogue.Manifest{}, false, nil + } + body := []byte(found.Result.Content) + if found.Result.Encoding == "base64" { + if body, err = base64.StdEncoding.DecodeString(strings.ReplaceAll(found.Result.Content, "\n", "")); err != nil { + return catalogue.Manifest{}, false, fmt.Errorf("%s/%s %s at %s cannot be read: %w", owner, repo, file, ref, err) + } + } + var m catalogue.Manifest + if err := json.Unmarshal(body, &m); err != nil { + return catalogue.Manifest{}, false, fmt.Errorf("%s/%s %s at %s is not a manifest: %w", owner, repo, file, ref, err) + } + return m, true, nil +} + +// parentOf is the first parent of a commit, from the forge. A variable so a test needs no forge. +var parentOf = func(ctx context.Context, owner, repo, commit string) (string, error) { + var found struct { + Result struct { + Parents []struct { + SHA string `json:"sha"` + } `json:"parents"` + } `json:"result"` + } + if _, err := askForgeApi(ctx, "/repos/"+owner+"/"+repo+"/git/commits/"+url.PathEscape(commit), &found); err != nil { + return "", err + } + if len(found.Result.Parents) == 0 { + return "", nil + } + return found.Result.Parents[0].SHA, nil +} + +func escapedPath(p string) string { + parts := strings.Split(p, "/") + for i := range parts { + parts[i] = url.PathEscape(parts[i]) + } + return strings.Join(parts, "/") +} + +// askForgeApi makes one read of the forge's API through its module's tool on the bus. It answers true when the forge +// says there is no such thing (404), and an error for anything else it cannot answer. +func askForgeApi(ctx context.Context, apiPath string, into any) (bool, error) { + js, err := aBus() + if err != nil { + return false, err + } + defer js.Close() + raw, _ := json.Marshal(map[string]any{"path": apiPath}) + answer, err := link.Ask(ctx, link.OverNATS{Conn: js.Conn()}, "gitea", "gitea_api", raw, 30*time.Second) + if err != nil { + return false, err + } + if answer.Error != "" { + if strings.Contains(answer.Error, " 404 ") { + return true, nil + } + return false, fmt.Errorf("the forge did not answer %s: %s", apiPath, answer.Error) + } + if err := json.Unmarshal(answer.Result, into); err != nil { + return false, fmt.Errorf("the forge's answer to %s cannot be read: %w", apiPath, err) + } + return false, nil +} + +// manifestsRead keeps what the forge said of a manifest at a commit, which never changes: one ask per commit. +var ( + manifestsRead = map[string]manifestRead{} + manifestsReadMu sync.Mutex +) + +type manifestRead struct { + manifest catalogue.Manifest + exists bool +} + +func manifestCached(ctx context.Context, owner, repo, file, ref string) (catalogue.Manifest, bool, error) { + key := strings.Join([]string{owner, repo, file, ref}, "\x00") + if len(ref) == 40 { + manifestsReadMu.Lock() + got, ok := manifestsRead[key] + manifestsReadMu.Unlock() + if ok { + return got.manifest, got.exists, nil + } + } + m, exists, err := manifestAt(ctx, owner, repo, file, ref) + if err == nil && len(ref) == 40 { + manifestsReadMu.Lock() + if len(manifestsRead) > 512 { + clear(manifestsRead) + } + manifestsRead[key] = manifestRead{m, exists} + manifestsReadMu.Unlock() + } + return m, exists, err +} + +// verbsServed are the seat verbs a manifest serves, as ".": those of the seats it defines, those of the +// seats it claims, and its own tools as ".". +func verbsServed(m catalogue.Manifest) []string { + var out []string + for _, s := range m.DefinesSeats { + for _, v := range s.Serves { + out = appendNew(out, s.Name+"."+v.Name) + } + } + for _, c := range m.Claims { + for _, v := range c.ServesFor(m) { + out = appendNew(out, c.Name+"."+v) + } + } + for _, t := range m.Tools { + out = appendNew(out, m.Module+"."+t) + } + return out +} + +// verbsInvoked are the verbs a manifest calls, as "." or ".". +func verbsInvoked(m catalogue.Manifest) []string { + var out []string + for _, i := range m.Invokes { + if i = strings.TrimPrefix(i, "seat:"); i != "*" && i != "" { + out = appendNew(out, i) + } + } + return out +} + +func added(now, was []string) []string { + var out []string + for _, v := range now { + if !slices.Contains(was, v) { + out = append(out, v) + } + } + return out +} + +// changesOf reads, for each manifest the changes touch, what they add to the verbs its module serves and invokes, by +// module. A change whose paths are unsaid or truncated names no manifest: nothing is read for it. An error is the +// forge's, never read as "no change": the plan is not cut on a guess. +func changesOf(ctx context.Context, sources []changeSource) (map[string]moduleChange, error) { + out := map[string]moduleChange{} + for _, s := range sources { + if s.Truncated { + continue + } + for _, p := range s.Paths { + if path.Base(p) != "module.json" || slices.Contains(s.Removed, p) { + continue + } + now, exists, err := manifestCached(ctx, s.Owner, s.Repo, p, s.New) + if err != nil { + return nil, err + } + if !exists || now.Module == "" { + continue + } + old := s.Old + if old == "" { + first := s.Since + if first == "" { + first = s.New + } + if old, err = parentOf(ctx, s.Owner, s.Repo, first); err != nil { + return nil, err + } + } + var was catalogue.Manifest + if old != "" { + if was, _, err = manifestCached(ctx, s.Owner, s.Repo, p, old); err != nil { + return nil, err + } + } + c := out[now.Module] + c.Manifest = true + for _, v := range added(verbsServed(now), verbsServed(was)) { + c.Serves = appendNew(c.Serves, v) + } + for _, v := range added(verbsInvoked(now), verbsInvoked(was)) { + c.Invokes = appendNew(c.Invokes, v) + } + out[now.Module] = c + } + } + return out, nil +} + +// truncatedPaths are the manifests a change that names only a beginning of its files is read as changing: those of the +// modules it moves that are built from its repository. Said, never skipped: no edge may be lost to a long list. +func truncatedPaths(entries []inventory.Entry, owner, repo string, moved []string) []string { + var out []string + for _, e := range entries { + if slices.Contains(moved, e.Manifest.Module) && sameRepository(e.Source.Repository, link.SourceMoved{Owner: owner, Repo: repo}) { + out = append(out, path.Join(strings.Trim(e.Source.Path, "/"), moduleManifestFile)) + } + } + fmt.Printf(" %s/%s changed more files than were listed: the manifests of its modules (%s) are read as changed\n", + owner, repo, strings.Join(out, ", ")) + return out +} + +// changeSourcesOf are a group's members as changes to read: each head over what its pull request merges into. +func changeSourcesOf(members []orderMember, entries []inventory.Entry, reach map[string]orderReach) []changeSource { + var out []changeSource + for _, m := range members { + p := m.pull() + if p.PathsTruncated { + p.Paths, p.PathsTruncated = truncatedPaths(entries, p.Owner, p.Repo, reach[m.ID].Moved), false + } + old := p.BaseSHA + if old == "" { + old = p.Base + } + out = append(out, changeSource{Owner: p.Owner, Repo: p.Repo, New: p.Commit, Old: old, Paths: p.Paths, + Truncated: p.PathsTruncated, Removed: p.Removed}) + } + return out +} diff --git a/cmd/mesh-controller/delivery_conditions_test.go b/cmd/mesh-controller/delivery_conditions_test.go index 916b3ca3..cb3176fa 100644 --- a/cmd/mesh-controller/delivery_conditions_test.go +++ b/cmd/mesh-controller/delivery_conditions_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/delivery_edges.go b/cmd/mesh-controller/delivery_edges.go new file mode 100644 index 00000000..8126201f --- /dev/null +++ b/cmd/mesh-controller/delivery_edges.go @@ -0,0 +1,170 @@ +package main + +import ( + "fmt" + "slices" + "sort" + "strings" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// The order of a plan's modules is read from the module graph and from what the change says of the manifests +// (novox/hq ADR 0419), per module and never per repository: no `after:` line is read, and no member of a group +// is ordered against another. Pure, so a test needs no store. + +// The rules that draw an edge of a plan's order, in the words the plan shows. +const ( + ruleVerbBeforeCaller = "verb before caller" + ruleVersionSkew = "version skew" + ruleEngineFirst = "engine before controller" +) + +// The two modules the rollout rules name. +const ( + theController = "mesh-controller" + theEngine = "mesh-host" +) + +// moduleChange is what a change says of one module's manifest: the seat verbs (".") it adds to what the +// module serves, the seat verbs it adds to what the module invokes, and whether the manifest changes at all. +type moduleChange struct { + Serves []string + Invokes []string + Manifest bool +} + +// planOrder is the edges the rules draw among a plan's modules, and the cycle they make when they make one. +type planOrder struct { + Edges []inventory.Edge + // Cycle is every edge of a cycle among the order's edges and the graph's, the module graph's included. + Cycle []inventory.Edge +} + +// CycleSaid names every module and edge of the cycle, for a check to fail with (ADR 0419 clause 6). +func (o planOrder) CycleSaid() string { + if len(o.Cycle) == 0 { + return "" + } + var parts []string + for _, e := range o.Cycle { + parts = append(parts, edgeSaid(e)) + } + return "the modules of the plan are ordered in a cycle: " + strings.Join(parts, "; ") +} + +// edgeSaid is an edge as a person reads it: the module after, the module before, and why. +func edgeSaid(e inventory.Edge) string { + why := e.Why + if why == "" { + why = e.Kind + } + return fmt.Sprintf("%s after %s (%s)", e.From, e.To, why) +} + +// orders is whether an edge of the module graph orders a plan's tiers (tiersOf). +func orders(e inventory.Edge) bool { + return e.Kind != inventory.EdgePackages && e.Kind != inventory.EdgeBuiltBy && e.From != e.To +} + +// orderEdges draws the edges of a plan's order among its moved modules (ADR 0419 clause 3, 4): a module whose change +// adds a verb to a seat goes before one whose change adds a call of that verb; the controller, when the plan moves +// it, goes before every module whose manifest the plan changes, the node-engine excepted; and the node-engine goes +// before the controller by default, when the plan moves both and no edge orders them either way, transitively. An +// edge is "From after To". graph is the module graph, whose edges order too and are never drawn again. +func orderEdges(moved []string, changes map[string]moduleChange, graph []inventory.Edge) planOrder { + var out planOrder + sorted := append([]string(nil), moved...) + sort.Strings(sorted) + for _, server := range sorted { + for _, caller := range sorted { + if server == caller { + continue + } + for _, verb := range changes[server].Serves { + if slices.Contains(changes[caller].Invokes, verb) { + out.Edges = append(out.Edges, inventory.Edge{From: caller, To: server, Kind: edgeChangeOrder, + Why: ruleVerbBeforeCaller + ": " + verb}) + } + } + } + } + if slices.Contains(sorted, theController) { + for _, m := range sorted { + if m != theController && m != theEngine && changes[m].Manifest { + out.Edges = append(out.Edges, inventory.Edge{From: m, To: theController, Kind: edgeChangeOrder, + Why: ruleVersionSkew}) + } + } + } + if slices.Contains(sorted, theController) && slices.Contains(sorted, theEngine) { + all := orderingEdges(graph, out.Edges) + switch { + case pathBetween(all, theController, theEngine) != nil: + // An edge already puts the controller after the engine: the default says the same. + case pathBetween(all, theEngine, theController) != nil: + // An edge puts the engine after the controller: it overrules the default, and says so. + path := pathBetween(all, theEngine, theController) + for i := range out.Edges { + if out.Edges[i] == path[0] { + out.Edges[i].Why += " (overrules " + ruleEngineFirst + ")" + break + } + } + default: + out.Edges = append(out.Edges, inventory.Edge{From: theController, To: theEngine, Kind: edgeChangeOrder, + Why: ruleEngineFirst}) + } + } + all := orderingEdges(graph, out.Edges) + for _, e := range out.Edges { + if back := pathBetween(all, e.To, e.From); back != nil { + out.Cycle = append(out.Cycle, e) + out.Cycle = append(out.Cycle, back...) + break + } + } + return out +} + +// orderingEdges is the graph's edges that order and the edges drawn, together. +func orderingEdges(graph, drawn []inventory.Edge) []inventory.Edge { + var out []inventory.Edge + for _, e := range graph { + if orders(e) { + out = append(out, e) + } + } + return append(out, drawn...) +} + +// pathBetween is the edges that lead from a module to another along "depends on" (From to To), the shortest, or nil +// when the first does not depend on the second. +func pathBetween(edges []inventory.Edge, from, to string) []inventory.Edge { + if from == to { + return nil + } + via := map[string]inventory.Edge{} + seen := map[string]bool{from: true} + queue := []string{from} + for len(queue) > 0 { + at := queue[0] + queue = queue[1:] + for _, e := range edges { + if e.From != at || seen[e.To] { + continue + } + seen[e.To] = true + via[e.To] = e + if e.To == to { + var path []inventory.Edge + for m := to; m != from; m = via[m].From { + path = append([]inventory.Edge{via[m]}, path...) + } + return path + } + queue = append(queue, e.To) + } + } + return nil +} diff --git a/cmd/mesh-controller/delivery_edges_test.go b/cmd/mesh-controller/delivery_edges_test.go new file mode 100644 index 00000000..195aa2eb --- /dev/null +++ b/cmd/mesh-controller/delivery_edges_test.go @@ -0,0 +1,212 @@ +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// The order of a plan's modules is read from the module graph and the manifests' change alone (novox/hq ADR 0419): +// no `after:` line, no member of a group. Pure, like orderOf was. + +func hasEdge(edges []inventory.Edge, from, to string) (inventory.Edge, bool) { + for _, e := range edges { + if e.From == from && e.To == to { + return e, true + } + } + return inventory.Edge{}, false +} + +// A module whose change adds a verb to a seat goes before the module whose change adds a call of it: the caller +// depends on the server, the plan puts them in two tiers, and the edge says why. +func TestAVerbIsServedBeforeItsCallerIsSent(t *testing.T) { + changes := map[string]moduleChange{ + "server": {Serves: []string{"mesh-delivery.groups"}, Manifest: true}, + "caller": {Invokes: []string{"mesh-delivery.groups"}, Manifest: true}, + } + got := orderEdges([]string{"caller", "server"}, changes, nil) + e, ok := hasEdge(got.Edges, "caller", "server") + if !ok || !strings.Contains(e.Why, "verb before caller") || !strings.Contains(e.Why, "mesh-delivery.groups") { + t.Fatalf("the caller does not follow the server by the verb: %+v", got.Edges) + } + if len(got.Cycle) != 0 { + t.Fatalf("a cycle: %+v", got.Cycle) + } + if tiers := tiersOf([]string{"caller", "server"}, got.Edges); len(tiers) != 2 || tiers[0][0] != "server" { + t.Fatalf("tiers %v, want the server first", tiers) + } +} + +// Edges of one rule both ways are a cycle: every module and edge is named. +func TestEdgesBothWaysAreACycleNamingEveryModuleAndEdge(t *testing.T) { + changes := map[string]moduleChange{ + "x": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}}, + "y": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}}, + } + got := orderEdges([]string{"x", "y"}, changes, nil) + if len(got.Cycle) < 2 { + t.Fatalf("no cycle found: %+v", got) + } + said := got.CycleSaid() + for _, want := range []string{"x", "y", "s.a", "s.b"} { + if !strings.Contains(said, want) { + t.Fatalf("the cycle is said as %q, without %s", said, want) + } + } +} + +// The node-engine goes before the controller by default, and says so; an edge of the manifests the other way +// overrules it, and the edge that did is said. +func TestTheEngineGoesBeforeTheControllerUnlessAnEdgeSaysOtherwise(t *testing.T) { + got := orderEdges([]string{"mesh-controller", "mesh-host"}, nil, nil) + e, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host") + if !ok || e.Why != "engine before controller" { + t.Fatalf("the engine does not go first: %+v", got.Edges) + } + changes := map[string]moduleChange{ + "mesh-host": {Serves: []string{"s.v"}}, + "mesh-controller": {Invokes: []string{"s.v"}}, + } + got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil) + if _, ok := hasEdge(got.Edges, "mesh-controller", "mesh-host"); !ok || len(got.Edges) != 1 || + !strings.Contains(got.Edges[0].Why, "verb before caller") { + t.Fatalf("the verb edge does not place the controller after the engine: %+v", got.Edges) + } + // The other way: the engine calls what the controller brings. + changes = map[string]moduleChange{ + "mesh-controller": {Serves: []string{"s.v"}}, + "mesh-host": {Invokes: []string{"s.v"}}, + } + got = orderEdges([]string{"mesh-controller", "mesh-host"}, changes, nil) + e, ok = hasEdge(got.Edges, "mesh-host", "mesh-controller") + if !ok || len(got.Edges) != 1 || !strings.Contains(e.Why, "overrules engine before controller") { + t.Fatalf("the verb edge does not say it overruled the default: %+v", got.Edges) + } +} + +// The controller goes before every module whose manifest the plan changes (version skew), the node-engine excepted. +func TestTheControllerGoesBeforeEveryChangedManifest(t *testing.T) { + changes := map[string]moduleChange{"app": {Manifest: true}, "other": {}} + got := orderEdges([]string{"mesh-controller", "app", "other"}, changes, nil) + if e, ok := hasEdge(got.Edges, "app", "mesh-controller"); !ok || e.Why != "version skew" { + t.Fatalf("the changed manifest does not follow the controller: %+v", got.Edges) + } + if _, ok := hasEdge(got.Edges, "other", "mesh-controller"); ok { + t.Fatalf("a module with an unchanged manifest was ordered: %+v", got.Edges) + } + if got := orderEdges([]string{"app", "other"}, changes, nil); len(got.Edges) != 0 { + t.Fatalf("version skew ordered a plan that does not move the controller: %+v", got.Edges) + } +} + +// The order is per module, not per repository: of two modules of one repository only the one with the edge moves. +func TestTheOrderIsPerModuleNotPerRepository(t *testing.T) { + changes := map[string]moduleChange{ + "a": {Serves: []string{"s.v"}}, + "c": {Invokes: []string{"s.v"}}, + } + got := orderEdges([]string{"a", "b", "c"}, changes, nil) + tiers := tiersOf([]string{"a", "b", "c"}, got.Edges) + if len(tiers) != 2 || strings.Join(tiers[0], ",") != "a,b" || strings.Join(tiers[1], ",") != "c" { + t.Fatalf("tiers %v, want [a b] then [c]", tiers) + } +} + +// The edges a plan's order was drawn with are kept on the plan, and its gates read them: a later tier waits for the +// earlier to be applied although the catalogue's graph holds no such edge (ADR 0419, the lead's settlement Q4). +func TestAPlansOwnOrderGatesItsTiersWhateverTheStoreGraphHolds(t *testing.T) { + p := inventory.Plan{Tiers: [][]string{{"server"}, {"caller"}}, Tier: 0, + Delivery: &inventory.PlanDelivery{Order: []inventory.Edge{{From: "caller", To: "server", Kind: edgeChangeOrder, + Why: ruleVerbBeforeCaller + ": s.v"}}}} + if got := gates(p, nil, func(string) bool { return true }); len(got) != 1 || got[0] != "server" { + t.Fatalf("the caller's tier waits for %v; want the server", got) + } + p.Delivery.Order = nil + if got := gates(p, nil, func(string) bool { return true }); len(got) != 0 { + t.Fatalf("a plan with no order waits for %v", got) + } +} + +// A verb an earlier merge of the same repository in the batch added is added by the batch: the change is compared with +// what the repository held before the batch, not with the first parent of its latest merge. +func TestAVerbAnEarlierMergeOfTheBatchAddedDrawsItsEdge(t *testing.T) { + was, wasParent := manifestAt, parentOf + t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) + manifestsReadMu.Lock() + clear(manifestsRead) + manifestsReadMu.Unlock() + const first, second, before = "1111111111111111111111111111111111111111", "2222222222222222222222222222222222222222", "0000000000000000000000000000000000000000" + parentOf = func(_ context.Context, _, _, commit string) (string, error) { + if commit == first { + return before, nil + } + return first, nil // the latest merge's parent holds the verb already + } + manifestAt = func(_ context.Context, _, _, _, ref string) (catalogue.Manifest, bool, error) { + m := catalogue.Manifest{Module: "server"} + if ref != before { + m.DefinesSeats = []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "v"}}}} + } + return m, true, nil + } + src := changeSource{Owner: "novox", Repo: "r", New: second, Paths: []string{"module.json"}} + if got, err := changesOf(context.Background(), []changeSource{src}); err != nil || len(got["server"].Serves) != 0 { + t.Fatalf("against the first parent of the latest merge: %+v %v", got, err) + } + src.Since = first + got, err := changesOf(context.Background(), []changeSource{src}) + if err != nil || len(got["server"].Serves) != 1 || got["server"].Serves[0] != "s.v" { + t.Fatalf("against what the batch found: %+v %v", got, err) + } +} + +// A change that names only a beginning of its files is read as changing the manifests of the modules it moves. +func TestATruncatedChangeIsReadAsChangingTheManifestsOfWhatItMoves(t *testing.T) { + entries := []inventory.Entry{ + {Manifest: catalogue.Manifest{Module: "a"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/a"}}, + {Manifest: catalogue.Manifest{Module: "b"}, Source: inventory.Source{Repository: "novox/mesh-catalog", Path: "modules/b"}}, + {Manifest: catalogue.Manifest{Module: "c"}, Source: inventory.Source{Repository: "novox/other", Path: "modules/c"}}, + } + got := truncatedPaths(entries, "novox", "mesh-catalog", []string{"a", "c"}) + if len(got) != 1 || got[0] != "modules/a/module.json" { + t.Fatalf("read as changed: %v", got) + } + members := []orderMember{{ID: "m", Repository: "novox/mesh-catalog", Head: "h", PathsTruncated: true, Paths: []string{"x"}}} + src := changeSourcesOf(members, entries, map[string]orderReach{"m": {Moved: []string{"a"}}}) + if len(src) != 1 || src[0].Truncated || len(src[0].Paths) != 1 || src[0].Paths[0] != "modules/a/module.json" { + t.Fatalf("the member's change is read as %+v", src) + } +} + +// A lone pull request's cycle fails its check, naming every module and edge, with no group and no members. +func TestALonePullRequestInACycleFailsWithNoGroup(t *testing.T) { + was, wasParent := manifestAt, parentOf + t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) + manifestsReadMu.Lock() + clear(manifestsRead) + manifestsReadMu.Unlock() + const head = "3333333333333333333333333333333333333333" + parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } + manifestAt = func(_ context.Context, _, _, file, ref string) (catalogue.Manifest, bool, error) { + if ref == "main" { + return catalogue.Manifest{}, false, nil + } + if file == "modules/x/module.json" { + return catalogue.Manifest{Module: "mx", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "a"}}}}, + Invokes: []string{"seat:s.b"}}, true, nil + } + return catalogue.Manifest{Module: "my", DefinesSeats: []catalogue.SeatDeclaration{{Name: "s", Serves: []catalogue.Verb{{Name: "b"}}}}, + Invokes: []string{"seat:s.a"}}, true, nil + } + m := orderMember{ID: "novox/r@" + head[:12], Repository: "novox/r", Base: "main", Head: head, Number: 5, + Paths: []string{"modules/x/module.json", "modules/y/module.json"}} + c, cyclic, err := cycleVerdict(context.Background(), "", []orderMember{m}, map[string]orderReach{m.ID: {Moved: []string{"mx", "my"}}}, nil, nil) + if err != nil || !cyclic || c.Verdict != "fail" || c.Group != "" || len(c.Members) != 0 || !strings.Contains(c.Summary, "s.a") || + !strings.Contains(c.Summary, "my") { + t.Fatalf("the verdict is %+v (cyclic %v, %v)", c, cyclic, err) + } +} diff --git a/cmd/mesh-controller/delivery_integration_test.go b/cmd/mesh-controller/delivery_integration_test.go new file mode 100644 index 00000000..ed85ec2f --- /dev/null +++ b/cmd/mesh-controller/delivery_integration_test.go @@ -0,0 +1,196 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + withConditionsInMemory(t) + for _, name := range []string{"app", "mesh-delivery"} { + m := catalogue.Manifest{Module: name, Version: "1"} + if name == "mesh-delivery" { + m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}} + } + if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + merge := func(commit string, paths ...string) inventory.Plan { + t.Helper() + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths, + ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + recent, err := inv.RecentPlans(ctx, 1) + if err != nil || len(recent) != 1 || recent[0].Commit != commit { + t.Fatalf("no plan for %s: %v %v", commit, recent, err) + } + return recent[0] + } + + // finish ends a walk as done, so the next merge has nothing of it to take over. + finish := func(id string) { + t.Helper() + w, err := inv.PlanByID(ctx, id) + if err != nil { + t.Fatal(err) + } + w.State = inventory.PlanDone + if err := inv.SavePlan(ctx, &w); err != nil { + t.Fatal(err) + } + } + + // No holder on record: the merge starts its walk, as before. + p := merge("c1aaaaaaaa", "modules/app/index.ts") + if p.Waiting() || len(*asked) != 1 { + t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) + } + + // The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk + // is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut. + finish(p.ID) + if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + p = merge("c2bbbbbbbb", "modules/app/index.ts") + if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) { + t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note) + } + advanceHeld(ctx, open) + if len(*asked) != 1 { + t.Fatalf("a waiting walk was advanced into asking: %v", *asked) + } + if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") { + t.Errorf("a waiting walk reads %q", line) + } + + // The delivery's word starts it. + if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil { + t.Fatal("a walk was let go twice") + } + advanceHeld(ctx, open) + if len(*asked) != 2 { + t.Fatalf("the word did not start the walk: %v", *asked) + } + got, err := inv.PlanByID(ctx, p.ID) + if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" { + t.Fatalf("the word was not kept: %+v %v", got.Delivery, err) + } + + // The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq + // ADR 0276): the merge's batch is cut once it ended. + finish(p.ID) + p = merge("c3cccccccc", "modules/mesh-delivery/main.go") + if p.Waiting() || len(*asked) != 3 { + t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) + } + + // The owner down: a person starts a waiting walk, with why. + finish(p.ID) + p = merge("c4dddddddd", "modules/app/index.ts") + if !p.Waiting() { + t.Fatal("the walk did not wait") + } + if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") { + t.Fatalf("a walk was started by hand without why: %v", err) + } + if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + got, _ = inv.PlanByID(ctx, p.ID) + if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 { + t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked) + } + + // The owner's stop: failed, said as stopped by it. + finish(p.ID) + p = merge("c5eeeeeeee", "modules/app/index.ts") + if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by", + "mesh-delivery for jochen"}); err != nil { + t.Fatal(err) + } + got, _ = inv.PlanByID(ctx, p.ID) + if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" || + !strings.Contains(got.Note, "the operator stopped it") { + t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note) + } + + // Unassigned: the mesh is back on the controller's own path. + if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + asks := len(*asked) + if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 { + t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) + } +} + +func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + url := testbus.URL(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + + waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7", + Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + var said inventory.Plan + if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil || + said.Delivery.Go == nil { + t.Fatalf("plan-moved said %s (%v)", m.Data, err) + } + case <-time.After(5 * time.Second): + t.Fatal("a walk let go by a verb was not said") + } + if checkEvents != nil || inventory.PlanSaved != nil { + t.Fatal("the command's own bus was left in place") + } +} diff --git a/cmd/mesh-controller/delivery_test.go b/cmd/mesh-controller/delivery_test.go index e352028e..d7f1b0df 100644 --- a/cmd/mesh-controller/delivery_test.go +++ b/cmd/mesh-controller/delivery_test.go @@ -1,50 +1,44 @@ package main import ( - "encoding/json" "reflect" - "slices" "strings" "testing" - "time" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) -// novox/hq ADR 0239 decision 4: a group's order, from the graph and the pull requests, the same on every -// reading — the build agent before what it builds, the controller before a manifest that needs it, the -// node-engine before the controller, a declared `after:` — and a contradiction named, never ordered. -func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) { +// novox/hq ADR 0419: a group's order, from the module graph and the manifests' change alone, the same on every +// reading — what a module stands on before it, the controller before a manifest that needs it, the node-engine +// before the controller, a verb's server before its caller — and no `after:` line read. +func TestAGroupIsOrderedByTheGraphAndTheManifestsChange(t *testing.T) { members := []orderMember{ {ID: "cat", Repository: "novox/mesh-catalog"}, {ID: "ctl", Repository: "novox/mesh-controller"}, {ID: "host", Repository: "novox/mesh-host"}, - {ID: "agent", Repository: "novox/build-agent"}, - {ID: "app", Repository: "novox/app", After: []string{"lab"}}, + {ID: "base", Repository: "novox/base"}, + {ID: "app", Repository: "novox/app"}, {ID: "lab", Repository: "novox/lab"}, } reach := map[string]orderReach{ - "cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}}, - "ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}}, - "host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}}, - "agent": {Moved: []string{"build-agent"}}, - "app": {Moved: []string{"app"}}, - "lab": {Moved: []string{"lab"}}, + "cat": {Moved: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}}, + "ctl": {Moved: []string{"mesh-controller"}, Manifests: []string{"module.json"}}, + "host": {Moved: []string{"mesh-host"}, Manifests: []string{"module.json"}}, + "base": {Moved: []string{"base"}}, + "app": {Moved: []string{"app"}}, + "lab": {Moved: []string{"lab"}}, } - edges := []inventory.Edge{{From: "app", To: "build-agent", Kind: inventory.EdgeBuiltBy}, + edges := []inventory.Edge{{From: "app", To: "base", Kind: inventory.EdgeStandsOn}, {From: "gitea", To: "postgres", Kind: inventory.EdgeDeclared}} - got := orderOf(members, reach, edges) + changes := map[string]moduleChange{"gitea": {Manifest: true}, "mesh-controller": {Manifest: true}, + "mesh-host": {Manifest: true}, "lab": {Serves: []string{"seat.v"}}, "app": {Invokes: []string{"seat.v"}}} + got := orderOf(members, reach, edges, changes) if len(got.Cycle) > 0 { t.Fatalf("a cycle where there is none: %v", got.Cycle) } - // By repository among those with nothing before them: the build agent, the lab, then what waited on both. - want := []string{"agent", "lab", "app", "host", "ctl", "cat"} + // By repository among those with nothing before them: app waits for base and lab; gitea's catalogue for the controller. + want := []string{"base", "lab", "app", "host", "ctl", "cat"} if !reflect.DeepEqual(got.Order, want) { t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs) } @@ -52,95 +46,44 @@ func TestAGroupIsOrderedByTheGraphAndWhatItsPullRequestsSay(t *testing.T) { for _, p := range got.Pairs { why[p.Before+">"+p.After] = p.Why } - for pair, reason := range map[string]string{"host>ctl": orderEngineFirst, "ctl>cat": orderVersionSkew, - "ctl>host": "", "agent>app": orderBuiltBy, "lab>app": orderDeclared} { + for pair, reason := range map[string]string{"host>ctl": ruleEngineFirst, "ctl>cat": ruleVersionSkew, + "ctl>host": "", "base>app": orderBuiltBy, "lab>app": ruleVerbBeforeCaller + ": seat.v"} { if why[pair] != reason { t.Errorf("%s is ordered %q, wanted %q", pair, why[pair], reason) } } // The same members in another order read the same. shuffled := []orderMember{members[5], members[3], members[0], members[4], members[2], members[1]} - if again := orderOf(shuffled, reach, edges); !reflect.DeepEqual(again.Order, want) { + if again := orderOf(shuffled, reach, edges, changes); !reflect.DeepEqual(again.Order, want) { t.Fatalf("another reading ordered %v", again.Order) } - - // A declared order against an inferred one wins (novox/hq ADR 0249), and the pair says what it won over. - members[1].After = []string{"mesh-catalog"} - got = orderOf(members, reach, edges) - if len(got.Cycle) > 0 || len(got.Contradictions) > 0 { - t.Fatalf("a declared order did not win: cycle %v, %+v", got.Cycle, got.Contradictions) - } - if want := []string{"agent", "lab", "app", "cat", "host", "ctl"}; !reflect.DeepEqual(got.Order, want) { - t.Fatalf("ordered %v, wanted %v", got.Order, want) - } - if !slices.ContainsFunc(got.Pairs, func(p orderPair) bool { - return p.Before == "cat" && p.After == "ctl" && p.Why == orderDeclared && reflect.DeepEqual(p.Over, []string{orderVersionSkew}) - }) || slices.ContainsFunc(got.Pairs, func(p orderPair) bool { return p.Before == "ctl" && p.After == "cat" }) { - t.Fatalf("the declared pair does not say it won over version skew: %+v", got.Pairs) - } - - // Declared both ways is a contradiction no precedence resolves: a cycle, and how to end it. - members[0].After = []string{"novox/mesh-controller"} - got = orderOf(members, reach, edges) - if !reflect.DeepEqual(got.Cycle, []string{"cat", "ctl"}) || len(got.Contradictions) != 1 { - t.Fatalf("the cycle is %v, ordered %v, contradictions %+v", got.Cycle, got.Order, got.Contradictions) - } - if c := got.Contradictions[0]; !reflect.DeepEqual(c.Members, []string{"cat", "ctl"}) || - !strings.Contains(c.Said, "remove one of the `after:` lines") { - t.Fatalf("the contradiction is said %+v", c) - } } -// novox/hq ADR 0249: when two rules order one pair both ways, the rule of higher precedence wins and the -// pair says what it overruled; rules of one rank both ways are refused, naming both and how to declare. -func TestAnOrderRuleYieldsToAStrongerOne(t *testing.T) { +// A cycle is the one refusal, and it names every module and edge. A description's `after:` line changes nothing. +func TestAGroupInACycleIsNamedAndNothingElseIsRefused(t *testing.T) { members := []orderMember{ - {ID: "ctl", Repository: "novox/mesh-controller", Number: 7}, - {ID: "host", Repository: "novox/mesh-host", Number: 9}, + {ID: "x", Repository: "novox/x", Number: 7}, + {ID: "y", Repository: "novox/y", Number: 9}, } - reach := map[string]orderReach{ - "ctl": {Moved: []string{"mesh-controller", "build-agent"}}, - "host": {Moved: []string{"mesh-host"}}, + reach := map[string]orderReach{"x": {Moved: []string{"mx"}}, "y": {Moved: []string{"my"}}} + changes := map[string]moduleChange{ + "mx": {Serves: []string{"s.a"}, Invokes: []string{"s.b"}}, + "my": {Serves: []string{"s.b"}, Invokes: []string{"s.a"}}, } - builtBy := []inventory.Edge{{From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}} - - // Built by outranks engine before controller. - got := orderOf(members, reach, builtBy) - if !reflect.DeepEqual(got.Order, []string{"ctl", "host"}) || len(got.Cycle) > 0 { - t.Fatalf("ordered %v, cycle %v", got.Order, got.Cycle) + got := orderOf(members, reach, nil, changes) + if !reflect.DeepEqual(got.Cycle, []string{"x", "y"}) || len(got.Order) > 0 { + t.Fatalf("a cycle was ordered: %v, cycle %v", got.Order, got.Cycle) } - if want := []orderPair{{Before: "ctl", After: "host", Why: orderBuiltBy, Over: []string{orderEngineFirst}}}; !reflect.DeepEqual(got.Pairs, want) { - t.Fatalf("pairs %+v, wanted %+v", got.Pairs, want) - } - - // A declared line outranks both. - members[0].After = []string{"mesh-host"} - got = orderOf(members, reach, builtBy) - if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || len(got.Cycle) > 0 || - !reflect.DeepEqual(got.Pairs[0].Over, []string{orderBuiltBy}) || got.Pairs[0].Why != orderDeclared { - t.Fatalf("a declared order did not win: %v %+v", got.Order, got.Pairs) - } - - // Without the build agent, engine before controller stands alone. - members[0].After = nil - got = orderOf(members, map[string]orderReach{"ctl": {Moved: []string{"mesh-controller"}}, "host": reach["host"]}, builtBy) - if !reflect.DeepEqual(got.Order, []string{"host", "ctl"}) || got.Pairs[0].Why != orderEngineFirst || len(got.Pairs[0].Over) > 0 { - t.Fatalf("engine before controller alone: %v %+v", got.Order, got.Pairs) - } - - // Built by both ways is one rank against itself: refused, both rules named, and how to declare. - both := append(builtBy, inventory.Edge{From: "build-agent", To: "mesh-host", Kind: inventory.EdgeStandsOn}) - got = orderOf(members, reach, both) - if !reflect.DeepEqual(got.Cycle, []string{"ctl", "host"}) || len(got.Order) > 0 || len(got.Contradictions) != 1 { - t.Fatalf("a contradiction was ordered: %v, cycle %v, %+v", got.Order, got.Cycle, got.Contradictions) - } - said := got.Contradictions[0].Said - for _, w := range []string{"built by puts ctl first", "built by puts host first", - "`after: novox/mesh-controller` in novox/mesh-host#9's description", "`after: novox/mesh-host` in novox/mesh-controller#7's"} { - if !strings.Contains(said, w) { - t.Errorf("the contradiction does not say %q: %s", w, said) + for _, w := range []string{"mx", "my", "s.a", "s.b"} { + if !strings.Contains(got.CycleSaid, w) { + t.Errorf("the cycle does not say %q: %s", w, got.CycleSaid) } } + // Without the edges the same members are ordered by repository. + got = orderOf(members, reach, nil, nil) + if !reflect.DeepEqual(got.Order, []string{"x", "y"}) || len(got.Cycle) > 0 || len(got.Pairs) > 0 { + t.Fatalf("members with no edge: %v %v %+v", got.Order, got.Cycle, got.Pairs) + } } // novox/hq ADR 0239 decision 8: a walk waits for the delivery's owner only while the seat has a holder on @@ -178,137 +121,6 @@ func TestAWalkWaitsOnlyWhileTheDeliverySeatIsHeldAndNeverForTheCore(t *testing.T // novox/hq ADR 0239: with mesh-delivery on record, a merge opens its walk and asks nothing until the // delivery's word; the word starts it; the core's own merge never waits; a person starts a waiting walk by // hand when the owner is down, and the owner's stop ends one as stopped. -func TestAMergeWaitsForItsDeliverysWordAndThePersonsWordWorksWithoutIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - withConditionsInMemory(t) - for _, name := range []string{"app", "mesh-delivery"} { - m := catalogue.Manifest{Module: name, Version: "1"} - if name == "mesh-delivery" { - m.Claims = []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}} - } - if err := inv.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/" + name, Ref: "main", BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - merge := func(commit string, paths ...string) inventory.Plan { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, Paths: paths, - ModuleDirs: []string{"modules/app", "modules/mesh-delivery"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || recent[0].Commit != commit { - t.Fatalf("no plan for %s: %v %v", commit, recent, err) - } - return recent[0] - } - - // finish ends a walk as done, so the next merge has nothing of it to take over. - finish := func(id string) { - t.Helper() - w, err := inv.PlanByID(ctx, id) - if err != nil { - t.Fatal(err) - } - w.State = inventory.PlanDone - if err := inv.SavePlan(ctx, &w); err != nil { - t.Fatal(err) - } - } - - // No holder on record: the merge starts its walk, as before. - p := merge("c1aaaaaaaa", "modules/app/index.ts") - if p.Waiting() || len(*asked) != 1 { - t.Fatalf("with no holder on record the walk waited (%v) or asked %v", p.Waiting(), *asked) - } - - // The holder on record: the next merge waits, asking nothing, and an advance asks nothing either. One walk - // is open at a time (novox/hq ADR 0276): the first ends before the next merge's batch is cut. - finish(p.ID) - if _, err := inv.Assign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - p = merge("c2bbbbbbbb", "modules/app/index.ts") - if !p.Waiting() || len(*asked) != 1 || !strings.Contains(p.Note, "plans go "+p.ID) { - t.Fatalf("the walk did not wait for its word: waiting %v, asked %v, note %q", p.Waiting(), *asked, p.Note) - } - advanceHeld(ctx, open) - if len(*asked) != 1 { - t.Fatalf("a waiting walk was advanced into asking: %v", *asked) - } - if line := planLine(p, p.Created); strings.Contains(line, "LATE") || !strings.Contains(line, "waits for") { - t.Errorf("a waiting walk reads %q", line) - } - - // The delivery's word starts it. - if err := deliveryCommand(ctx, []string{"go", p.ID, "--by", catalogue.DeliverySeat, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", p.ID}); err == nil { - t.Fatal("a walk was let go twice") - } - advanceHeld(ctx, open) - if len(*asked) != 2 { - t.Fatalf("the word did not start the walk: %v", *asked) - } - got, err := inv.PlanByID(ctx, p.ID) - if err != nil || got.Delivery == nil || got.Delivery.By != catalogue.DeliverySeat || got.Delivery.Why != "its turn" { - t.Fatalf("the word was not kept: %+v %v", got.Delivery, err) - } - - // The delivery's owner's own merge never waits for it. A walk that started is never taken over (novox/hq - // ADR 0276): the merge's batch is cut once it ended. - finish(p.ID) - p = merge("c3cccccccc", "modules/mesh-delivery/main.go") - if p.Waiting() || len(*asked) != 3 { - t.Fatalf("mesh-delivery's own walk waited for mesh-delivery: %v %v", p.Waiting(), *asked) - } - - // The owner down: a person starts a waiting walk, with why. - finish(p.ID) - p = merge("c4dddddddd", "modules/app/index.ts") - if !p.Waiting() { - t.Fatal("the walk did not wait") - } - if err := plansCommand(ctx, []string{"go", p.ID}); err == nil || !strings.Contains(err.Error(), "--why") { - t.Fatalf("a walk was started by hand without why: %v", err) - } - if err := plansCommand(ctx, []string{"go", p.ID, "--why", "mesh-delivery is down"}); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - got, _ = inv.PlanByID(ctx, p.ID) - if got.Waiting() || !strings.HasPrefix(got.Delivery.By, "a person") || len(*asked) < 4 { - t.Fatalf("a person's word did not start the walk: %+v, asked %v", got.Delivery, *asked) - } - - // The owner's stop: failed, said as stopped by it. - finish(p.ID) - p = merge("c5eeeeeeee", "modules/app/index.ts") - if err := deliveryCommand(ctx, []string{"stop", p.ID, "--why", "the operator stopped it", "--by", - "mesh-delivery for jochen"}); err != nil { - t.Fatal(err) - } - got, _ = inv.PlanByID(ctx, p.ID) - if got.State != inventory.PlanFailed || got.Delivery.Stopped != "mesh-delivery for jochen" || - !strings.Contains(got.Note, "the operator stopped it") { - t.Fatalf("the stop was kept as %s %+v %q", got.State, got.Delivery, got.Note) - } - - // Unassigned: the mesh is back on the controller's own path. - if err := inv.Unassign(ctx, "anchor", "mesh-delivery"); err != nil { - t.Fatal(err) - } - asks := len(*asked) - if p = merge("c6ffffffff", "modules/app/index.ts"); p.Waiting() || len(*asked) != asks+1 { - t.Fatalf("with the holder gone the walk waited: %v", p.Waiting()) - } -} // The verbs mesh-delivery asks with become the commands they name, and nothing a caller sends is passed over. func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { @@ -345,49 +157,3 @@ func TestTheDeliveryVerbsComposeTheirCommands(t *testing.T) { // A verb runs as a command of its own: what it keeps of a walk is still said as plan-moved, on a bus of the // command's own, so the delivery's owner hears it at once and not only when it reads the walks back. -func TestAWalkLetGoByAVerbIsSaidAsPlanMoved(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - - waiting := inventory.Plan{ID: "plan-waits", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c7c7c7c7", - Created: time.Now().UTC(), State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - var said inventory.Plan - if err := json.Unmarshal(m.Data, &said); err != nil || said.ID != waiting.ID || said.Delivery == nil || - said.Delivery.Go == nil { - t.Fatalf("plan-moved said %s (%v)", m.Data, err) - } - case <-time.After(5 * time.Second): - t.Fatal("a walk let go by a verb was not said") - } - if checkEvents != nil || inventory.PlanSaved != nil { - t.Fatal("the command's own bus was left in place") - } -} diff --git a/cmd/mesh-controller/desk_secret_integration_test.go b/cmd/mesh-controller/desk_secret_integration_test.go new file mode 100644 index 00000000..39ba3b72 --- /dev/null +++ b/cmd/mesh-controller/desk_secret_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.command") + for _, args := range [][]string{ + {"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, + {"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"}, + } { + err := secretCommand(context.Background(), args) + if err == nil || !strings.Contains(err.Error(), "never through a verb") { + t.Errorf("%v: %v", args, err) + } + } +} + +func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) { + var order []string + announce := func(fail bool) func() error { + return func() error { + order = append(order, "announce") + if fail { + return errors.New("no channel") + } + return nil + } + } + keep := func() (bool, error) { order = append(order, "keep"); return false, nil } + + order = nil + if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil || + strings.Join(order, ",") != "announce,keep" { + t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order) + } + order = nil + if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" { + t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order) + } + order = nil + if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil || + strings.Join(order, ",") != "keep,announce" { + t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order) + } + order = nil + failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") } + if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" { + t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order) + } +} + +func TestASecretAskIsNeverASecretRead(t *testing.T) { + t.Setenv(verbVar, "mesh-controller.secret-ask") + for _, args := range [][]string{ + {"ask", "anchor", "telegram", "telegram-token", "the-value"}, + {"ask", "anchor", "telegram"}, + {"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, + } { + if err := secretCommand(context.Background(), args); err == nil { + t.Errorf("secret %v was taken", args) + } + } + for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} { + if err := terminalOnly(append([]string{"secret"}, args...)); err == nil { + t.Errorf("secret %v passed the terminal rule", args) + } + } +} + +func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) { + ctx := context.Background() + askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes }) + // conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be + // raised either: both of ADR 0380's ways to tell the operator fail. + kept := false + if _, _, err := keepGiven(true, + func() error { + return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me") + }, + func() (bool, error) { kept = true; return false, nil }); err == nil { + t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed") + } + if kept { + t.Error("the value was kept despite the failure") + } +} diff --git a/cmd/mesh-controller/desk_secret_test.go b/cmd/mesh-controller/desk_secret_test.go index 85bbb40e..363dba59 100644 --- a/cmd/mesh-controller/desk_secret_test.go +++ b/cmd/mesh-controller/desk_secret_test.go @@ -263,18 +263,6 @@ func TestOnlyTheControllerMayAskTheDesksPrompt(t *testing.T) { // A value for a secret comes from the terminal or the desk, never through a verb (the review of 2026-10-09, // M4): `secret accept` with a value, run for a verb, is refused before anything is read. -func TestASecretValueIsNeverAcceptedThroughAVerb(t *testing.T) { - t.Setenv(verbVar, "mesh-controller.command") - for _, args := range [][]string{ - {"accept", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, - {"accept", "anchor", "app", "db", "--from", "/dev/null", "--provider", "store"}, - } { - err := secretCommand(context.Background(), args) - if err == nil || !strings.Contains(err.Error(), "never through a verb") { - t.Errorf("%v: %v", args, err) - } - } -} // The `give` and `secret-ask` verbs' own line passes the terminal-only rule of ADR 0266, and no `secret accept` // does: a value, a file, a provider or an extra word is still the terminal's alone (novox/hq ADR 0277). @@ -355,39 +343,6 @@ func TestOnlyTheDeskMachinesLauncherMayAnswerItsPrompt(t *testing.T) { // N1-give at the controller's terminal (the confirmation review of 2026-10-09): a trusted party's secret is // announced before it is kept, and not kept when the announcement fails; another module's is kept first and // a failed announcement is said, not undone. -func TestATrustedPartysSecretGivenAtTheTerminalIsAnnouncedBeforeItIsKept(t *testing.T) { - var order []string - announce := func(fail bool) func() error { - return func() error { - order = append(order, "announce") - if fail { - return errors.New("no channel") - } - return nil - } - } - keep := func() (bool, error) { order = append(order, "keep"); return false, nil } - - order = nil - if _, unannounced, err := keepGiven(true, announce(false), keep); err != nil || unannounced != nil || - strings.Join(order, ",") != "announce,keep" { - t.Errorf("trusted: %v %v, order %v; want announced, then kept", unannounced, err, order) - } - order = nil - if _, _, err := keepGiven(true, announce(true), keep); err == nil || strings.Join(order, ",") != "announce" { - t.Errorf("trusted, announcement failed: %v, order %v; want refused and nothing kept", err, order) - } - order = nil - if _, unannounced, err := keepGiven(false, announce(true), keep); err != nil || unannounced == nil || - strings.Join(order, ",") != "keep,announce" { - t.Errorf("not trusted: %v %v, order %v; want kept, then the failed announcement said", unannounced, err, order) - } - order = nil - failing := func() (bool, error) { order = append(order, "keep"); return false, errors.New("store away") } - if _, _, err := keepGiven(false, announce(false), failing); err == nil || strings.Join(order, ",") != "keep" { - t.Errorf("not trusted, keep failed: %v, order %v; want refused and nothing announced", err, order) - } -} // A machine the mesh does not know, as the secret's or as the desk, is refused before anybody is asked to type. func TestAGiveNamingAMachineTheMeshDoesNotKnowAsksNobody(t *testing.T) { @@ -539,23 +494,6 @@ func TestASecretAskIsBoundedAndItsEndRecorded(t *testing.T) { // A secret ask cannot be turned into a secret read: the line carries no value, the answer carries none, and every // other `secret` line is the terminal's. -func TestASecretAskIsNeverASecretRead(t *testing.T) { - t.Setenv(verbVar, "mesh-controller.secret-ask") - for _, args := range [][]string{ - {"ask", "anchor", "telegram", "telegram-token", "the-value"}, - {"ask", "anchor", "telegram"}, - {"ask", "anchor", "telegram", "telegram-token", "--from", "/dev/null"}, - } { - if err := secretCommand(context.Background(), args); err == nil { - t.Errorf("secret %v was taken", args) - } - } - for _, args := range [][]string{{"recover", "anchor", "telegram", "telegram-token"}, {"export"}} { - if err := terminalOnly(append([]string{"secret"}, args...)); err == nil { - t.Errorf("secret %v passed the terminal rule", args) - } - } -} // askOperatorChannelFake replaces askOperatorChannel for the test and records every call it was given. func askOperatorChannelFake(t *testing.T, answer func(args map[string]any) (json.RawMessage, error)) *[]map[string]any { @@ -664,23 +602,6 @@ func TestASuccessfulTellRetiresEveryOpenSecretGivenCondition(t *testing.T) { // novox/hq ADR 0380 clause 7: a trusted party's secret given at the controller's terminal is announced // before it is kept, and not kept when the announcement fails. It still fails when telling the operator // AND raising the fallback condition both fail — nothing silently keeps a trusted party's value unheard. -func TestATrustedPartysSecretAtTheTerminalIsNotKeptWhenTellAndTheConditionBothFail(t *testing.T) { - ctx := context.Background() - askOperatorChannelFake(t, func(map[string]any) (json.RawMessage, error) { return nil, link.ErrNothingServes }) - // conditionsFrom is left nil and there is no bus to dial here, so the fallback condition cannot be - // raised either: both of ADR 0380's ways to tell the operator fail. - kept := false - if _, _, err := keepGiven(true, - func() error { - return announceSecretGiven(ctx, "anchor", "telegram", "telegram-token", "at the controller's terminal", "me") - }, - func() (bool, error) { kept = true; return false, nil }); err == nil { - t.Error("a trusted party's secret was kept though telling the operator and raising the condition both failed") - } - if kept { - t.Error("the value was kept despite the failure") - } -} // Review round 2 of mesh-controller (feat/mail-seats, head 5f24a47d): ADR 0380 §9's fallback — raising the // secret-given condition — answers only link.ErrNothingServes and link.ErrGrantRefused. Any other failure of diff --git a/cmd/mesh-controller/doctor_integration_test.go b/cmd/mesh-controller/doctor_integration_test.go new file mode 100644 index 00000000..33a1a2e6 --- /dev/null +++ b/cmd/mesh-controller/doctor_integration_test.go @@ -0,0 +1,246 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) { + failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused", + Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"} + var broken atomic.Bool + broken.Store(true) + withProbes(t, + probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }}, + probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if broken.Load() { + return []conditions.Observation{failing}, nil + } + return nil, nil + }}, + probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if broken.Load() { + return nil, errors.New("the store is away") + } + return nil, nil + }}, + probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1, + run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) { + if broken.Load() { + <-ctx.Done() + time.Sleep(50 * time.Millisecond) + } + return nil, nil + }}, + probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"}, + ) + before := probeWithin + probeWithin = 200 * time.Millisecond + t.Cleanup(func() { probeWithin = before }) + + store := conditions.NewInMemory() + told := &conditions.Told{} + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, teller: told, host: "anchor"} + // A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot + // run it either (novox/hq issue 277): one timed-out question is not a probe gone blind. + first := d.runOnce(t.Context(), "a test") + if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { + t.Fatalf("counted %+v", first.Counts) + } + if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" { + t.Fatalf("after one run, open: %+v", open) + } + run := d.runOnce(t.Context(), "a test") + if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { + t.Fatalf("counted %+v", run.Counts) + } + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, c := range open { + keys = append(keys, c.Key+"="+c.Kind) + } + for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed", + "probe.P4.failed=probe-failed"} { + if !slices.Contains(keys, want) { + t.Errorf("%s is not open: %v", want, keys) + } + } + // The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel). + if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent { + t.Fatalf("said %v", told.Names) + } + if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run { + t.Fatal("the run is not the last verdict") + } + body, _ := json.Marshal(run) + var shape map[string]any + _ = json.Unmarshal(body, &shape) + for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} { + if _, ok := shape[field]; !ok { + t.Errorf("the heartbeat carries no %q: %s", field, body) + } + } + + // Mended: the next run clears every one of them. + broken.Store(false) + d.runOnce(t.Context(), "a test") + if open, _ := k.Open(t.Context()); len(open) != 0 { + t.Fatalf("a passing run left open %+v", open) + } +} + +func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) { + open := aMesh(t) + got, err := probeDeclarations(t.Context(), &doctor{open: open}) + if err != nil { + t.Fatal(err) + } + if len(got) != 0 { + t.Fatalf("a healthy mesh failed D1: %+v", got) + } +} + +func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + one, two := rivals() + register(t, open, one) + register(t, open, two) + for _, m := range []string{"rival-one", "rival-two"} { + if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" { + t.Fatal(err) + } + } + got, err := probeDeclarations(ctx, &doctor{open: open}) + linted(got) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") { + t.Fatalf("%+v", got) + } +} + +func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) { + url := testbus.URL(t) + open := aMesh(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { + _ = js.Context().DeleteStream(s) + } + if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + d := &doctor{open: open, js: js} + for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} { + got, err := p(t.Context(), d) + if err != nil || len(got) != 0 { + t.Fatalf("a bus just raised fails: %+v %v", got, err) + } + } + if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { + t.Fatal(err) + } + info, err := js.Context().StreamInfo("EVENTS") + if err != nil { + t.Fatal(err) + } + cfg := info.Config + cfg.MaxMsgsPerSubject = 3 + if _, err := js.Context().UpdateStream(&cfg); err != nil { + t.Fatal(err) + } + consumers, err := probeConsumers(t.Context(), d) + if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" { + t.Fatalf("the deleted consumer: %+v %v", consumers, err) + } + streams, err := probeStreams(t.Context(), d) + if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") { + t.Fatalf("the redefined stream: %+v %v", streams, err) + } +} + +func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + heard := make(chan *nats.Msg, 16) + for _, subject := range broker.BusAdvisories { + if _, err := conn.ChanSubscribe(subject, heard); err != nil { + t.Fatal(err) + } + } + api, _ := jetstream.New(conn) + _ = api.DeleteStream(t.Context(), "SEAT_ADVISED") + stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}}) + if err != nil { + t.Fatal(err) + } + defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }() + consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker", + AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond}) + if err != nil { + t.Fatal(err) + } + if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil { + t.Fatal(err) + } + if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil { + t.Fatal(err) + } + // A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed). + // Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch. + time.Sleep(300 * time.Millisecond) + _, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond)) + if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil { + t.Fatal(err) + } + kinds := map[string]string{} + deadline := time.After(5 * time.Second) + for len(kinds) < 2 { + select { + case m := <-heard: + if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok { + kinds[a.Kind] = a.Said + } + case <-deadline: + t.Fatalf("the bus said only %v", kinds) + } + } + if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") { + t.Fatalf("%v", kinds) + } +} diff --git a/cmd/mesh-controller/doctor_test.go b/cmd/mesh-controller/doctor_test.go index f62990d3..8f5b0506 100644 --- a/cmd/mesh-controller/doctor_test.go +++ b/cmd/mesh-controller/doctor_test.go @@ -2,25 +2,18 @@ package main import ( "context" - "encoding/json" - "errors" "net" "slices" "strings" - "sync/atomic" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" "golang.org/x/net/dns/dnsmessage" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // The self-check (novox/hq to-be 45 §4): a probe that fails raises its condition, one that cannot run @@ -34,98 +27,6 @@ func withProbes(t *testing.T, probes ...probe) { t.Cleanup(func() { probeRegistry = before }) } -func TestARunKeepsWhatEachProbeFoundAndSaysItsHeartbeat(t *testing.T) { - failing := conditions.Observation{Scope: conditions.ScopeMachine, ID: "anchor", Token: "refused", - Severity: conditions.Urgent, Summary: "anchor's node-engine would refuse its declaration"} - var broken atomic.Bool - broken.Store(true) - withProbes(t, - probe{ID: "P1", Asserts: "passes", Kind: "never", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { return nil, nil }}, - probe{ID: "P2", Asserts: "finds a fault", Kind: "declaration-refused", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if broken.Load() { - return []conditions.Observation{failing}, nil - } - return nil, nil - }}, - probe{ID: "P3", Asserts: "cannot run", Kind: "x", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if broken.Load() { - return nil, errors.New("the store is away") - } - return nil, nil - }}, - probe{ID: "P4", Asserts: "hangs", Kind: "x", Phase: 1, - run: func(ctx context.Context, _ *doctor) ([]conditions.Observation, error) { - if broken.Load() { - <-ctx.Done() - time.Sleep(50 * time.Millisecond) - } - return nil, nil - }}, - probe{ID: "P5", Asserts: "later", Kind: "x", Phase: 2, Deferred: "not yet"}, - ) - before := probeWithin - probeWithin = 200 * time.Millisecond - t.Cleanup(func() { probeWithin = before }) - - store := conditions.NewInMemory() - told := &conditions.Told{} - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - d := &doctor{keeper: k, teller: told, host: "anchor"} - // A probe that cannot run is said in the verdict at once, and as a condition when the next run cannot - // run it either (novox/hq issue 277): one timed-out question is not a probe gone blind. - first := d.runOnce(t.Context(), "a test") - if first.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { - t.Fatalf("counted %+v", first.Counts) - } - if open, _ := k.Open(t.Context()); len(open) != 1 || open[0].Key != "machine.anchor.refused" { - t.Fatalf("after one run, open: %+v", open) - } - run := d.runOnce(t.Context(), "a test") - if run.Counts != (doctorCounts{Passed: 1, Failed: 1, FailedToRun: 2, Deferred: 1}) { - t.Fatalf("counted %+v", run.Counts) - } - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var keys []string - for _, c := range open { - keys = append(keys, c.Key+"="+c.Kind) - } - for _, want := range []string{"machine.anchor.refused=declaration-refused", "probe.P3.failed=probe-failed", - "probe.P4.failed=probe-failed"} { - if !slices.Contains(keys, want) { - t.Errorf("%s is not open: %v", want, keys) - } - } - // The heartbeat, in the shape mesh-watcher reads (the contract with the operator's channel). - if len(told.Names) != 2 || told.Names[1] != conditions.HeartbeatEvent { - t.Fatalf("said %v", told.Names) - } - if d.lastRunEnded().IsZero() || d.lastRun().Run != run.Run { - t.Fatal("the run is not the last verdict") - } - body, _ := json.Marshal(run) - var shape map[string]any - _ = json.Unmarshal(body, &shape) - for _, field := range []string{"run", "at", "interval-seconds", "counts", "probes", "controller"} { - if _, ok := shape[field]; !ok { - t.Errorf("the heartbeat carries no %q: %s", field, body) - } - } - - // Mended: the next run clears every one of them. - broken.Store(false) - d.runOnce(t.Context(), "a test") - if open, _ := k.Open(t.Context()); len(open) != 0 { - t.Fatalf("a passing run left open %+v", open) - } -} - // **The registry says what each probe asserts**, and a probe not built says why and when. func TestTheRegistryIsTheDesignsLiveForm(t *testing.T) { seen := map[string]bool{} @@ -164,38 +65,8 @@ func TestWatchdogsThatStoppedAreSaid(t *testing.T) { } // **D1 composes every machine of a healthy mesh and the host's own validator takes each.** -func TestEveryMachineOfAHealthyMeshComposesAndValidates(t *testing.T) { - open := aMesh(t) - got, err := probeDeclarations(t.Context(), &doctor{open: open}) - if err != nil { - t.Fatal(err) - } - if len(got) != 0 { - t.Fatalf("a healthy mesh failed D1: %+v", got) - } -} // **D1 names a machine nothing can be sent to**, and the network that cannot be computed for it. -func TestAMachineWhoseDeclarationDoesNotComposeIsSaid(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - one, two := rivals() - register(t, open, one) - register(t, open, two) - for _, m := range []string{"rival-one", "rival-two"} { - if _, err := assign(ctx, open, "laptop", m); err != nil && m == "rival-one" { - t.Fatal(err) - } - } - got, err := probeDeclarations(ctx, &doctor{open: open}) - linted(got) - if err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0].Key() != "machine.laptop.uncomposable" || !strings.Contains(got[0].Summary, "the-seat") { - t.Fatalf("%+v", got) - } -} // **D2: a resolver answering NXDOMAIN for IPv6 is wrong** — musl takes it as no such name (issue 262). func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) { @@ -250,107 +121,8 @@ func TestAResolverAnsweringNoSuchNameForIPv6IsWrong(t *testing.T) { // **D6, D7: what the controller defines is what it finds**, and a consumer deleted or a stream // redefined is said — against a real bus, raised by the same derivation the controller starts with. -func TestNatsTheBusIsWhatTheControllerDefines(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - if _, err := assertBusObjects(t.Context(), open.inventory, js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - d := &doctor{open: open, js: js} - for _, p := range []func(context.Context, *doctor) ([]conditions.Observation, error){probeConsumers, probeStreams} { - got, err := p(t.Context(), d) - if err != nil || len(got) != 0 { - t.Fatalf("a bus just raised fails: %+v %v", got, err) - } - } - if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { - t.Fatal(err) - } - info, err := js.Context().StreamInfo("EVENTS") - if err != nil { - t.Fatal(err) - } - cfg := info.Config - cfg.MaxMsgsPerSubject = 3 - if _, err := js.Context().UpdateStream(&cfg); err != nil { - t.Fatal(err) - } - consumers, err := probeConsumers(t.Context(), d) - if err != nil || len(consumers) != 1 || consumers[0].Key() != "bus.NODES.laptop.missing" { - t.Fatalf("the deleted consumer: %+v %v", consumers, err) - } - streams, err := probeStreams(t.Context(), d) - if err != nil || len(streams) != 1 || !strings.Contains(streams[0].Summary, "per subject") { - t.Fatalf("the redefined stream: %+v %v", streams, err) - } -} // **S9 hears the bus**: a consumer that gives up on a message, and one deleted, as the server says. -func TestNatsTheBusSaysAConsumerGaveUpAndOneWasDeleted(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - heard := make(chan *nats.Msg, 16) - for _, subject := range broker.BusAdvisories { - if _, err := conn.ChanSubscribe(subject, heard); err != nil { - t.Fatal(err) - } - } - api, _ := jetstream.New(conn) - _ = api.DeleteStream(t.Context(), "SEAT_ADVISED") - stream, err := api.CreateStream(t.Context(), jetstream.StreamConfig{Name: "SEAT_ADVISED", Subjects: []string{"advised.>"}}) - if err != nil { - t.Fatal(err) - } - defer func() { _ = api.DeleteStream(context.Background(), "SEAT_ADVISED") }() - consumer, err := stream.CreateConsumer(t.Context(), jetstream.ConsumerConfig{Durable: "SEAT_ADVISED_worker", - AckPolicy: jetstream.AckExplicitPolicy, MaxDeliver: 1, AckWait: 100 * time.Millisecond}) - if err != nil { - t.Fatal(err) - } - if _, err := api.Publish(t.Context(), "advised.x", []byte("x")); err != nil { - t.Fatal(err) - } - if _, err := consumer.Fetch(1, jetstream.FetchMaxWait(time.Second)); err != nil { - t.Fatal(err) - } - // A seat's worker, so the deletion is of a consumer the mesh names (link.MeshNamed). - // Not acknowledged: after its one delivery the consumer gives up on it — on the next fetch. - time.Sleep(300 * time.Millisecond) - _, _ = consumer.Fetch(1, jetstream.FetchMaxWait(300*time.Millisecond)) - if err := stream.DeleteConsumer(t.Context(), "SEAT_ADVISED_worker"); err != nil { - t.Fatal(err) - } - kinds := map[string]string{} - deadline := time.After(5 * time.Second) - for len(kinds) < 2 { - select { - case m := <-heard: - if a, ok := link.ReadAdvisory(m.Subject, m.Data); ok { - kinds[a.Kind] = a.Said - } - case <-deadline: - t.Fatalf("the bus said only %v", kinds) - } - } - if !strings.Contains(kinds["max-deliveries"], "gave up") || !strings.Contains(kinds["consumer-lost"], "was deleted") { - t.Fatalf("%v", kinds) - } -} // **D10 compares a node-engine with what it is delivered as, not with its commit** (2026-10-06: every // machine read as behind right after a push sent it the current build — it says the digest-named diff --git a/cmd/mesh-controller/dry_run_integration_test.go b/cmd/mesh-controller/dry_run_integration_test.go new file mode 100644 index 00000000..a8cab99e --- /dev/null +++ b/cmd/mesh-controller/dry_run_integration_test.go @@ -0,0 +1,20 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) { + err := builds{}.Built(t.Context(), link.BuildResult{ + ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true, + Manifest: json.RawMessage(`{"module":"app","version":"1"}`), + }) + if err != nil { + t.Fatalf("a dry run's outcome was not simply set aside: %v", err) + } +} diff --git a/cmd/mesh-controller/dry_run_test.go b/cmd/mesh-controller/dry_run_test.go index 0f087346..53767a0a 100644 --- a/cmd/mesh-controller/dry_run_test.go +++ b/cmd/mesh-controller/dry_run_test.go @@ -9,15 +9,6 @@ import ( // A dry run's outcome is looked at, never taken in (novox/hq issue 240). The daemon here holds no // store at all, so anything that tried to record or register would fail rather than pass quietly. -func TestADryRunsOutcomeIsTakenInByNothing(t *testing.T) { - err := builds{}.Built(t.Context(), link.BuildResult{ - ID: "build-1", Repository: "ssh://forge/app.git", Ref: "unreviewed", Module: "app", DryRun: true, - Manifest: json.RawMessage(`{"module":"app","version":"1"}`), - }) - if err != nil { - t.Fatalf("a dry run's outcome was not simply set aside: %v", err) - } -} // The mark survives the wire both ways: asked as a dry run, answered as one. func TestTheDryRunMarkTravelsWithTheBuild(t *testing.T) { diff --git a/cmd/mesh-controller/epoch_send_test.go b/cmd/mesh-controller/epoch_send_test.go index ae6adbdc..9e138c3c 100644 --- a/cmd/mesh-controller/epoch_send_test.go +++ b/cmd/mesh-controller/epoch_send_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/facts_test.go b/cmd/mesh-controller/facts_test.go index d5b3ac63..7f3c9544 100644 --- a/cmd/mesh-controller/facts_test.go +++ b/cmd/mesh-controller/facts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/failed_wait_provider_test.go b/cmd/mesh-controller/failed_wait_provider_test.go index 64a7a960..2790e340 100644 --- a/cmd/mesh-controller/failed_wait_provider_test.go +++ b/cmd/mesh-controller/failed_wait_provider_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/found_wait_integration_test.go b/cmd/mesh-controller/found_wait_integration_test.go new file mode 100644 index 00000000..1a1f193c --- /dev/null +++ b/cmd/mesh-controller/found_wait_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + for _, c := range []struct { + name string + found time.Duration // when the directory was found, against now + passes bool + }{ + {"found a day before the send", -24 * time.Hour, true}, + {"found by this send", time.Hour, false}, + } { + t.Run(c.name, func(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true}, nil + } + backlogFacts := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := backlogFacts(ctx, open, component) + // The used-as-found condition, raised after the send (its second statement): its own kind, never a + // fault the gate reads as the build's. + f.judged, f.openErr = true, nil + f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound, + Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, + Raised: time.Now()}) + f.health = map[string]inventory.NodeHealth{} + for _, n := range []string{"anchor", "laptop"} { + f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ + {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, + foundDirectory("app", time.Now().Add(c.found)), + {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} + } + return f, err + } + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + advancePlans(ctx, b.open) + if p := b.release(t); p.State != inventory.PlanRolling { + break + } + time.Sleep(20 * time.Millisecond) + } + p := b.release(t) + v, found, err := inv.GateOf(ctx, "build-app-c2") + if c.passes { + if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) + } + if !strings.Contains(v.Why+p.Note, "hand it over") { + t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) + } + return + } + if p.State == inventory.PlanDone { + t.Fatalf("a directory this send found let the walk through: %s", p.Note) + } + }) + } +} + +func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + k, _ := withConditionsInMemory(t) + ctx := t.Context() + rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} + for i := 0; i < 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil { + t.Fatal(err) + } + } + open, _ := k.Open(ctx) + var got *conditions.Condition + for i, c := range open { + if c.Key == usedAsFoundKey("notes", "laptop") { + got = &open[i] + } + if c.Kind == kindModuleUnhealthy { + t.Fatalf("raised as a fault: %+v", c) + } + } + if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent || + !strings.Contains(got.Summary, "notes.data") { + t.Fatalf("the condition: %+v", got) + } + // Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait. + if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil { + t.Fatal(err) + } + open, _ = k.Open(ctx) + for _, c := range open { + if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent { + t.Fatal("a directory used as found became urgent") + } + } + if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + open, _ = k.Open(ctx) + for _, c := range open { + if c.Key == usedAsFoundKey("notes", "laptop") { + t.Fatal("not cleared once handed over") + } + } +} diff --git a/cmd/mesh-controller/found_wait_test.go b/cmd/mesh-controller/found_wait_test.go index caa75ac7..54621738 100644 --- a/cmd/mesh-controller/found_wait_test.go +++ b/cmd/mesh-controller/found_wait_test.go @@ -1,12 +1,10 @@ package main import ( - "context" "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -48,116 +46,5 @@ func TestADirectoryFoundBeforeTheSendIsAWaitForAPerson(t *testing.T) { // The whole walk: a module whose directory was used as found long before still gets its fix to every machine, // its pass kept and the wait said; a directory this very send found holds it and puts it back. -func TestAFixGoesThroughPastADirectoryFoundBefore(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - for _, c := range []struct { - name string - found time.Duration // when the directory was found, against now - passes bool - }{ - {"found a day before the send", -24 * time.Hour, true}, - {"found by this send", time.Hour, false}, - } { - t.Run(c.name, func(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - // The used-as-found condition, raised after the send (its second statement): its own kind, never a - // fault the gate reads as the build's. - f.judged, f.openErr = true, nil - f.open = append(f.open, conditions.Condition{Key: usedAsFoundKey("app", "anchor"), Kind: kindUsedAsFound, - Subject: conditions.Subject{Scope: conditions.ScopeModule, ID: "app.anchor", Machine: "anchor"}, - Raised: time.Now()}) - f.health = map[string]inventory.NodeHealth{} - for _, n := range []string{"anchor", "laptop"} { - f.health[n] = inventory.NodeHealth{Node: n, HeardAt: time.Now(), Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - foundDirectory("app", time.Now().Add(c.found)), - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateHealthy}}} - } - return f, err - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - gateWithin, gateEvery, gateBound = time.Hour, 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - v, found, err := inv.GateOf(ctx, "build-app-c2") - if c.passes { - if p.State != inventory.PlanDone || err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the walk is %s (%s); app's verdict %+v: want the fix through", p.State, p.Note, v) - } - if !strings.Contains(v.Why+p.Note, "hand it over") { - t.Errorf("the wait is not carried: verdict %q, walk %q", v.Why, p.Note) - } - return - } - if p.State == inventory.PlanDone { - t.Fatalf("a directory this send found let the walk through: %s", p.Note) - } - }) - } -} // The condition says the wait in its own kind: the operator's, never urgent, and cleared once handed over. -func TestADirectoryUsedAsFoundIsItsOwnCondition(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"notes": {foundDirectory("notes", time.Now().Add(-time.Hour))}} - for i := 0; i < 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": i + 1}, time.Now()); err != nil { - t.Fatal(err) - } - } - open, _ := k.Open(ctx) - var got *conditions.Condition - for i, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - got = &open[i] - } - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault: %+v", c) - } - } - if got == nil || got.Resolver != conditions.ResolverOperator || got.Severity == conditions.Urgent || - !strings.Contains(got.Summary, "notes.data") { - t.Fatalf("the condition: %+v", got) - } - // Long open is still not urgent: only a person can hand it over, and nothing is broken by the wait. - if err := judgeModuleHealth(ctx, nil, k, "laptop", rs, map[string]int{"notes": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") && c.Severity == conditions.Urgent { - t.Fatal("a directory used as found became urgent") - } - } - if err := judgeModuleHealth(ctx, nil, k, "laptop", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - open, _ = k.Open(ctx) - for _, c := range open { - if c.Key == usedAsFoundKey("notes", "laptop") { - t.Fatal("not cleared once handed over") - } - } -} diff --git a/cmd/mesh-controller/gate_followup_test.go b/cmd/mesh-controller/gate_followup_test.go index a78963dd..9ba07bf3 100644 --- a/cmd/mesh-controller/gate_followup_test.go +++ b/cmd/mesh-controller/gate_followup_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/gate_integration_test.go b/cmd/mesh-controller/gate_integration_test.go new file mode 100644 index 00000000..fff52fa6 --- /dev/null +++ b/cmd/mesh-controller/gate_integration_test.go @@ -0,0 +1,482 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +func aGateMesh(t *testing.T) *gateMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + g := &gateMesh{open: open, health: map[string]health{}} + // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. + wasGated := gateFirstMachine + gateFirstMachine = true + t.Cleanup(func() { gateFirstMachine = wasGated }) + g.keeper, _ = withConditionsInMemory(t) + g.told = &conditions.Told{} + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} + t.Cleanup(func() { doctorFrom = was }) + + for _, b := range []inventory.Build{ + {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, + {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, + } { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) + b.Manifest = manifest + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + register := func(commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + register("c1", time.Now().Add(-2*time.Hour)) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + register("c2", time.Now().Add(-time.Minute)) + + // Every send: recorded with the build the module is at, applied and reported by the machine. + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + g.sent = append(g.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + // What a judging reads: the store's reports, and a health the test says per machine. + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if g.health[r.Node] == healthBroken { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + for node, h := range g.health { + if h == healthNotYet { + // Not reported on the send: neither the send made last, nor the gate's own (issue 352). + f.rolledBack[node] = nil + r := f.reports[node] + r.Current, r.Declared, r.ReportedSequence = false, "", 0 + f.reports[node] = r + } + } + return f, nil + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + return g +} + +func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + + advancePlans(ctx, g.open) // the first machine is sent the new build + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v, not the first machine alone", g.sent) + } + if p := g.plan(t); p.Modules["app"].Previous != "c1" { + t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) + } + + g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging + gateEvery = 0 + advancePlans(ctx, g.open) + + p := g.plan(t) + gate := p.Modules["app"].Gate + if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || + gate.Rollback != inventory.RolledBack { + t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) + } + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { + t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) + } + if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { + t.Fatalf("anchor was last sent %v, not the previous build", sent) + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { + t.Fatalf("laptop was sent the failed build: %v", sent) + } + if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { + t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) + } + + // Said: a condition for the operator, and the event. + open, err := g.keeper.Open(ctx) + if err != nil { + t.Fatal(err) + } + var key string + for _, c := range open { + if c.Kind == kindRolledBack { + key = c.Key + } + } + if key != "build.app.anchor.rolled-back" { + t.Fatalf("no rolled-back condition for app on anchor: %+v", open) + } + saidIt := false + for _, e := range g.told.Said() { + saidIt = saidIt || e.Event == link.KeyRolledBack + } + if !saidIt { + t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) + } + + // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build + // heard again is not registered. + advancePlans(ctx, g.open) + state := p.Modules["app"] + gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") + if len(g.sent) != 2 { + t.Fatalf("sent again after the rollback: %v", g.sent) + } + _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", + Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}), + Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}) + if err == nil || !strings.Contains(err.Error(), "failed its gate") { + t.Fatalf("the failed build was registered again: %v", err) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module moved to %s", current["app"].Commit) + } + if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { + t.Fatalf("a plan stopped at a failed gate was retried: %v", err) + } + + // The probe keeps the condition while the newest verdict is the failure. + obs, err := probeGates(ctx, doctorFrom) + if err != nil || len(obs) != 1 || obs[0].Key() != key { + t.Fatalf("the gate's probe found %+v %v", obs, err) + } +} + +func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", g.sent) + } + gate := p.Modules["app"].Gate + if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { + t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) + } + if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a passing build left a condition: %+v %v", obs, err) + } +} + +func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { + g := aGateMesh(t) + gateFirstMachine = false + ctx := t.Context() + judged := false + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + judged = true + return wasGather(ctx, open, component) + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + for i := 0; i < 6; i++ { + advancePlans(ctx, g.open) + } + p := g.plan(t) + if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { + t.Fatalf("sent %v, not both machines in one send", g.sent) + } + if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { + t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) + } + if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { + t.Fatalf("a rolled build left a condition: %+v %v", obs, err) + } +} + +func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { + g := aGateMesh(t) + ctx := t.Context() + gateWithin = time.Hour // a core module could not pass on fewer readings for an hour + advancePlans(ctx, g.open) // the first machine is sent the new build + advancePlans(ctx, g.open) // one healthy judging + p := g.plan(t) + gate := p.Modules["app"].Gate + if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { + t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) + } + if !strings.Contains(gate.Why, "healthy 1 times") { + t.Errorf("the verdict says %q", gate.Why) + } +} + +func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + advancePlans(ctx, g.open) + g.health["anchor"] = healthNotYet + gateEvery = 0 + advancePlans(ctx, g.open) + if p := g.plan(t); !p.Open() || len(g.sent) != 1 { + t.Fatalf("judged before the bound: %s %v", p.State, g.sent) + } + gateBound = -time.Second + advancePlans(ctx, g.open) + p := g.plan(t) + if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || + !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { + t.Fatalf("the plan is %s, its gate %+v", p.State, gate) + } +} + +func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, + Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} + if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", + Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { + t.Fatal(err) + } + u, err := inv.UpgradeOf(ctx, "nats") + if err != nil || u.RollOut || u.From != catalogue.FromBus { + t.Fatalf("the bus's policy is %+v %v", u, err) + } + if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { + t.Fatalf("a person rolled the bus out: %v", err) + } + var sent [][]string + was := sendRollout + sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { + sent = append(sent, names) + return names, nil + } + t.Cleanup(func() { sendRollout = was }) + now := time.Now().UTC() + plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, + State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, + Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, open) + if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { + t.Fatalf("a plan sent the bus: %s %v", p.State, sent) + } + held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) + if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { + t.Fatalf("a push may send the bus's machine: %v %v", held, err) + } + // Nor may any other send — a plan's for another module on that machine carried the bus with it. + if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { + t.Fatalf("a send to the bus's machine was not refused: %v", err) + } + // A rebuild that made the same artifacts is no move. + for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { + b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} + b.Asked, b.At = time.Now(), time.Now() + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { + t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), + At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", + Reference: "registry/nats@sha256:new"}}}); err != nil { + t.Fatal(err) + } + // The planned step refuses to start without its word on reversibility, and without a snapshot taken + // first by the bus machine's backup holder. + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { + t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) + } + wasSnapshot := takeBusSnapshot + t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) + takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || + !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { + t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) + } + takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { + return node + "'s restore point of " + module, nil + } + if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { + t.Fatalf("the step sent %v, not the bus's machine", sent) + } + s, found, err := inv.LatestBusStep(ctx) + if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || + !reflect.DeepEqual(s.Machines, []string{"anchor"}) { + t.Fatalf("the step is %+v %v %v", s, found, err) + } +} + +func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + for _, name := range []string{"gone", "kept"} { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", + Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, + Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + if len(*asked) != 0 { + t.Fatalf("a deleted module was asked to build: %v", *asked) + } + if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { + t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) + } + shelf, err := inv.Catalogue(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := shelf["gone"]; still { + t.Fatal("a deleted module nothing holds was not forgotten") + } + + // Without the announcer saying what went: the build finds no manifest, and the plan goes on. + asked0 := time.Now().UTC().Add(-time.Minute) + plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, + State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, + Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+ + "there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k") + p, _ := inv.PlanByID(ctx, "plan-deleted") + if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { + t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) + } +} + +func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { + open := aMesh(t) + d := &doctor{open: open} + at := time.Now().UTC() + witnessed.heard("control", []lease.Rollback{ + {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, + Why: "the new controller did not take the lease within 60s", At: at}, + {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, + }, at) + t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) + obs, err := probeGates(t.Context(), d) + if err != nil { + t.Fatal(err) + } + got := map[string]conditions.Severity{} + for _, o := range obs { + got[o.Key()] = o.Severity + } + want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, + "core.node-tools.control.unwitnessed": conditions.Warning} + if !reflect.DeepEqual(got, want) { + t.Fatalf("the witness's verdicts are %v", got) + } + witnessed.heard("control", nil, time.Now()) + if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { + t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) + } +} diff --git a/cmd/mesh-controller/gate_test.go b/cmd/mesh-controller/gate_test.go index 651c90ec..589c4aa4 100644 --- a/cmd/mesh-controller/gate_test.go +++ b/cmd/mesh-controller/gate_test.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "errors" - "fmt" - "reflect" "strings" "testing" "time" @@ -30,124 +28,6 @@ type gateMesh struct { told *conditions.Told } -func aGateMesh(t *testing.T) *gateMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - g := &gateMesh{open: open, health: map[string]health{}} - // The gate's own tests judge a first machine, as walks a controller before novox/hq ADR 0360 started still are. - wasGated := gateFirstMachine - gateFirstMachine = true - t.Cleanup(func() { gateFirstMachine = wasGated }) - g.keeper, _ = withConditionsInMemory(t) - g.told = &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: g.keeper, teller: g.told} - t.Cleanup(func() { doctorFrom = was }) - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - register("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - register("c2", time.Now().Add(-time.Minute)) - - // Every send: recorded with the build the module is at, applied and reported by the machine. - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - g.sent = append(g.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - // What a judging reads: the store's reports, and a health the test says per machine. - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if g.health[r.Node] == healthBroken { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - for node, h := range g.health { - if h == healthNotYet { - // Not reported on the send: neither the send made last, nor the gate's own (issue 352). - f.rolledBack[node] = nil - r := f.reports[node] - r.Current, r.Declared, r.ReportedSequence = false, "", 0 - f.reports[node] = r - } - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-gate", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return g -} - func (g *gateMesh) plan(t *testing.T) inventory.Plan { t.Helper() p, err := g.open.inventory.PlanByID(t.Context(), "plan-gate") @@ -160,187 +40,17 @@ func (g *gateMesh) plan(t *testing.T) inventory.Plan { // A module's build that fails its gate on the first machine is put back there — the previous build // registered and sent to it again — and never reaches the second machine; it is marked, said as a // condition and an event, never registered or rolled back again. -func TestABuildThatFailsItsGateIsRolledBackOnItsFirstMachineAndGoesNoFurther(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - - advancePlans(ctx, g.open) // the first machine is sent the new build - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v, not the first machine alone", g.sent) - } - if p := g.plan(t); p.Modules["app"].Previous != "c1" { - t.Fatalf("the build the first machine ran before was not kept: %+v", p.Modules["app"]) - } - - g.health["anchor"] = healthBroken // the new build breaks its first machine, after one good judging - gateEvery = 0 - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate == nil || gate.Verdict != inventory.GateFailed || - gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the rollback goes to the first machine, and nothing reaches laptop", g.sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } - if sent, _, _ := inv.SentBuilds(ctx, "anchor"); sent["app"] != "c1" { - t.Fatalf("anchor was last sent %v, not the previous build", sent) - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["app"] != "c1" { - t.Fatalf("laptop was sent the failed build: %v", sent) - } - if failed, err := inv.GateFailed(ctx, "build-2"); err != nil || !failed { - t.Fatalf("the build is not marked failed at its gate: %v %v", failed, err) - } - - // Said: a condition for the operator, and the event. - open, err := g.keeper.Open(ctx) - if err != nil { - t.Fatal(err) - } - var key string - for _, c := range open { - if c.Kind == kindRolledBack { - key = c.Key - } - } - if key != "build.app.anchor.rolled-back" { - t.Fatalf("no rolled-back condition for app on anchor: %+v", open) - } - saidIt := false - for _, e := range g.told.Said() { - saidIt = saidIt || e.Event == link.KeyRolledBack - } - if !saidIt { - t.Fatalf("the rollback was not said as an event: %+v", g.told.Said()) - } - - // Never again: more passes send nothing, a second judging rolls nothing back, and the failed build - // heard again is not registered. - advancePlans(ctx, g.open) - state := p.Modules["app"] - gateFailed(ctx, g.open, &p, "app", state, []string{"anchor"}, "again") - if len(g.sent) != 2 { - t.Fatalf("sent again after the rollback: %v", g.sent) - } - _, _, err = takeIn(ctx, inv, link.BuildResult{ID: "build-2", Repository: "novox/mesh-catalog", Path: "modules/app", - Commit: "c2", Manifest: mustJSON(t, catalogue.Manifest{Module: "app", Version: "c2"}), - Source: &link.SourceOnSeat{Seat: "git", Repository: "novox/mesh-catalog"}}) - if err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("the failed build was registered again: %v", err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module moved to %s", current["app"].Commit) - } - if _, err := retryPlan(ctx, g.open, "plan-gate"); err == nil || !strings.Contains(err.Error(), "failed its gate") { - t.Fatalf("a plan stopped at a failed gate was retried: %v", err) - } - - // The probe keeps the condition while the newest verdict is the failure. - obs, err := probeGates(ctx, doctorFrom) - if err != nil || len(obs) != 1 || obs[0].Key() != key { - t.Fatalf("the gate's probe found %+v %v", obs, err) - } -} // A passing build rolls everywhere with no hand: judged healthy on its first machine three times, then // the rest are sent, the verdict kept, and the plan done. -func TestABuildThatPassesItsGateRollsEverywhereUnattended(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", g.sent) - } - gate := p.Modules["app"].Gate - if p.State != inventory.PlanDone || gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes < gateWanted(gate) { - t.Fatalf("the plan is %s (%s), its gate %+v", p.State, p.Note, gate) - } - if v, found, err := g.open.inventory.GateOf(ctx, "build-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("the verdict was not kept: %+v %v %v", v, found, err) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a passing build left a condition: %+v %v", obs, err) - } -} // A build rolls everywhere with no hand and no gate (novox/hq ADR 0360 §1): both machines in one send, no health // judged, and the walk done once both reported on the send. Before, anchor was judged first, then laptop sent. -func TestABuildRollsToEveryMachineAtOnceWithNoGate(t *testing.T) { - g := aGateMesh(t) - gateFirstMachine = false - ctx := t.Context() - judged := false - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - judged = true - return wasGather(ctx, open, component) - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - for i := 0; i < 6; i++ { - advancePlans(ctx, g.open) - } - p := g.plan(t) - if !reflect.DeepEqual(g.sent, [][]string{{"anchor", "laptop"}}) { - t.Fatalf("sent %v, not both machines in one send", g.sent) - } - if s := p.Modules["app"]; p.State != inventory.PlanDone || s.Gate != nil || s.FirstAt != nil || judged { - t.Fatalf("the plan is %s (%s), app %+v, judged %v", p.State, p.Note, s, judged) - } - if obs, err := probeGates(ctx, doctorFrom); err != nil || len(obs) != 0 { - t.Fatalf("a rolled build left a condition: %+v %v", obs, err) - } -} // A leaf module passes its gate on one reading, as soon as its first machine is healthy, without waiting out the // core's settle time (novox/hq ADR 0282 decision 4 as ADR 0304 decision 3 changed it, issue 524). -func TestALeafModulePassesOnOneHealthyReading(t *testing.T) { - g := aGateMesh(t) - ctx := t.Context() - gateWithin = time.Hour // a core module could not pass on fewer readings for an hour - advancePlans(ctx, g.open) // the first machine is sent the new build - advancePlans(ctx, g.open) // one healthy judging - p := g.plan(t) - gate := p.Modules["app"].Gate - if gate == nil || gate.Verdict != inventory.GatePassed || gate.Passes != 1 || gate.Wants != 1 { - t.Fatalf("a leaf module's gate after one healthy judging: %+v (plan %s: %s)", gate, p.State, p.Note) - } - if !strings.Contains(gate.Why, "healthy 1 times") { - t.Errorf("the verdict says %q", gate.Why) - } -} // A first machine that never becomes healthy fails its gate at the bound, and is put back. -func TestABuildNeverHealthyFailsAtTheBound(t *testing.T) { - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - advancePlans(ctx, g.open) - g.health["anchor"] = healthNotYet - gateEvery = 0 - advancePlans(ctx, g.open) - if p := g.plan(t); !p.Open() || len(g.sent) != 1 { - t.Fatalf("judged before the bound: %s %v", p.State, g.sent) - } - gateBound = -time.Second - advancePlans(ctx, g.open) - p := g.plan(t) - if gate := p.Modules["app"].Gate; p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || - !strings.Contains(gate.Why, "not healthy within") || gate.Rollback != inventory.RolledBack { - t.Fatalf("the plan is %s, its gate %+v", p.State, gate) - } -} // The health a judging finds, per core component and for a module. func TestTheHealthDefinitions(t *testing.T) { @@ -458,147 +168,9 @@ func TestTheHealthDefinitions(t *testing.T) { // The bus is never rolled out: its policy records whatever is said, a person's roll-out is refused, // a plan builds it and sends nothing, and a push naming its machine is refused while a new build waits. -func TestTheBusIsNeverRolledOutAutomatically(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - bus := catalogue.Manifest{Module: "nats", Version: "2", Provides: []catalogue.Offer{{Name: "mesh-bus"}}, - Upgrade: &catalogue.UpgradePolicy{Policy: catalogue.PolicyRoll}} - if err := inv.RegisterModule(ctx, bus, inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", - Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "nats"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"nats": "n1"}); err != nil { - t.Fatal(err) - } - u, err := inv.UpgradeOf(ctx, "nats") - if err != nil || u.RollOut || u.From != catalogue.FromBus { - t.Fatalf("the bus's policy is %+v %v", u, err) - } - if err := inv.SetUpgradeOf(ctx, "nats", inventory.Upgrade{RollOut: true}); !errors.Is(err, inventory.ErrBusIsPlanned) { - t.Fatalf("a person rolled the bus out: %v", err) - } - var sent [][]string - was := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = was }) - now := time.Now().UTC() - plan := inventory.Plan{ID: "plan-bus", Repository: "novox/mesh-catalog", Commit: "n2", Created: now, - State: inventory.PlanBuilding, Tiers: [][]string{{"nats"}}, - Modules: map[string]*inventory.PlanModule{"nats": {State: "built", BuiltAt: &now, Commit: "n2", Build: "b"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - if p, _ := inv.PlanByID(ctx, "plan-bus"); p.State != inventory.PlanDone || len(sent) != 0 { - t.Fatalf("a plan sent the bus: %s %v", p.State, sent) - } - held, err := busHeld(ctx, inv, []string{"anchor", "laptop"}) - if err != nil || !strings.Contains(held["anchor"], "planned step") || held["laptop"] != "" { - t.Fatalf("a push may send the bus's machine: %v %v", held, err) - } - // Nor may any other send — a plan's for another module on that machine carried the bus with it. - if _, err := sendToEach(ctx, open, []string{"laptop", "anchor"}); !errors.Is(err, errBusWaits) { - t.Fatalf("a send to the bus's machine was not refused: %v", err) - } - // A rebuild that made the same artifacts is no move. - for _, b := range []inventory.Build{{ID: "nb1", Module: "nats", Commit: "n1"}, {ID: "nb2", Module: "nats", Commit: "n2"}} { - b.Made = []inventory.Artifact{{Name: "server", Kind: "image", Reference: "registry/nats@sha256:same"}} - b.Asked, b.At = time.Now(), time.Now() - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - if held, err := busHeld(ctx, inv, []string{"anchor"}); err != nil || len(held) != 0 { - t.Fatalf("a rebuild that changes nothing held the bus's machine: %v %v", held, err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "nb3", Module: "nats", Commit: "n2", Asked: time.Now().Add(time.Second), - At: time.Now().Add(time.Second), Made: []inventory.Artifact{{Name: "server", Kind: "image", - Reference: "registry/nats@sha256:new"}}}); err != nil { - t.Fatal(err) - } - // The planned step refuses to start without its word on reversibility, and without a snapshot taken - // first by the bus machine's backup holder. - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11"}); err == nil || !strings.Contains(err.Error(), "reversible") { - t.Fatalf("a bus upgrade started without saying whether it can be reverted: %v", err) - } - wasSnapshot := takeBusSnapshot - t.Cleanup(func() { takeBusSnapshot = wasSnapshot }) - takeBusSnapshot = func(context.Context, string, string) (string, error) { return "", errors.New("no holder answers") } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err == nil || - !strings.Contains(err.Error(), "snapshotted") || len(sent) != 0 { - t.Fatalf("a bus upgrade started without its snapshot: %v, sent %v", err, sent) - } - takeBusSnapshot = func(_ context.Context, module, node string) (string, error) { - return node + "'s restore point of " + module, nil - } - if err := busCommand(ctx, []string{"upgrade", "--why", "2.11", "--reversible"}); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(sent, [][]string{{"anchor"}}) { - t.Fatalf("the step sent %v, not the bus's machine", sent) - } - s, found, err := inv.LatestBusStep(ctx) - if err != nil || !found || s.Snapshot != "anchor's restore point of nats" || s.Ended != nil || - !reflect.DeepEqual(s.Machines, []string{"anchor"}) { - t.Fatalf("the step is %+v %v %v", s, found, err) - } -} // A merge that deletes a module's directory builds nothing for it: the module is forgotten where // nothing holds it, and a plan whose build finds no manifest goes on instead of failing. -func TestAMergeThatDeletesAModulePlansNothingToBuildForIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - for _, name := range []string{"gone", "kept"} { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: name, Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + name, BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1deadbeef", - Paths: []string{"modules/gone/module.json", "modules/gone/index.ts"}, - Removed: []string{"modules/gone/module.json", "modules/gone/index.ts"}} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - if len(*asked) != 0 { - t.Fatalf("a deleted module was asked to build: %v", *asked) - } - if plans, _ := inv.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a merge that only deleted a module made a plan: %+v", plans) - } - shelf, err := inv.Catalogue(ctx) - if err != nil { - t.Fatal(err) - } - if _, still := shelf["gone"]; still { - t.Fatal("a deleted module nothing holds was not forgotten") - } - - // Without the announcer saying what went: the build finds no manifest, and the plan goes on. - asked0 := time.Now().UTC().Add(-time.Minute) - plan := inventory.Plan{ID: "plan-deleted", Repository: "novox/mesh-catalog", Commit: "c2", Created: asked0, - State: inventory.PlanBuilding, Tiers: [][]string{{"kept"}}, - Modules: map[string]*inventory.PlanModule{"kept": {State: "asked", AskedAt: &asked0, Build: "build-k"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - planBuilt(ctx, open, "kept", "", "http://forge/novox/mesh-catalog.git has no module.json at modules/kept, so "+ - "there is nothing saying what it is: open …/module.json: no such file or directory", asked0, "build-k") - p, _ := inv.PlanByID(ctx, "plan-deleted") - if p.State == inventory.PlanFailed || p.Modules["kept"].State != planDeleted { - t.Fatalf("a module deleted at its source failed the plan: %s %+v", p.State, p.Modules["kept"]) - } -} func mustJSON(t *testing.T, v any) []byte { t.Helper() @@ -621,34 +193,6 @@ func nodeID(t *testing.T, open *stores, name string) string { // What a machine's witness says in its reports is a condition while it says it, by the host's words: // `core...`, urgent for a rollback, a warning when it could not judge — and // gone with the first report that no longer carries it. -func TestAWitnessesVerdictIsAConditionWhileItsReportsSayIt(t *testing.T) { - open := aMesh(t) - d := &doctor{open: open} - at := time.Now().UTC() - witnessed.heard("control", []lease.Rollback{ - {Component: lease.ComponentController, From: "sha256:new", To: "sha256:old", Outcome: lease.OutcomeRolledBack, - Why: "the new controller did not take the lease within 60s", At: at}, - {Component: lease.ComponentNodeTools, From: "sha256:t2", Outcome: lease.OutcomeUnwitnessed, Why: "no grant", At: at}, - }, at) - t.Cleanup(func() { witnessed.heard("control", nil, time.Now()) }) - obs, err := probeGates(t.Context(), d) - if err != nil { - t.Fatal(err) - } - got := map[string]conditions.Severity{} - for _, o := range obs { - got[o.Key()] = o.Severity - } - want := map[string]conditions.Severity{"core.controller.control.rolled-back": conditions.Urgent, - "core.node-tools.control.unwitnessed": conditions.Warning} - if !reflect.DeepEqual(got, want) { - t.Fatalf("the witness's verdicts are %v", got) - } - witnessed.heard("control", nil, time.Now()) - if obs, err := probeGates(t.Context(), d); err != nil || len(obs) != 0 { - t.Fatalf("a verdict the reports no longer carry is still said: %+v %v", obs, err) - } -} // judgeLeavesAsTheCore makes a test's leaf modules judged as a core module is, three readings after the settle // time, for a test of the gate's other rules (a fault, the bound, a supersession) that needs a judging still diff --git a/cmd/mesh-controller/grant_follows_binding_integration_test.go b/cmd/mesh-controller/grant_follows_binding_integration_test.go new file mode 100644 index 00000000..74622b3d --- /dev/null +++ b/cmd/mesh-controller/grant_follows_binding_integration_test.go @@ -0,0 +1,157 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, m := range storeManifests() { + register(t, open, m) + } + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil { + t.Fatal(err) + } + for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + + // Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded. + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { + t.Fatal(err) + } + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { + t.Fatalf("pinned to the anchor and bound to %s", n.From) + } + if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { + t.Fatal(err) + } + grants, _, unbound, err := grantsFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { + t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound) + } + + // Pinned back beside its data, as the operator did. + if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil { + t.Fatal(err) + } + plan, _, err = planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" { + t.Fatalf("pinned back to the laptop and bound to %s", n.From) + } + if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { + t.Fatal(err) + } + holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop") + if err != nil { + t.Fatal(err) + } + if len(holders) != 2 { + t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders) + } + + // The store it left: no longer granted, and said. + grants, _, unbound, err = grantsFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" { + t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g) + } + if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" || + strings.Join(unbound[0].BoundTo, ",") != "laptop" { + t.Fatalf("the consumer that moved is not the one said: %+v", unbound) + } + planned, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, "anchor", planned, settings) + if err != nil { + t.Fatal(err) + } + if got := declared.Received["store"]["postgres-database"]; len(got) != 0 { + t.Fatalf("the anchor's store is still told about %+v", got) + } + said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) + if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") || + !strings.Contains(said, "cleanup delete") { + t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said) + } + + // The store it is bound to: granted. + grants, _, unbound, err = grantsFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { + t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound) + } + + // And the credential from the store it left is kept: the key to the login and data held there. + if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 { + t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err) + } +} + +func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + for _, m := range storeManifests() { + register(t, open, m) + } + for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + if _, _, err := planFor(ctx, open, "laptop"); err != nil { + t.Fatal(err) + } + + // The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and + // that is not its set failing to compose. + laptop, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil { + t.Fatal(err) + } + if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) { + t.Fatalf("the seam this test relies on moved: %v", err) + } + grants, _, unbound, err := grantsFor(ctx, open, "anchor") + if err == nil { + t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound) + } + if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") { + t.Fatalf("the error does not say whose resolution could not be read: %v", err) + } +} diff --git a/cmd/mesh-controller/grant_follows_binding_test.go b/cmd/mesh-controller/grant_follows_binding_test.go index 306de2ba..9b9c6cd4 100644 --- a/cmd/mesh-controller/grant_follows_binding_test.go +++ b/cmd/mesh-controller/grant_follows_binding_test.go @@ -26,154 +26,9 @@ func grantOf(grants []catalogue.Grant, provision, consumer, module string) (cata // credentials are on record. The store it left is no longer granted it — so it retires it and keeps // its data (ADR 0230) — and says so; the store it is bound to keeps its grant; and the credential from // the store it left stays on record. -func TestAConsumerPinnedBackIsNoLongerGrantedByTheProviderItLeft(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, m := range storeManifests() { - register(t, open, m) - } - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - if err := inv.HoldSeat(ctx, "mesh-store", catalogue.ScopeMesh, "anchor", "store"); err != nil { - t.Fatal(err) - } - for _, a := range [][2]string{{"laptop", "store"}, {"laptop", "board"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - - // Bound to the anchor's store, and sent so: the credential from the anchor is made and recorded. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "anchor", "store"); err != nil { - t.Fatal(err) - } - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "anchor" { - t.Fatalf("pinned to the anchor and bound to %s", n.From) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - grants, _, unbound, err := grantsFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { - t.Fatalf("a consumer bound to the anchor is not granted there: %+v, unbound %+v", grants, unbound) - } - - // Pinned back beside its data, as the operator did. - if err := inv.PinProvision(ctx, "laptop", "postgres-database", "laptop", "store"); err != nil { - t.Fatal(err) - } - plan, _, err = planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if n := need(t, plan, "board", "postgres-database"); n.From != "laptop" { - t.Fatalf("pinned back to the laptop and bound to %s", n.From) - } - if err := inv.RecordBindings(ctx, "laptop", boundToData(plan, nil)); err != nil { - t.Fatal(err) - } - holders, err := inv.HoldersOf(ctx, "postgres-database", "laptop") - if err != nil { - t.Fatal(err) - } - if len(holders) != 2 { - t.Fatalf("today's state is two credentials on record, one from each store: %+v", holders) - } - - // The store it left: no longer granted, and said. - grants, _, unbound, err = grantsFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); ok && g.From != "" { - t.Fatalf("the anchor's store is still granted a consumer bound to the laptop's: %+v", g) - } - if len(unbound) != 1 || unbound[0].Module != "board" || unbound[0].Provider != "anchor" || - strings.Join(unbound[0].BoundTo, ",") != "laptop" { - t.Fatalf("the consumer that moved is not the one said: %+v", unbound) - } - planned, settings, err := planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(ctx, open, "anchor", planned, settings) - if err != nil { - t.Fatal(err) - } - if got := declared.Received["store"]["postgres-database"]; len(got) != 0 { - t.Fatalf("the anchor's store is still told about %+v", got) - } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, "board on laptop is bound to laptop for postgres-database, not to anchor") || - !strings.Contains(said, "cleanup delete") { - t.Fatalf("the push does not say whom the anchor no longer grants:\n%s", said) - } - - // The store it is bound to: granted. - grants, _, unbound, err = grantsFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if g, ok := grantOf(grants, "postgres-database", "laptop", "board"); !ok || g.From != "board" || len(unbound) != 0 { - t.Fatalf("the consumer is not granted by the store it is bound to: %+v, unbound %+v", grants, unbound) - } - - // And the credential from the store it left is kept: the key to the login and data held there. - if holders, err = inv.HoldersOf(ctx, "postgres-database", "laptop"); err != nil || len(holders) != 2 { - t.Fatalf("a credential was forgotten while its provider still holds the login: %+v, %v", holders, err) - } -} // A consumer whose resolution cannot be read is an error, never a consumer bound nowhere — withdrawing // a grant on that reading would take its access away (issue 152). -func TestAConsumerWhoseResolutionCannotBeReadIsNotWithdrawn(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - for _, m := range storeManifests() { - register(t, open, m) - } - for _, a := range [][2]string{{"anchor", "store"}, {"laptop", "board"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - if _, _, err := planFor(ctx, open, "laptop"); err != nil { - t.Fatal(err) - } - - // The laptop's key changes to one nothing can seal to: its resolution cannot be completed, and - // that is not its set failing to compose. - laptop, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSealingKey(ctx, laptop.ID, "not a key"); err != nil { - t.Fatal(err) - } - if _, _, err := planFor(ctx, open, "laptop"); err == nil || unresolvable(err) { - t.Fatalf("the seam this test relies on moved: %v", err) - } - grants, _, unbound, err := grantsFor(ctx, open, "anchor") - if err == nil { - t.Fatalf("an unreadable consumer was answered: grants %+v, unbound %+v", grants, unbound) - } - if !strings.Contains(err.Error(), "what laptop asked of postgres-database cannot be read") { - t.Fatalf("the error does not say whose resolution could not be read: %v", err) - } -} // The rule itself, on a resolution: bound is the provider a need for exactly that credential is // answered by, never one answered by a record, and a different local name is a different credential. diff --git a/cmd/mesh-controller/grants_step_integration_test.go b/cmd/mesh-controller/grants_step_integration_test.go new file mode 100644 index 00000000..c40c1bd3 --- /dev/null +++ b/cmd/mesh-controller/grants_step_integration_test.go @@ -0,0 +1,27 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, +// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). +func TestTheGrantsStepFixturesAreManifests(t *testing.T) { + for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), + "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { + m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" + raw, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if _, err := catalogue.ParseManifest(raw); err != nil { + t.Errorf("%s: %v", name, err) + } + } +} diff --git a/cmd/mesh-controller/grants_step_store_test.go b/cmd/mesh-controller/grants_step_store_test.go index 095a84f1..705dc8fb 100644 --- a/cmd/mesh-controller/grants_step_store_test.go +++ b/cmd/mesh-controller/grants_step_store_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/grants_step_test.go b/cmd/mesh-controller/grants_step_test.go index 70e275b2..48b34bb5 100644 --- a/cmd/mesh-controller/grants_step_test.go +++ b/cmd/mesh-controller/grants_step_test.go @@ -177,22 +177,6 @@ func TestTheGrantsStepKeepsOneMachinesBuilds(t *testing.T) { } } -// The store test's manifests are manifests the controller takes in: checked here, where no store is needed, -// so a fixture the module rules refuse fails before the store test is run (repo-check of #230 at 325575b2). -func TestTheGrantsStepFixturesAreManifests(t *testing.T) { - for name, m := range map[string]map[string]any{"nats": natsFixture(), "restic-c1": resticFixture(false), - "restic-c2": resticFixture(true), "wallpaper": wallpaperFixture()} { - m["module"], m["version"] = strings.SplitN(name, "-", 2)[0], "1" - raw, err := json.Marshal(m) - if err != nil { - t.Fatal(err) - } - if _, err := catalogue.ParseManifest(raw); err != nil { - t.Errorf("%s: %v", name, err) - } - } -} - // The delivery plan says the step before the merge. func TestAChangePlanSaysTheGrantsStep(t *testing.T) { const catalogue_ = "http://forge.internal:20000/novox/mesh-catalog.git" diff --git a/cmd/mesh-controller/group_one_batch_test.go b/cmd/mesh-controller/group_one_batch_test.go new file mode 100644 index 00000000..b7c69e57 --- /dev/null +++ b/cmd/mesh-controller/group_one_batch_test.go @@ -0,0 +1,139 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +// A group is one batch and one plan (novox/hq ADR 0418 clauses 3 and 6): the batch holding a merge of a group with a +// member still to merge does not close by the merge window, however long that member takes, and a group with a +// member of each kind is one batch whatever order its members are heard in. + +func TestAGroupsBatchWaitsForItsLastMemberHoweverLongItTakes(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + states := map[string]string{"novox/one": "ready", "novox/two": "ready"} + groupStates(t, "feat/g", states) + first := fromHead(repoMerge("one", "c1", t0), "feat/g") + hear(t, open, first, t0) + // Past the window and past its maximum, with the second member still to merge: one batch, no walk. + for _, after := range []time.Duration{90 * time.Second, 11 * time.Minute, 30 * time.Minute} { + cutAt(t, open, t0.Add(after)) + ws, bs := walks(t, open) + if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 1 { + t.Fatalf("%s after the first member: %d walk(s), %d batch(es) %+v", after, len(ws), len(bs), bs) + } + } + // The last member merges: the window runs again from that merge. + at := t0.Add(30 * time.Minute) + states["novox/two"] = "merged" + hear(t, open, fromHead(repoMerge("two", "c2", at), "feat/g"), at.Add(time.Second)) + cutAt(t, open, at.Add(time.Second+89*time.Second)) + if ws, _ := walks(t, open); len(ws) != 0 { + t.Fatalf("cut before the window ran again: %+v", ws) + } + cutAt(t, open, at.Add(time.Second+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/one") != "c1" || ws[0].CommitOf("novox/two") != "c2" { + t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) + } +} + +// A group with a member of each kind is one batch and one walk, in both orders of hearing, started without +// mesh-delivery's word. +func TestAGroupWithAMemberOfEachKindIsOneBatchAndOneWalk(t *testing.T) { + for name, ownFirst := range map[string]bool{"controller member first": true, "catalogue member first": false} { + t.Run(strings.ReplaceAll(name, " ", "_"), func(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} + groupStates(t, "feat/both-kinds", states) + own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds") + cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds") + first, second := cat, own + if ownFirst { + first, second = own, cat + } + second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano) + hear(t, open, first, t0) + states["novox/"+second.Repo] = "merged" + hear(t, open, second, t0.Add(10*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 0 || len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { + t.Fatalf("both members heard: %d walk(s), %d batch(es) %+v; want one batch of both", len(ws), len(bs), bs) + } + cutAt(t, open, t0.Add(10*time.Second+90*time.Second)) + ws, bs = walks(t, open) + if len(ws) != 1 || len(bs) != 0 || ws[0].CommitOf("novox/mesh-controller") != "k1" || + ws[0].CommitOf("novox/mesh-catalog") != cat.Commit { + t.Fatalf("after the window: %d walk(s) %+v, %d batch(es)", len(ws), ws, len(bs)) + } + if ws[0].Delivery != nil && ws[0].Delivery.Awaits != "" { + t.Fatalf("a plan carrying the controller waits for %s's word", ws[0].Delivery.Awaits) + } + }) + } +} + +// A group whose last member has not merged ten minutes after its first raises a condition naming it, and the held +// batch is not the batch S18 reads as a cut the controller failed to make. +func TestAHeldBatchRaisesTheGroupConditionAndNotTheBatchNotCutOne(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) + now := t0.Add(12 * time.Minute) + batches, err := gatherBatches(t.Context(), open.inventory, now) + if err != nil || len(batches) != 1 { + t.Fatalf("batches %+v: %v", batches, err) + } + f := &signalFacts{now: now, batches: batches} + if got := watchBatchesNotCut(f); len(got) != 0 { + t.Fatalf("a held batch is read as not cut: %+v", got) + } + got := watchGroupsNotMerged(f) + if len(got) != 1 || got[0].Kind != kindGroupNotMerged || !strings.Contains(got[0].Summary, "novox/two") || + !strings.Contains(got[0].Summary, "feat/g") { + t.Fatalf("the group's condition: %+v", got) + } + if early := watchGroupsNotMerged(&signalFacts{now: t0.Add(5 * time.Minute), batches: batches}); len(early) != 0 { + t.Fatalf("raised before the maximum: %+v", early) + } +} + +// The batch held for a group survives a restart: the hold is in the record. +func TestAHeldBatchSurvivesARestart(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + groupStates(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) + reopened, err := open.inventory.Batches(t.Context()) + if err != nil || len(reopened) != 1 || len(reopened[0].Delivery.Batch.Holds) != 1 { + t.Fatalf("the record of the held batch: %+v %v", reopened, err) + } + cutAt(t, open, t0.Add(time.Hour)) + if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 || bs[0].State != inventory.PlanAssembling { + t.Fatalf("after an hour: %d walk(s), batches %+v", len(ws), bs) + } +} + +// A mesh-delivery that does not yet say the group's gate is composed merges member two only after member one is +// delivered, so a batch held for it would wait for ever: the merge waits for the window, as before. +func TestAnOlderMeshDeliverysGroupIsNotHeldOpen(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + groupStatesFrom(t, "feat/g", map[string]string{"novox/one": "ready", "novox/two": "ready"}, false) + hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/g"), t0) + if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Batch.Holds) != 0 { + t.Fatalf("held for an older mesh-delivery's group: %+v", bs) + } + cutAt(t, open, t0.Add(90*time.Second)) + if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { + t.Fatalf("at its window: %d walk(s), %d batch(es)", len(ws), len(bs)) + } +} diff --git a/cmd/mesh-controller/group_wait.go b/cmd/mesh-controller/group_wait.go index 1be26410..46a22e03 100644 --- a/cmd/mesh-controller/group_wait.go +++ b/cmd/mesh-controller/group_wait.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "sort" "strings" "time" @@ -34,6 +35,10 @@ var openPullStates = map[string]bool{"proposed": true, "checked": true, "ready": // groupMember is one member of a delivery group as mesh-delivery's `groups` says it: `@ (state)`. type groupMember struct { Repository, State string + // Composed says the group's line says its gate is the composed check's (a mesh-delivery that merges a group in + // one pass): only such a group is held open for its last member, since an older mesh-delivery merges member two + // only after member one is delivered, and a held batch would wait for ever. + Composed bool } // groupsLine is one group as mesh-delivery's `groups` says it: its id (the head branch name), its members as text and, @@ -43,6 +48,10 @@ type groupsLine struct { ID string `json:"id"` Members []string `json:"members"` Details []memberLine `json:"member_details,omitempty"` + // ComposedGate says the group's gate is the composed check's (novox/hq ADR 0403): a member's own check runs the + // repository's check alone. Said by a mesh-delivery that merges a group in one pass; absent from an older one, whose + // members keep their own gate. + ComposedGate bool `json:"composed_gate,omitempty"` } // memberLine is one member of a group, as `groups` says it beside its text. @@ -98,7 +107,9 @@ var deliveryGroupOf = func(ctx context.Context, held bool, head string) ([]group } var members []groupMember for _, s := range l.Members { - members = append(members, memberOf(s)) + m := memberOf(s) + m.Composed = l.ComposedGate + members = append(members, m) } return members, true, nil } @@ -116,75 +127,102 @@ func memberOf(s string) groupMember { } // groupWait says whether a merge heard now waits for the window for its delivery group, and why in words; nothing -// when it names no head, its head names no group, or every other member is merged or closed. own is the merge's -// kind (ownPath). +// when it names no head, its head names no group, or every other member is merged or closed. // -// A member of another repository whose merge this controller already keeps in a batch not yet cut of the same -// kind is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins -// that batch by their head. One kept in a batch of the other kind keeps the merge waiting, whatever mesh-delivery -// says (review of #250): kinds never share a batch, so the merge's batch is a new one, and cut at once it would -// walk before that member's batch is cut and break the group's order (ADR 0276 decision 5). Waiting, the two -// batches are cut in the order their windows close, an own-path one first at one look, as cutBatchesHeld cuts -// every pair of kinds. +// A member of another repository whose merge this controller already keeps in a batch not yet cut, of either +// kind, is merged, whatever mesh-delivery says yet (it marks a member merged once its walk opened): the merge joins +// that batch by their head. **Kinds do not part a group** (novox/hq ADR 0418 clause 6): a controller member and a +// catalogue member share one batch. // // Asked before the plans are held: a slow answer holds up no walk. func groupWait(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held, own bool) (string, error) { + why, _, err := groupWaitHolding(ctx, inv, m, held) + return why, err +} + +// groupWaitHolding is groupWait, and whether the wait is for a member still to merge: such a batch is held open +// however long it takes (ADR 0418 clause 3). A wait only because mesh-delivery did not say is no hold. +func groupWaitHolding(ctx context.Context, inv *inventory.Inventory, m link.SourceMoved, held bool) (string, bool, error) { if m.Head == "" { - return "", nil + return "", false, nil } - same, other, err := keptFromHead(ctx, inv, m.Head, own) - if err != nil { - return "", err - } - if len(other) > 0 { - return "group " + m.Head + " has a member waiting in a batch of the other kind: " + strings.Join(other, ", "), nil - } - members, isGroup, err := deliveryGroupOf(ctx, held, m.Head) + open, isGroup, composed, err := openMembersOf(ctx, inv, held, m.Head, m.Owner+"/"+m.Repo) if err != nil { return fmt.Sprintf("%s did not say whether group %s has a member still to merge (%v): it waits for the "+ - "window, the safe side", catalogue.DeliverySeat, m.Head, err), nil + "window, the safe side", catalogue.DeliverySeat, m.Head, err), false, nil } - if !isGroup { - return "", nil + if !isGroup || len(open) == 0 { + return "", false, nil + } + return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), composed, nil +} + +// openMembersOf are the members of the group a head names that are still to merge, in words: those whose pull +// request is open and whose merge no batch not yet cut keeps. self is a repository to leave out, the merge being heard. +func openMembersOf(ctx context.Context, inv *inventory.Inventory, held bool, head, self string) ([]string, bool, bool, error) { + kept, err := keptFromHead(ctx, inv, head) + if err != nil { + return nil, false, false, err + } + members, isGroup, err := deliveryGroupOf(ctx, held, head) + if err != nil || !isGroup { + return nil, isGroup, false, err } var open []string + composed := false for _, g := range members { - if strings.EqualFold(g.Repository, m.Owner+"/"+m.Repo) || same[strings.ToLower(g.Repository)] || - !openPullStates[g.State] { + composed = composed || g.Composed + if strings.EqualFold(g.Repository, self) || kept[strings.ToLower(g.Repository)] || !openPullStates[g.State] { continue } open = append(open, g.Repository+" ("+g.State+")") } - if len(open) == 0 { - return "", nil - } - return "group " + m.Head + " has a member still to merge: " + strings.Join(open, ", "), nil + sort.Strings(open) + return open, true, composed, nil } -// keptFromHead are the merges from a head that batches not yet cut hold: the repositories (lower case) of those in -// a batch of the kind own says, and, in words, each of the other kind with its batch. -func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string, own bool) (map[string]bool, []string, error) { +// keptFromHead are the repositories (lower case) of the merges from a head that batches not yet cut hold, of +// either kind. +func keptFromHead(ctx context.Context, inv *inventory.Inventory, head string) (map[string]bool, error) { batches, err := inv.Batches(ctx) if err != nil { - return nil, nil, err + return nil, err } - same := map[string]bool{} - var other []string + kept := map[string]bool{} for _, b := range batches { merges, err := inv.MergesOf(ctx, b.ID) if err != nil { - return nil, nil, err + return nil, err } for _, k := range merges { - if announcedOf(k).Head != head { - continue - } - if b.OwnPath() == own { - same[strings.ToLower(k.Repository)] = true - } else { - other = append(other, fmt.Sprintf("%s@%s in %s", k.Repository, short(k.Commit), b.ID)) + if announcedOf(k).Head == head { + kept[strings.ToLower(k.Repository)] = true } } } - return same, other, nil + return kept, nil +} + +// batchOfGroup is the batch not yet cut that holds a merge of another repository from the same head: the group's +// batch, whatever kind it is. +func batchOfGroup(ctx context.Context, inv *inventory.Inventory, head, repository string) (inventory.Plan, bool, error) { + if head == "" { + return inventory.Plan{}, false, nil + } + batches, err := inv.Batches(ctx) + if err != nil { + return inventory.Plan{}, false, err + } + for _, b := range batches { + merges, err := inv.MergesOf(ctx, b.ID) + if err != nil { + return inventory.Plan{}, false, err + } + for _, k := range merges { + if announcedOf(k).Head == head && !strings.EqualFold(k.Repository, repository) { + return b, true, nil + } + } + } + return inventory.Plan{}, false, nil } diff --git a/cmd/mesh-controller/healers_integration_test.go b/cmd/mesh-controller/healers_integration_test.go new file mode 100644 index 00000000..3a55294d --- /dev/null +++ b/cmd/mesh-controller/healers_integration_test.go @@ -0,0 +1,449 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, clock := healingOn(t, open) + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil { + t.Fatal(err) + } + key := sentNotReported(t, "laptop") + + // First: the report had not reached the mesh, and asking for it brings it. + asked := 0 + h.askReport = func(ctx context.Context, node string) error { + asked++ + if node != "laptop" { + t.Errorf("asked %s", node) + } + _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied, + At: time.Now().Add(time.Second), Declared: "d2"}) + return err + } + h.tick(ctx) + c, found, err := conditionsFrom.Get(ctx, key) + if err != nil || !found { + t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err) + } + if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") || + c.Resolver != "healer:H1" { + t.Fatalf("after the first act: asked %d, %+v", asked, c) + } + if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed || + acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 { + t.Fatalf("the act was not said as healer-acted: %+v", acts) + } + + // Within its settle, nothing more: the observation has its turn. + clock.pass(time.Minute) + h.tick(ctx) + if asked != 1 { + t.Fatalf("acted again inside the settle: asked %d", asked) + } + + // Second: the machine says nothing, so it is sent again. + clock.pass(3 * time.Minute) + sent := 0 + h.askReport = func(context.Context, string) error { asked++; return nil } + h.sendAgain = func(_ context.Context, node string) error { sent++; return nil } + h.tick(ctx) + if asked != 2 || sent != 1 { + t.Fatalf("the second act: asked %d, sent %d", asked, sent) + } + c, _, _ = conditionsFrom.Get(ctx, key) + if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") { + t.Fatalf("tried %+v", c.Tried) + } + + // The budget is spent: the operator's, urgent, said; and no healer touches it again. + clock.pass(4 * time.Minute) + h.tick(ctx) + c, _, _ = conditionsFrom.Get(ctx, key) + if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 || + !strings.Contains(c.Tried[2].Outcome, "budget of 2") { + t.Fatalf("not handed to the operator: %+v", c) + } + if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated { + t.Fatalf("the escalation was not said: %+v", acts) + } + clock.pass(time.Hour) + h.tick(ctx) + if asked != 2 || sent != 1 || len(told.said()) != 3 { + t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said())) + } + heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour)) + if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed || + heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 { + t.Fatalf("the heals kept: %+v %v", heals, err) + } + // And a heal is not a hand act: what S15 counts never sees it. + for _, x := range heals { + if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") { + t.Errorf("a heal reads as a hand act: %+v", x) + } + } +} + +func TestNoHealerActsWithoutTheLease(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, _ := healingOn(t, open) + sentNotReported(t, "laptop") + h.epoch = func(context.Context) (uint64, error) { return 0, nil } + h.tick(ctx) + h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") } + h.tick(ctx) + h.epoch = func(context.Context) (uint64, error) { return 57, nil } + h.acting = func() bool { return false } + h.tick(ctx) + if len(told.said()) != 0 { + t.Fatalf("a healer acted without the lease: %+v", told.said()) + } +} + +func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, clock := healingOn(t, open) + for i := 0; i < healBrakeLimit; i++ { + if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent", + Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed, + At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil { + t.Fatal(err) + } + } + sentNotReported(t, "laptop") + h.tick(ctx) // the fakes fail the test if anything acts + braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked") + if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") { + t.Fatalf("the brake was not said: %+v %v", braked, err) + } + if len(told.said()) != 0 { + t.Fatalf("a healer acted under the brake: %+v", told.said()) + } + // Past the hour of the first act, still held: it lets go an hour after the last. + clock.pass(30 * time.Minute) + h.tick(ctx) + if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held { + t.Fatal("the brake let go before an hour had passed since the last act") + } + clock.pass(31 * time.Minute) + asked := 0 + h.askReport = func(context.Context, string) error { asked++; return nil } + h.sendAgain = func(context.Context, string) error { return nil } + h.tick(ctx) + if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held { + t.Fatal("the brake held an hour after the last act") + } + if asked != 1 { + t.Fatalf("the healers did not act again after the brake let go: asked %d", asked) + } +} + +func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, told, _ := healingOn(t, open) + created := time.Now().UTC().Add(-time.Hour) + older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}} + waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created, + State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}} + newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute), + State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}} + for _, p := range []*inventory.Plan{&older, &waiting, &newer} { + if err := open.inventory.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + } + for _, id := range []string{"plan-old", "plan-other"} { + if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled", + Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil { + t.Fatal(err) + } + } + h.tick(ctx) + closed, err := open.inventory.PlanByID(ctx, "plan-old") + if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") || + !strings.Contains(closed.Note, "plan-new") { + t.Fatalf("the superseded plan: %+v %v", closed, err) + } + if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding { + t.Fatalf("a plan still waiting was closed: %+v", other) + } + if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf { + t.Fatalf("a plan H2 does not repair was touched: %+v", c) + } + if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" { + t.Fatalf("said %+v", acts) + } + // Finished: every module built, none rolled out unsent — closed as done. + done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created, + State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}} + if err := open.inventory.SavePlan(ctx, &done); err != nil { + t.Fatal(err) + } + if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") { + t.Fatalf("a finished plan reads %q %q %v", state, why, err) + } +} + +func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + h, _, _ := healingOn(t, open) + marked := 0 + for _, c := range broker.MeshConsumers() { + if c.Resettable != "" { + marked++ + if c.Stream != broker.EventsStream || c.Name != broker.ControllerName { + t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream) + } + } + } + if marked != 1 { + t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked) + } + for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} { + if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind", + Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil { + t.Fatal(err) + } + } + h.tick(ctx) // the fake reset fails the test if it is called + reset := "" + h.resetConsumer = func(stream, name string) (string, error) { + reset = stream + "." + name + return "it was 1500 behind", nil + } + if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller", + Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil { + t.Fatal(err) + } + h.tick(ctx) + if reset != "EVENTS.controller" { + t.Fatalf("reset %q", reset) + } +} + +func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) { + url := testbus.URL(t) + open := aMesh(t) + ctx := t.Context() + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { + _ = js.Context().DeleteStream(s) + } + if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { + t.Fatal(err) + } + h, told, clock := healingOn(t, open) + h.js = js + h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") } + d := &doctor{open: open, js: js} + probe := func() { + t.Helper() + found, err := probeConsumers(ctx, d) + if err != nil { + t.Fatal(err) + } + if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { + t.Fatal(err) + } + } + const key = "bus.NODES.laptop.missing" + + if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { + t.Fatal(err) + } + probe() + if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" { + t.Fatalf("the deleted consumer was not raised: %+v", c) + } + h.tick(ctx) + if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil { + t.Fatalf("H3 did not make the consumer again: %v", err) + } + c, still, _ := conditionsFrom.Get(ctx, key) + if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" { + t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c) + } + probe() + if _, still, _ := conditionsFrom.Get(ctx, key); still { + t.Fatal("the probe's next run did not clear what H3 repaired") + } + // Kept in the history as the keeper says it, a moment later. + var cleared *conditions.Event + for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) { + history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute)) + if err != nil { + t.Fatal(err) + } + for i := range history { + if history[i].Key == key && history[i].Change == conditions.ChangeCleared { + cleared = &history[i] + } + } + } + if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 { + t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared) + } + + // Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops. + clock.pass(10 * time.Minute) + if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { + t.Fatal(err) + } + probe() + h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil } + h.tick(ctx) + c, _, _ = conditionsFrom.Get(ctx, key) + if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 { + t.Fatalf("the spent budget was not handed to the operator: %+v", c) + } + acts := told.said() + if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated { + t.Fatalf("said %+v", acts) + } + clock.pass(2 * time.Hour) + h.tick(ctx) + if len(told.said()) != 2 { + t.Fatal("a healer acted on what the operator holds") + } +} + +func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) { + url := testbus.URL(t) + open := aMesh(t) + ctx := t.Context() + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { + _ = js.Context().DeleteStream(s) + } + if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { + t.Fatal(err) + } + // Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none. + events := make(chan *nats.Msg, 64) + sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + followed := broker.ControllerFollows[0] + for i := 0; i < consumerFarBehind+200; i++ { + if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil { + t.Fatal(err) + } + } + d := &doctor{open: open, js: js} + probe := func() []conditions.Observation { + t.Helper() + found, err := probeConsumers(ctx, d) + if err != nil { + t.Fatal(err) + } + if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { + t.Fatal(err) + } + return found + } + probe() + const key = "bus.EVENTS.controller.behind" + if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind { + t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c) + } + h, told, _ := healingOn(t, open) + h.resetConsumer = func(stream, name string) (string, error) { + before, after, err := js.ResetConsumer(stream, name) + if err != nil { + return "", err + } + return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) + + " pending now", nil + } + h.tick(ctx) + if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed { + t.Fatalf("said %+v", acts) + } + if found := probe(); len(found) != 0 { + t.Fatalf("after the reset the probe still finds %+v", found) + } + if _, still, _ := conditionsFrom.Get(ctx, key); still { + t.Fatal("the probe's next run did not clear what H4 repaired") + } + // What comes next still reaches the controller. + for len(events) > 0 { + <-events + } + if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil { + t.Fatal(err) + } + deadline := time.After(10 * time.Second) + for { + select { + case m := <-events: + if strings.Contains(string(m.Data), "after") { + return + } + _ = m.Ack() + case <-deadline: + t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf") + } + } +} + +func TestNatsAskToReportReachesTheMachine(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop")) + if err != nil { + t.Fatal(err) + } + if err := askToReport(t.Context(), conn, "laptop"); err != nil { + t.Fatal(err) + } + msg, err := asked.NextMsg(5 * time.Second) + if err != nil || !strings.Contains(string(msg.Data), "healer H1") { + t.Fatalf("the machine heard %v, %v", msg, err) + } + perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"}) + if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") || + slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") { + t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err) + } +} diff --git a/cmd/mesh-controller/healers_test.go b/cmd/mesh-controller/healers_test.go index dc2d0754..11829431 100644 --- a/cmd/mesh-controller/healers_test.go +++ b/cmd/mesh-controller/healers_test.go @@ -6,20 +6,14 @@ import ( "errors" "os" "slices" - "strconv" "strings" "sync" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // The test generated from the healer registry (novox/hq to-be 45 §7, ADR 0227 rule 7 "how it is @@ -56,9 +50,14 @@ var inducedFailures = map[string]string{ func TestEveryHealerAnswersAKindTheMeshRaisesWithABudgetABrakeAndItsEvent(t *testing.T) { kinds := raisedKinds() - source, err := os.ReadFile("healers_test.go") - if err != nil { - t.Fatal(err) + // The induced failures are tests of the integration level (a bus), in the file beside this one (issue 895). + var source []byte + for _, name := range []string{"healers_test.go", "healers_integration_test.go"} { + more, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + source = append(source, more...) } seen := map[string]bool{} answered := map[string]string{} @@ -195,446 +194,24 @@ func sentNotReported(t *testing.T, node string) string { // times): the machine is asked to report; a report that names what it was sent is all it takes, and one // that does not — or none — is a send of its current declaration again. Each act kept in `tried` as // `healer H1`, said as healer-acted, counted; twice, then the operator's, urgent; then nothing more. -func TestH1AsksAMachineToReportAndSendsItAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSent(ctx, record.ID, "d2", nil); err != nil { - t.Fatal(err) - } - key := sentNotReported(t, "laptop") - - // First: the report had not reached the mesh, and asking for it brings it. - asked := 0 - h.askReport = func(ctx context.Context, node string) error { - asked++ - if node != "laptop" { - t.Errorf("asked %s", node) - } - _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeApplied, - At: time.Now().Add(time.Second), Declared: "d2"}) - return err - } - h.tick(ctx) - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found { - t.Fatalf("the condition is gone after the act — a healer cleared it, not an observation: %v", err) - } - if asked != 1 || len(c.Tried) != 1 || c.Tried[0].By != "healer H1" || !strings.Contains(c.Tried[0].Outcome, "acted") || - c.Resolver != "healer:H1" { - t.Fatalf("after the first act: asked %d, %+v", asked, c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H1" || acts[0].Outcome != inventory.HealActed || - acts[0].Condition != key || acts[0].By != "healer H1" || acts[0].Epoch != 57 { - t.Fatalf("the act was not said as healer-acted: %+v", acts) - } - - // Within its settle, nothing more: the observation has its turn. - clock.pass(time.Minute) - h.tick(ctx) - if asked != 1 { - t.Fatalf("acted again inside the settle: asked %d", asked) - } - - // Second: the machine says nothing, so it is sent again. - clock.pass(3 * time.Minute) - sent := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(_ context.Context, node string) error { sent++; return nil } - h.tick(ctx) - if asked != 2 || sent != 1 { - t.Fatalf("the second act: asked %d, sent %d", asked, sent) - } - c, _, _ = conditionsFrom.Get(ctx, key) - if len(c.Tried) != 2 || !strings.Contains(c.Tried[1].Outcome, "sent again") || !strings.Contains(c.Tried[1].Outcome, "act 2 of 2") { - t.Fatalf("tried %+v", c.Tried) - } - - // The budget is spent: the operator's, urgent, said; and no healer touches it again. - clock.pass(4 * time.Minute) - h.tick(ctx) - c, _, _ = conditionsFrom.Get(ctx, key) - if !c.Escalated() || c.Severity != conditions.Urgent || len(c.Tried) != 3 || - !strings.Contains(c.Tried[2].Outcome, "budget of 2") { - t.Fatalf("not handed to the operator: %+v", c) - } - if acts := told.said(); len(acts) != 3 || acts[2].Outcome != inventory.HealEscalated { - t.Fatalf("the escalation was not said: %+v", acts) - } - clock.pass(time.Hour) - h.tick(ctx) - if asked != 2 || sent != 1 || len(told.said()) != 3 { - t.Fatalf("a healer acted on a condition the operator holds: asked %d sent %d said %d", asked, sent, len(told.said())) - } - heals, err := open.inventory.HealsSince(ctx, time.Now().Add(-time.Hour)) - if err != nil || len(heals) != 3 || heals[0].Outcome != inventory.HealActed || heals[1].Outcome != inventory.HealActed || - heals[2].Outcome != inventory.HealEscalated || heals[0].Epoch != 57 { - t.Fatalf("the heals kept: %+v %v", heals, err) - } - // And a heal is not a hand act: what S15 counts never sees it. - for _, x := range heals { - if x.Healer == "" || strings.HasPrefix(x.Act, "hand-act") { - t.Errorf("a heal reads as a hand act: %+v", x) - } - } -} // **Only the controller holding the lease heals** (to-be 45 §6): unleased, or standing by, nothing. -func TestNoHealerActsWithoutTheLease(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - sentNotReported(t, "laptop") - h.epoch = func(context.Context) (uint64, error) { return 0, nil } - h.tick(ctx) - h.epoch = func(context.Context) (uint64, error) { return 0, errors.New("the lease is not held") } - h.tick(ctx) - h.epoch = func(context.Context) (uint64, error) { return 57, nil } - h.acting = func() bool { return false } - h.tick(ctx) - if len(told.said()) != 0 { - t.Fatalf("a healer acted without the lease: %+v", told.said()) - } -} // **The mesh-wide brake**: a dozen acts in an hour and every healer stops, said urgently, until an hour // after the last. -func TestTheBrakeStopsEveryHealerAndSaysSo(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, clock := healingOn(t, open) - for i := 0; i < healBrakeLimit; i++ { - if _, err := open.inventory.BeginHeal(ctx, inventory.Heal{Healer: "H3", ConditionKey: "seat.x.anchor.silent", - Kind: "holder-silent", Act: "asserted", Outcome: inventory.HealActed, - At: clock.now().Add(-time.Duration(healBrakeLimit-i) * time.Minute)}); err != nil { - t.Fatal(err) - } - } - sentNotReported(t, "laptop") - h.tick(ctx) // the fakes fail the test if anything acts - braked, found, err := conditionsFrom.Get(ctx, "mesh.healers.braked") - if err != nil || !found || braked.Severity != conditions.Urgent || !strings.Contains(braked.Evidence[0].Said, "H3 on seat.x.anchor.silent ×12") { - t.Fatalf("the brake was not said: %+v %v", braked, err) - } - if len(told.said()) != 0 { - t.Fatalf("a healer acted under the brake: %+v", told.said()) - } - // Past the hour of the first act, still held: it lets go an hour after the last. - clock.pass(30 * time.Minute) - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); !held { - t.Fatal("the brake let go before an hour had passed since the last act") - } - clock.pass(31 * time.Minute) - asked := 0 - h.askReport = func(context.Context, string) error { asked++; return nil } - h.sendAgain = func(context.Context, string) error { return nil } - h.tick(ctx) - if _, held, _ := conditionsFrom.Get(ctx, "mesh.healers.braked"); held { - t.Fatal("the brake held an hour after the last act") - } - if asked != 1 { - t.Fatalf("the healers did not act again after the brake let go: asked %d", asked) - } -} // **H2 closes a plan another has taken over**, with its note — and leaves a plan alone whose wait is // still to come: that one is its own signal's, not a healer's. -func TestH2ClosesAPlanAnotherHasTakenOver(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, told, _ := healingOn(t, open) - created := time.Now().UTC().Add(-time.Hour) - older := inventory.Plan{ID: "plan-old", Repository: "novox/app", Branch: "main", Commit: "0ld0ld0", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "asked"}}} - waiting := inventory.Plan{ID: "plan-other", Repository: "novox/other", Branch: "main", Commit: "07he707", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"b"}}, Modules: map[string]*inventory.PlanModule{"b": {State: "asked"}}} - newer := inventory.Plan{ID: "plan-new", Repository: "novox/app", Branch: "main", Commit: "new0new", Created: created.Add(time.Minute), - State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{"a": {State: "built"}}} - for _, p := range []*inventory.Plan{&older, &waiting, &newer} { - if err := open.inventory.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - } - for _, id := range []string{"plan-old", "plan-other"} { - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopePlan, ID: id, Kind: "stalled", - Severity: conditions.Warning, Summary: id + " is stalled", Source: "S3"}); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) - closed, err := open.inventory.PlanByID(ctx, "plan-old") - if err != nil || closed.State != inventory.PlanSuperseded || !strings.Contains(closed.Note, "closed by healer H2") || - !strings.Contains(closed.Note, "plan-new") { - t.Fatalf("the superseded plan: %+v %v", closed, err) - } - if other, _ := open.inventory.PlanByID(ctx, "plan-other"); other.State != inventory.PlanBuilding { - t.Fatalf("a plan still waiting was closed: %+v", other) - } - if c, _, _ := conditionsFrom.Get(ctx, "plan.plan-other.stalled"); len(c.Tried) != 0 || c.Resolver != conditions.ResolverSelf { - t.Fatalf("a plan H2 does not repair was touched: %+v", c) - } - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H2" || acts[0].Condition != "plan.plan-old.stalled" { - t.Fatalf("said %+v", acts) - } - // Finished: every module built, none rolled out unsent — closed as done. - done := inventory.Plan{ID: "plan-done", Repository: "novox/third", Branch: "main", Commit: "d0ned0n", Created: created, - State: inventory.PlanRolling, Tier: 0, Tiers: [][]string{{"c"}}, Modules: map[string]*inventory.PlanModule{"c": {State: "built"}}} - if err := open.inventory.SavePlan(ctx, &done); err != nil { - t.Fatal(err) - } - if state, why, err := planStale(ctx, open.inventory, done); err != nil || state != inventory.PlanDone || !strings.Contains(why, "finished") { - t.Fatalf("a finished plan reads %q %q %v", state, why, err) - } -} // **H4 only for a consumer the stream table marks resettable**: a module's consumer far behind is said // and left — what a reset drops, nothing would catch up for it. -func TestH4ResetsOnlyWhatTheTableMarksResettable(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - h, _, _ := healingOn(t, open) - marked := 0 - for _, c := range broker.MeshConsumers() { - if c.Resettable != "" { - marked++ - if c.Stream != broker.EventsStream || c.Name != broker.ControllerName { - t.Errorf("%s on %s is marked resettable: only the controller's own events consumer is", c.Name, c.Stream) - } - } - } - if marked != 1 { - t.Fatalf("%d consumers are marked resettable, want the controller's events consumer alone", marked) - } - for _, who := range []string{"EVENTS.anchor_shop", "CONTROL.controller"} { - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: who, Token: "behind", - Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: who + " is far behind", Source: "D6"}); err != nil { - t.Fatal(err) - } - } - h.tick(ctx) // the fake reset fails the test if it is called - reset := "" - h.resetConsumer = func(stream, name string) (string, error) { - reset = stream + "." + name - return "it was 1500 behind", nil - } - if _, err := conditionsFrom.Observe(ctx, conditions.Observation{Scope: conditions.ScopeBus, ID: "EVENTS.controller", - Token: "behind", Kind: kindConsumerBehind, Severity: conditions.Warning, Summary: "far behind", Source: "D6"}); err != nil { - t.Fatal(err) - } - h.tick(ctx) - if reset != "EVENTS.controller" { - t.Fatalf("reset %q", reset) - } -} // **H3 against a real bus** (issue 208's note): a consumer the mesh expects, deleted; D6 says so; H3 // asserts the bus's objects the way a send does, and D6's next run clears it — the healer never does. // Deleted again within the hour, the budget is spent: the operator's, urgent, and H3 stops. -func TestNatsH3AssertsAMissingConsumerAgainAndBrakesAfterItsBudget(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - ctx := t.Context() - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { - t.Fatal(err) - } - h, told, clock := healingOn(t, open) - h.js = js - h.assertObjects = func(ctx context.Context) error { return assertOnSend(ctx, open.inventory, js, " ") } - d := &doctor{open: open, js: js} - probe := func() { - t.Helper() - found, err := probeConsumers(ctx, d) - if err != nil { - t.Fatal(err) - } - if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { - t.Fatal(err) - } - } - const key = "bus.NODES.laptop.missing" - - if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { - t.Fatal(err) - } - probe() - if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != "consumer-lost" { - t.Fatalf("the deleted consumer was not raised: %+v", c) - } - h.tick(ctx) - if _, err := js.Context().ConsumerInfo("NODES", "laptop"); err != nil { - t.Fatalf("H3 did not make the consumer again: %v", err) - } - c, still, _ := conditionsFrom.Get(ctx, key) - if !still || len(c.Tried) != 1 || c.Tried[0].By != "healer H3" || c.Resolver != "healer:H3" { - t.Fatalf("the act is not in the condition, or the healer cleared it: %+v", c) - } - probe() - if _, still, _ := conditionsFrom.Get(ctx, key); still { - t.Fatal("the probe's next run did not clear what H3 repaired") - } - // Kept in the history as the keeper says it, a moment later. - var cleared *conditions.Event - for wait := time.Now().Add(5 * time.Second); cleared == nil && time.Now().Before(wait); time.Sleep(20 * time.Millisecond) { - history, err := conditionsFrom.HistorySince(ctx, time.Now().Add(-time.Minute)) - if err != nil { - t.Fatal(err) - } - for i := range history { - if history[i].Key == key && history[i].Change == conditions.ChangeCleared { - cleared = &history[i] - } - } - } - if cleared == nil || !strings.Contains(cleared.Why, "D6 no longer observes it") || len(cleared.Tried) != 1 { - t.Fatalf("the clearing is not the probe's, or forgets what was tried: %+v", cleared) - } - - // Again within the hour: the budget is one, so after its settle the operator is told, and H3 stops. - clock.pass(10 * time.Minute) - if err := js.Context().DeleteConsumer("NODES", "laptop"); err != nil { - t.Fatal(err) - } - probe() - h.assertObjects = func(context.Context) error { t.Error("H3 acted past its budget"); return nil } - h.tick(ctx) - c, _, _ = conditionsFrom.Get(ctx, key) - if !c.Escalated() || c.Severity != conditions.Urgent || c.Count != 2 { - t.Fatalf("the spent budget was not handed to the operator: %+v", c) - } - acts := told.said() - if len(acts) != 2 || acts[0].Outcome != inventory.HealActed || acts[1].Outcome != inventory.HealEscalated { - t.Fatalf("said %+v", acts) - } - clock.pass(2 * time.Hour) - h.tick(ctx) - if len(told.said()) != 2 { - t.Fatal("a healer acted on what the operator holds") - } -} // **H4 against a real bus** (issue 248): the controller's events consumer a long way behind — the week it // once replayed — is re-made from now by the mesh itself, and the controller's bound subscription still // receives what comes next: a reset that left the controller deaf would be the incident. -func TestNatsH4ResetsTheControllersEventsConsumerAndItStillDelivers(t *testing.T) { - url := testbus.URL(t) - open := aMesh(t) - ctx := t.Context() - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - for _, s := range []string{"CONTROL", "NODES", "ASSIGNMENTS", "EVENTS"} { - _ = js.Context().DeleteStream(s) - } - if _, err := assertBusObjects(ctx, open.inventory, js); err != nil { - t.Fatal(err) - } - // Bound as the controller binds it (link/receive_nats.go), taking one and acknowledging none. - events := make(chan *nats.Msg, 64) - sub, err := js.Context().ChanSubscribe("", events, nats.Bind(broker.EventsStream, broker.ControllerName)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - followed := broker.ControllerFollows[0] - for i := 0; i < consumerFarBehind+200; i++ { - if _, err := js.Context().Publish(followed, []byte(`{"n":`+strconv.Itoa(i)+`}`)); err != nil { - t.Fatal(err) - } - } - d := &doctor{open: open, js: js} - probe := func() []conditions.Observation { - t.Helper() - found, err := probeConsumers(ctx, d) - if err != nil { - t.Fatal(err) - } - if err := conditionsFrom.Reconcile(ctx, "D6", kindedAs(found, "consumer-wrong")); err != nil { - t.Fatal(err) - } - return found - } - probe() - const key = "bus.EVENTS.controller.behind" - if c, raised, _ := conditionsFrom.Get(ctx, key); !raised || c.Kind != kindConsumerBehind { - t.Fatalf("a consumer %d behind was not raised: %+v", consumerFarBehind+200, c) - } - h, told, _ := healingOn(t, open) - h.resetConsumer = func(stream, name string) (string, error) { - before, after, err := js.ResetConsumer(stream, name) - if err != nil { - return "", err - } - return "it was " + strconv.FormatUint(before.Pending, 10) + " behind; " + strconv.FormatUint(after.Pending, 10) + - " pending now", nil - } - h.tick(ctx) - if acts := told.said(); len(acts) != 1 || acts[0].Healer != "H4" || acts[0].Outcome != inventory.HealActed { - t.Fatalf("said %+v", acts) - } - if found := probe(); len(found) != 0 { - t.Fatalf("after the reset the probe still finds %+v", found) - } - if _, still, _ := conditionsFrom.Get(ctx, key); still { - t.Fatal("the probe's next run did not clear what H4 repaired") - } - // What comes next still reaches the controller. - for len(events) > 0 { - <-events - } - if _, err := js.Context().Publish(followed, []byte(`{"after":"the reset"}`)); err != nil { - t.Fatal(err) - } - deadline := time.After(10 * time.Second) - for { - select { - case m := <-events: - if strings.Contains(string(m.Data), "after") { - return - } - _ = m.Ack() - case <-deadline: - t.Fatal("the controller's subscription heard nothing after its consumer was reset: it would be deaf") - } - } -} // **H1's question reaches the machine**, on the subject its grant lets it hear and nothing else. -func TestNatsAskToReportReachesTheMachine(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - asked, err := conn.SubscribeSync(broker.AskReportSubject("laptop")) - if err != nil { - t.Fatal(err) - } - if err := askToReport(t.Context(), conn, "laptop"); err != nil { - t.Fatal(err) - } - msg, err := asked.NextMsg(5 * time.Second) - if err != nil || !strings.Contains(string(msg.Data), "healer H1") { - t.Fatalf("the machine heard %v, %v", msg, err) - } - perms, err := broker.PermissionsFor(broker.Principal{Kind: broker.KindNode, Node: "laptop", PasswordHash: "x"}) - if err != nil || !slices.Contains(perms.Subscribe, "mesh.node.laptop.ask.report") || - slices.Contains(perms.Subscribe, "mesh.node.anchor.ask.report") { - t.Fatalf("a machine's grant for the question: %v %v", perms.Subscribe, err) - } -} diff --git a/cmd/mesh-controller/held_back_integration_test.go b/cmd/mesh-controller/held_back_integration_test.go new file mode 100644 index 00000000..e8e2eb5d --- /dev/null +++ b/cmd/mesh-controller/held_back_integration_test.go @@ -0,0 +1,182 @@ +//go:build integration + +package main + +import ( + "bytes" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := time.Now().Add(-time.Hour) + aResolver(t, open, "c1c1c1c1c1", asked) + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, "resolver"); err != nil { + t.Fatal(err) + } + } + // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. + if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + compose := composeForPush(open, gens) + d := &recordedDelivery{inv: inv} + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { + t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) + } + digestOfLaptop := func() string { + sent, err := inv.Outstanding(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + return sent + } + before := digestOfLaptop() + + // The change merges; the policy is record. `push anchor` sends the anchor... + aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) + d.declared = nil + if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + // ...and its cascade leaves the laptop, saying so. + var said bytes.Buffer + d.declared = nil + refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) + if err != nil || len(refused) != 0 { + t.Fatalf("the cascade failed: %v %v", refused, err) + } + if len(d.declared) != 0 { + t.Fatalf("the cascade sent %v a build its policy records", d.declared) + } + if digestOfLaptop() != before { + t.Fatal("the laptop's last send moved: it was sent the held build") + } + for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", + "upgrade policy records", "`push laptop` sends it"} { + if !strings.Contains(said.String(), want) { + t.Errorf("the push did not say %q:\n%s", want, said.String()) + } + } + + // Held and owed something else at once — a peer joined: still not sent, and both said: why it + // is held, and that what else it is owed waits with it. + aThirdMachine(t, open) + d.declared = nil + if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + d.declared = nil + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if len(d.declared) != 0 || digestOfLaptop() != before { + t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) + } + if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { + t.Fatalf("the push did not say both:\n%s", said.String()) + } + + // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it + // (novox/hq ADR 0236). + if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { + t.Fatal(err) + } + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { + t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) + } + // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, + compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { + t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) + } + if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { + t.Fatalf("the new send did not record the new build: %v", builds) + } +} + +func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) + if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { + t.Fatal(err) + } + gens, err := generators(ctx, open) + if err != nil { + t.Fatal(err) + } + compose := composeForPush(open, gens) + d := &recordedDelivery{inv: inv} + if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + + // `push spare`, the machine just placed: the others' peers change with it. + aThirdMachine(t, open) + if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { + t.Fatal(err) + } + d.declared = nil + var said bytes.Buffer + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { + t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) + } + if strings.Contains(said.String(), "was not sent") { + t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) + } + + // A machine whose last send was not recorded — a declaration sent by hand — is held until named. + record, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { + t.Fatal(err) + } + d.declared = nil + said.Reset() + if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { + t.Fatal(err) + } + // The anchor, the hub, may still be settling from the machine placed above; the laptop is the + // question. + if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { + t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) + } +} diff --git a/cmd/mesh-controller/held_back_test.go b/cmd/mesh-controller/held_back_test.go index f9d2ba06..2d5eb506 100644 --- a/cmd/mesh-controller/held_back_test.go +++ b/cmd/mesh-controller/held_back_test.go @@ -1,11 +1,9 @@ package main import ( - "bytes" "context" "encoding/json" "reflect" - "slices" "strings" "testing" "time" @@ -182,172 +180,6 @@ func aThirdMachine(t *testing.T, open *stores) { // The issue as it happened, against the real stores: a change merged with the policy `record`, a push // naming the anchor, and the laptop — running the same module — left with what it had, by name. -func TestANamedPushLeavesAMachineAPolicyHoldsBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := time.Now().Add(-time.Hour) - aResolver(t, open, "c1c1c1c1c1", asked) - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - } - // Recorded by a person's choice: the default rolls out since novox/hq ADR 0236. - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{Why: "each machine checked by hand"}); err != nil { - t.Fatal(err) - } - gens, err := generators(ctx, open) - if err != nil { - t.Fatal(err) - } - compose := composeForPush(open, gens) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - if builds, known, err := inv.SentBuilds(ctx, "laptop"); err != nil || !known || builds["resolver"] != "c1c1c1c1c1" { - t.Fatalf("the send did not record the build it carried: %v %v %v", builds, known, err) - } - digestOfLaptop := func() string { - sent, err := inv.Outstanding(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - return sent - } - before := digestOfLaptop() - - // The change merges; the policy is record. `push anchor` sends the anchor... - aResolver(t, open, "c2c2c2c2c2", asked.Add(time.Minute)) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"anchor"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - // ...and its cascade leaves the laptop, saying so. - var said bytes.Buffer - d.declared = nil - refused, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true}, compose, d, "", nil, &said) - if err != nil || len(refused) != 0 { - t.Fatalf("the cascade failed: %v %v", refused, err) - } - if len(d.declared) != 0 { - t.Fatalf("the cascade sent %v a build its policy records", d.declared) - } - if digestOfLaptop() != before { - t.Fatal("the laptop's last send moved: it was sent the held build") - } - for _, want := range []string{"laptop is behind and was not sent", "resolver would move from c1c1c1c1 to c2c2c2c2", - "upgrade policy records", "`push laptop` sends it"} { - if !strings.Contains(said.String(), want) { - t.Errorf("the push did not say %q:\n%s", want, said.String()) - } - } - - // Held and owed something else at once — a peer joined: still not sent, and both said: why it - // is held, and that what else it is owed waits with it. - aThirdMachine(t, open) - d.declared = nil - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || digestOfLaptop() != before { - t.Fatalf("a held machine owed a consequence was sent: %v", d.declared) - } - if !strings.Contains(said.String(), "resolver would move") || !strings.Contains(said.String(), "anything else it is owed") { - t.Fatalf("the push did not say both:\n%s", said.String()) - } - - // A policy that rolls out, and a build no gate has seen: still held — a cascade does not judge it - // (novox/hq ADR 0236). - if err := inv.SetUpgradeOf(ctx, "resolver", inventory.Upgrade{RollOut: true}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if len(d.declared) != 0 || !strings.Contains(said.String(), "has passed no gate yet") { - t.Fatalf("a cascade carried a build no gate has seen: %v\n%s", d.declared, said.String()) - } - // Once it passed a gate on some machine, the laptop is a consequence like any other, and sent. - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: "build-c2", Module: "resolver", Commit: "c2c2c2c2c2", - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"anchor": true, "spare": true}, - compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"laptop"}) || digestOfLaptop() == before { - t.Fatalf("a rolled-out upgrade's machine was not sent: %v\n%s", d.declared, said.String()) - } - if builds, _, _ := inv.SentBuilds(ctx, "laptop"); builds["resolver"] != "c2c2c2c2c2" { - t.Fatalf("the new send did not record the new build: %v", builds) - } -} // Issue 057's case is unchanged: a machine whose builds are all current and whose declaration moved // for another reason is sent by a push that names someone else. -func TestANamedPushStillSendsAConsequenceNothingHolds(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - aResolver(t, open, "c1c1c1c1c1", time.Now().Add(-time.Hour)) - if _, err := inv.Assign(ctx, "laptop", "resolver"); err != nil { - t.Fatal(err) - } - gens, err := generators(ctx, open) - if err != nil { - t.Fatal(err) - } - compose := composeForPush(open, gens) - d := &recordedDelivery{inv: inv} - if _, err := sendRound(ctx, open, []string{"anchor", "laptop"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - - // `push spare`, the machine just placed: the others' peers change with it. - aThirdMachine(t, open) - if _, err := sendRound(ctx, open, []string{"spare"}, compose, d, "", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - var said bytes.Buffer - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(d.declared, []string{"anchor", "laptop"}) { - t.Fatalf("a consequence nothing holds was not sent: %v\n%s", d.declared, said.String()) - } - if strings.Contains(said.String(), "was not sent") { - t.Fatalf("a machine nothing holds was said to be held:\n%s", said.String()) - } - - // A machine whose last send was not recorded — a declaration sent by hand — is held until named. - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "sent-by-hand", nil); err != nil { - t.Fatal(err) - } - d.declared = nil - said.Reset() - if _, err := flushBehind(ctx, open, mustNodes(t, open), map[string]bool{"spare": true}, compose, d, "", nil, &said); err != nil { - t.Fatal(err) - } - // The anchor, the hub, may still be settling from the machine placed above; the laptop is the - // question. - if slices.Contains(d.declared, "laptop") || !strings.Contains(said.String(), "laptop is behind and was not sent") { - t.Fatalf("a machine whose last send is not known was sent: %v\n%s", d.declared, said.String()) - } -} diff --git a/cmd/mesh-controller/hold_test.go b/cmd/mesh-controller/hold_test.go index 8301b42c..818247f7 100644 --- a/cmd/mesh-controller/hold_test.go +++ b/cmd/mesh-controller/hold_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/holdings_test.go b/cmd/mesh-controller/holdings_test.go index 83267096..efab46cd 100644 --- a/cmd/mesh-controller/holdings_test.go +++ b/cmd/mesh-controller/holdings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/hosts_behind_integration_test.go b/cmd/mesh-controller/hosts_behind_integration_test.go new file mode 100644 index 00000000..4bced031 --- /dev/null +++ b/cmd/mesh-controller/hosts_behind_integration_test.go @@ -0,0 +1,42 @@ +//go:build integration + +package main + +import "testing" + +func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) { + // The machine has sent this since ADR 0141 and the controller's own copy of the report did not + // have the field, so it was unmarshalled into nothing. End to end through the store, because the + // fault was a field that existed on one side of the wire only. + open := aMesh(t) + record, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if record.HostVersion != "" { + t.Fatalf("a machine that never reported one has host version %q", record.HostVersion) + } + if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil { + t.Fatal(err) + } + again, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if again.HostVersion != "ced54d4" { + t.Fatalf("the reported host version read back as %q", again.HostVersion) + } + // An empty report never clears what a machine last said: a bare word that the node is there says + // nothing about its host. + if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil { + t.Fatal(err) + } + kept, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if kept.HostVersion != "ced54d4" { + t.Fatalf("a report carrying no host version cleared what the machine had said: %q", + kept.HostVersion) + } +} diff --git a/cmd/mesh-controller/hosts_behind_test.go b/cmd/mesh-controller/hosts_behind_test.go index 75634937..5fb0f822 100644 --- a/cmd/mesh-controller/hosts_behind_test.go +++ b/cmd/mesh-controller/hosts_behind_test.go @@ -72,40 +72,3 @@ func TestAMeshWhereNothingReportedAHostStatesNoDisagreement(t *testing.T) { t.Fatalf("a mesh told no host version reported a split: %v", split) } } - -func TestAReportedHostVersionIsKeptAndReadBack(t *testing.T) { - // The machine has sent this since ADR 0141 and the controller's own copy of the report did not - // have the field, so it was unmarshalled into nothing. End to end through the store, because the - // fault was a field that existed on one side of the wire only. - open := aMesh(t) - record, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if record.HostVersion != "" { - t.Fatalf("a machine that never reported one has host version %q", record.HostVersion) - } - if err := open.inventory.RecordHostVersion(t.Context(), record.ID, "ced54d4"); err != nil { - t.Fatal(err) - } - again, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if again.HostVersion != "ced54d4" { - t.Fatalf("the reported host version read back as %q", again.HostVersion) - } - // An empty report never clears what a machine last said: a bare word that the node is there says - // nothing about its host. - if err := open.inventory.RecordHostVersion(t.Context(), record.ID, " "); err != nil { - t.Fatal(err) - } - kept, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if kept.HostVersion != "ced54d4" { - t.Fatalf("a report carrying no host version cleared what the machine had said: %q", - kept.HostVersion) - } -} diff --git a/cmd/mesh-controller/identity_bound_integration_test.go b/cmd/mesh-controller/identity_bound_integration_test.go new file mode 100644 index 00000000..8195ab04 --- /dev/null +++ b/cmd/mesh-controller/identity_bound_integration_test.go @@ -0,0 +1,107 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "objects", Version: "1", + Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh, + Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}}, + Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}) + register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", + Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) + register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", + Requires: []string{"wildcard-resolution"}}) + register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}}) + register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}}) + for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, + {"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + + // The consumer's machine resolves, and says which of its modules no provider will grant. + consumer, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + over := consumer.Overflowing() + if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" { + t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over) + } + + // The provider's machine composes. Under ADR 0049's one bound this was a refusal naming + // networkmanager, and no push to the provider could go through. + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + declared, err := declarationFor(ctx, open, "anchor", plan, settings) + if err != nil { + t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err) + } + if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" { + t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld) + } + grants, _, _, err := grantsFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + var keyless bool + for _, g := range grants { + keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager" + } + if !keyless { + t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants) + } + var granted []string + for _, c := range declared.Received["objects"]["s3-bucket"] { + granted = append(granted, c.From) + } + if strings.Join(granted, ",") != "files" { + t.Fatalf("the object store grants %v; files and only files fit", granted) + } + said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) + if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) || + !strings.Contains(said, "left out of anchor's grants") { + t.Fatalf("the push does not say whom it leaves out:\n%s", said) + } + + // And `status` names it, and does not call the mesh well while it stands. + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" { + t.Fatalf("status does not carry the overflow: %+v", asked.overflowing) + } + if asked.well() { + t.Fatal("a mesh with a consumer left out of its grants reads as well") + } + shown := printed(t, func() error { return printStatus(asked) }) + if !strings.Contains(shown, "identified too long for a provision they require") || + !strings.Contains(shown, "mesh_laptop_photoalbum") { + t.Fatalf("status does not say it:\n%s", shown) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Overflowing []catalogue.Overflow `json:"overflowing"` + } + if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 || + doc.Overflowing[0].Bound.Max != 20 { + t.Fatalf("the document does not carry it: %v\n%s", err, body) + } +} diff --git a/cmd/mesh-controller/identity_bound_test.go b/cmd/mesh-controller/identity_bound_test.go index 25745587..a3160af9 100644 --- a/cmd/mesh-controller/identity_bound_test.go +++ b/cmd/mesh-controller/identity_bound_test.go @@ -2,13 +2,10 @@ package main import ( "bytes" - "encoding/json" "os" "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" ) // novox/hq ADR 0225, issue 263: a consumer's identity is bounded by the provision it requires, an @@ -53,98 +50,3 @@ func TestModuleCheckRefusesAnIdentityThatOverflowsWhatItRequires(t *testing.T) { // resolver provision, and a second consumer there overflows an object store's access key. The // provider's machine still composes; the overflowing consumer is left out of its grants and named, // by push and by `status`, and the keyless consumer is granted with its long name. -func TestAnOverflowingConsumerNeverRefusesItsProvidersMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "objects", Version: "1", - Provides: []catalogue.Offer{{Name: "s3-bucket", Scope: catalogue.ScopeMesh, - Identity: &catalogue.OfferIdentity{Max: 20, In: "an S3 access key"}}}, - Receives: map[string]string{"s3-bucket": "/var/lib/mesh/objects/mesh.json"}}) - register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", - Requires: []string{"wildcard-resolution"}}) - register(t, open, catalogue.Manifest{Module: "photoalbum", Version: "1", Requires: []string{"s3-bucket"}}) - register(t, open, catalogue.Manifest{Module: "files", Version: "1", Requires: []string{"s3-bucket"}}) - for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, - {"laptop", "networkmanager"}, {"laptop", "photoalbum"}, {"laptop", "files"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - - // The consumer's machine resolves, and says which of its modules no provider will grant. - consumer, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - over := consumer.Overflowing() - if len(over) != 1 || over[0].Module != "photoalbum" || over[0].Provision != "s3-bucket" { - t.Fatalf("the consumer's side does not name exactly photoalbum: %+v", over) - } - - // The provider's machine composes. Under ADR 0049's one bound this was a refusal naming - // networkmanager, and no push to the provider could go through. - plan, settings, err := planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - declared, err := declarationFor(ctx, open, "anchor", plan, settings) - if err != nil { - t.Fatalf("one consumer's identity refused its provider's whole machine: %v", err) - } - if len(declared.withheld) != 1 || declared.withheld[0].Identity != "mesh_laptop_photoalbum" { - t.Fatalf("the overflowing consumer is not the one withheld: %+v", declared.withheld) - } - grants, _, _, err := grantsFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - var keyless bool - for _, g := range grants { - keyless = keyless || g.Provision == "wildcard-resolution" && g.From == "networkmanager" - } - if !keyless { - t.Fatalf("networkmanager, 26 characters, is not granted the keyless resolver provision: %+v", grants) - } - var granted []string - for _, c := range declared.Received["objects"]["s3-bucket"] { - granted = append(granted, c.From) - } - if strings.Join(granted, ",") != "files" { - t.Fatalf("the object store grants %v; files and only files fit", granted) - } - said := printed(t, func() error { reportLeftOut("anchor", declared); return nil }) - if !strings.Contains(said, `photoalbum on laptop requires s3-bucket from anchor`) || - !strings.Contains(said, "left out of anchor's grants") { - t.Fatalf("the push does not say whom it leaves out:\n%s", said) - } - - // And `status` names it, and does not call the mesh well while it stands. - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if len(asked.overflowing) != 1 || asked.overflowing[0].Module != "photoalbum" { - t.Fatalf("status does not carry the overflow: %+v", asked.overflowing) - } - if asked.well() { - t.Fatal("a mesh with a consumer left out of its grants reads as well") - } - shown := printed(t, func() error { return printStatus(asked) }) - if !strings.Contains(shown, "identified too long for a provision they require") || - !strings.Contains(shown, "mesh_laptop_photoalbum") { - t.Fatalf("status does not say it:\n%s", shown) - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Overflowing []catalogue.Overflow `json:"overflowing"` - } - if err := json.Unmarshal(body, &doc); err != nil || len(doc.Overflowing) != 1 || - doc.Overflowing[0].Bound.Max != 20 { - t.Fatalf("the document does not carry it: %v\n%s", err, body) - } -} diff --git a/cmd/mesh-controller/issue201_integration_test.go b/cmd/mesh-controller/issue201_integration_test.go new file mode 100644 index 00000000..413b2f1c --- /dev/null +++ b/cmd/mesh-controller/issue201_integration_test.go @@ -0,0 +1,181 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +func core201(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + record := func(commit string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) + b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, + Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + return b + } + before := record("c1aaaaaaaaaa", old) + if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, + Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { + t.Fatal(err) + } + record("c2bbbbbbbbbb", now) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { + t.Fatal(err) + } + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + wasMoves := machineMoves + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + t.Cleanup(func() { machineMoves = wasMoves }) + return open +} + +func TestReplay201(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + + judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) + if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) + } + keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") + if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { + t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) + } + shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} + if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { + t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) + } + key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() + if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { + t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) + } + + putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) + if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) + } + + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) + if _, found, _ := k.Get(ctx, key); !found { + t.Fatal("a send that kept the newer controller cleared the condition") + } + settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) + if _, found, _ := k.Get(ctx, key); found { + t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") + } +} + +func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { + open := core201(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", + Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, + Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { + t.Fatal(err) + } + healer := withScope(ctx, sendScope{}) + if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { + t.Fatalf("the healer's send is made: %v %v", kept, err) + } + if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { + t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) + } + if open, _ := k.Open(ctx); len(open) != 0 { + t.Fatalf("a completed put-back raised a condition: %+v", open) + } +} + +func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { + open := core201(t) + ctx := t.Context() + withConditionsInMemory(t) + inv := open.inventory + old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) + for _, b := range []struct { + commit string + at time.Time + }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, + Asked: b.at}); err != nil { + t.Fatal(err) + } + } + if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { + t.Fatal(err) + } + // The send as it is made: the controller kept at c2, the app moved to the record. + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", + map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) + } + t.Cleanup(func() { sendRollout = wasSend }) + + moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) + if err != nil || len(sent) != 1 { + t.Fatalf("the gated send is made: %v %v", sent, err) + } + if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { + t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) + } + builds, _, err := inv.SentBuilds(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + g := &inventory.PlanGate{Carried: moves} + f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} + for _, m := range []string{"app", "mesh-controller"} { + if why, superseded := f.supersededOn(m, "anchor"); superseded { + t.Fatalf("the gate is superseded by its own send: %s", why) + } + } +} diff --git a/cmd/mesh-controller/issue201_test.go b/cmd/mesh-controller/issue201_test.go index 2e9c5c91..7bd7085c 100644 --- a/cmd/mesh-controller/issue201_test.go +++ b/cmd/mesh-controller/issue201_test.go @@ -1,14 +1,10 @@ package main import ( - "context" - "encoding/json" "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) @@ -82,178 +78,16 @@ func TestACoreModuleIsKeptOverAnOlderRecord(t *testing.T) { // core201 is a mesh whose control node, anchor, was sent the newer controller c2 while the mesh's record of the // controller is the build before it, c1, which passed its gate when it rolled out. -func core201(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - record := func(commit string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: "1"}) - b := inventory.Build{ID: "build-mesh-controller-" + commit, Module: "mesh-controller", Commit: commit, - Repository: "novox/mesh-controller", Manifest: manifest, Asked: asked, At: asked, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:" + commit + "0000000000000000"}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - return b - } - before := record("c1aaaaaaaaaa", old) - if err := inv.RecordGate(ctx, inventory.GateVerdict{Build: before.ID, Module: "mesh-controller", Commit: before.Commit, - Machines: []string{"anchor"}, Verdict: inventory.GatePassed}); err != nil { - t.Fatal(err) - } - record("c2bbbbbbbbbb", now) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-controller", BuiltFrom: "c2bbbbbbbbbb", Head: "c2bbbbbbbbbb", Asked: now}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "mesh-controller"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - wasMoves := machineMoves - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - t.Cleanup(func() { machineMoves = wasMoves }) - return open -} // TestReplay201 replays the race on a store: the control node runs the newer controller and the mesh's record is // the build before it. A plan's send there — the node-engine's plan of 2026-10-02 — goes, keeps the newer // controller, and says so as a condition; the gate's own put-back carries it back; a send that carries the record, // as a person's push does, clears the condition. -func TestReplay201(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - - judged := withScope(ctx, sendScope{judged: map[string]bool{"anchor": true}}) - if kept, err := ungatedIn(judged, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("a plan's send to the machine is made, the rest of it with it: %v %v", kept, err) - } - keeps, err := recordedKept(keepingRecorded(judged), open, "anchor") - if err != nil || keeps["mesh-controller"] != "c2bbbbbbbbbb" { - t.Fatalf("the send keeps the newer controller there: %v %v", keeps, err) - } - shelf := map[string]catalogue.Manifest{"mesh-controller": {Module: "mesh-controller", Version: "old"}} - if _, err := keepRecorded(keepingRecorded(judged), open, "anchor", shelf); err != nil || shelf["mesh-controller"].Version != "1" { - t.Fatalf("the send composes the kept build's manifest: %+v %v", shelf, err) - } - key := coreDowngradeObservation(lease.ComponentController, "anchor", "").Key() - if c, found, err := k.Get(ctx, key); err != nil || !found || !strings.Contains(c.Summary, "c2bbbbbb") { - t.Fatalf("the condition %s is raised, saying the build kept: %+v %v %v", key, c, found, err) - } - - putBack := keepingRecorded(withScope(ctx, sendScope{modules: map[string]bool{"mesh-controller": true}, puttingBack: true})) - if keeps, err := recordedKept(putBack, open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the gate's own put-back of the controller carries it back: %v %v", keeps, err) - } - - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c2bbbbbbbbbb"}) - if _, found, _ := k.Get(ctx, key); !found { - t.Fatal("a send that kept the newer controller cleared the condition") - } - settleCoreDowngrades(ctx, open.inventory, "anchor", map[string]string{"mesh-controller": "c1aaaaaaaaaa"}) - if _, found, _ := k.Get(ctx, key); found { - t.Fatal("a send carrying the mesh's record — a person's push — leaves the condition open") - } -} // The review of #273: a gate failed the newer controller, put the record back, and the put-back's own send was // refused. The next automatic send — a healer's, which judges nothing — completes the put-back: a build that // failed its gate is never kept. -func TestARollbackWhoseSendWasRefusedIsCompletedByTheNextSend(t *testing.T) { - open := core201(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - if err := open.inventory.RecordGate(ctx, inventory.GateVerdict{Build: "build-mesh-controller-c2bbbbbbbbbb", - Module: "mesh-controller", Commit: "c2bbbbbbbbbb", Previous: "c1aaaaaaaaaa", Machines: []string{"anchor"}, - Verdict: inventory.GateFailed, Rollback: inventory.NotRolledBack}); err != nil { - t.Fatal(err) - } - healer := withScope(ctx, sendScope{}) - if kept, err := ungatedIn(healer, open, []string{"anchor"}, ""); err != nil || len(kept) != 1 { - t.Fatalf("the healer's send is made: %v %v", kept, err) - } - if keeps, err := recordedKept(keepingRecorded(healer), open, "anchor"); err != nil || len(keeps) != 0 { - t.Fatalf("the failed controller was kept over the build put back: %v %v", keeps, err) - } - if open, _ := k.Open(ctx); len(open) != 0 { - t.Fatalf("a completed put-back raised a condition: %+v", open) - } -} // Review 91 of #273: a gated send to a machine whose controller is kept over an older record leaves the kept // controller out of what its gate judges. Judged as moving back to the record, the gate would read the send's own // kept controller as another build sent there after it (issue 352) and be superseded, judging nothing. -func TestAGateJudgesTheOtherMovesBesideAKeptCore(t *testing.T) { - open := core201(t) - ctx := t.Context() - withConditionsInMemory(t) - inv := open.inventory - old, now := time.Now().Add(-2*time.Hour), time.Now().Add(-time.Minute) - for _, b := range []struct { - commit string - at time.Time - }{{"a1aaaaaaaaaa", old}, {"a2bbbbbbbbbb", now}} { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-" + b.commit, Module: "app", Commit: b.commit, - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: b.at, At: b.at, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-" + b.commit}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: b.commit, Head: b.commit, - Asked: b.at}); err != nil { - t.Fatal(err) - } - } - if _, err := inv.Assign(ctx, "anchor", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-2", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a1aaaaaaaaaa"}); err != nil { - t.Fatal(err) - } - // The send as it is made: the controller kept at c2, the app moved to the record. - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor-3", - map[string]string{"mesh-controller": "c2bbbbbbbbbb", "app": "a2bbbbbbbbbb"}) - } - t.Cleanup(func() { sendRollout = wasSend }) - - moves, sent, _, err := gatedSend(ctx, open, "anchor", nil) - if err != nil || len(sent) != 1 { - t.Fatalf("the gated send is made: %v %v", sent, err) - } - if len(moves) != 1 || moves[0].Module != "app" || moves[0].To != "a2bbbbbbbbbb" { - t.Fatalf("the gate judges the app's move and not the kept controller: %+v", moves) - } - builds, _, err := inv.SentBuilds(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - g := &inventory.PlanGate{Carried: moves} - f := gateFacts{commits: judgedCommits(g, nil), sentBuilds: map[string]map[string]string{"anchor": builds}} - for _, m := range []string{"app", "mesh-controller"} { - if why, superseded := f.supersededOn(m, "anchor"); superseded { - t.Fatalf("the gate is superseded by its own send: %s", why) - } - } -} diff --git a/cmd/mesh-controller/issue348_integration_test.go b/cmd/mesh-controller/issue348_integration_test.go new file mode 100644 index 00000000..b2c5f29c --- /dev/null +++ b/cmd/mesh-controller/issue348_integration_test.go @@ -0,0 +1,199 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/link" +) + +func TestReplay348(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + say := func(at time.Time, n *link.NetworkHealth) { + t.Helper() + if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + network := func() (conditions.Condition, bool) { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key == "machine.anchor.network" { + return c, true + } + } + return conditions.Condition{}, false + } + + // 10:58:45 — the second failing look: raised. + say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2))) + raised, ok := network() + if !ok { + t.Fatal("the resolver refusing on the control node raised nothing") + } + time.Sleep(5 * time.Millisecond) + sent := time.Now().UTC() + time.Sleep(5 * time.Millisecond) + + // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. + say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1))) + if _, ok := network(); !ok { + t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " + + "said the fault was gone while it still refused") + } + // 11:00:23 — its second look: unhealthy again, the same raising. + say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2))) + again, ok := network() + if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 { + t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count) + } + + // The gate on the control node, for a build sent after the fault began. + open2, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + f := gateFacts{judged: true, open: open2} + if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 { + t.Fatalf("a fault from before the send held the build: %+v", w) + } + + // Decided healthy: cleared. + say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0})) + if c, ok := network(); ok { + t.Fatalf("a statement that decides the names healthy left %s open", c.Key) + } +} + +// namesRefused is the control node's names part as its node-engine said it in the outage: its own +// resolver, at its own address, refusing. +func namesRefused(state string, streak int) link.NetworkPart { + p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} + if state == link.StateUnhealthy { + p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" + p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" + p.Toward = []string{"10.77.0.1"} + } + return p +} + +func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth { + state := link.StateHealthy + for _, p := range parts { + switch { + case p.State == link.StateUnhealthy: + state = link.StateUnhealthy + case p.State == link.StateUnknown && state == link.StateHealthy: + state = link.StateUnknown + } + } + return &link.NetworkHealth{State: state, Since: h0, Parts: parts} +} + +// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared +// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review +// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings. +func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) { + since := h0 + network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition { + raised := since.Add(time.Minute) + if len(gaps) > 0 { + raised = gaps[len(gaps)-1].Reopened + } + return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}, + Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised} + } + held := func(c conditions.Condition) bool { + return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true, + open: []conditions.Condition{c}}).whole != "" + } + // The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it. + flapped := network(since.Add(-49*time.Second), + conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)}) + if held(flapped) { + t.Fatal("a fault there at the send, flapping after it, held the machine") + } + // Recovered before the send, broken again after it: the send's. + recovered := network(since.Add(-time.Hour), + conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)}) + if !held(recovered) { + t.Fatal("a machine recovered at the send and broken after it passed the gate") + } + // An older gap, before the send, and the fault there at the send: not the send's. + twice := network(since.Add(-time.Hour), + conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)}, + conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)}) + if held(twice) { + t.Fatal("a fault there at the send, with an older gap, held the machine") + } + // Raised after the send, never cleared: the send's. + if !held(network(time.Time{})) { + t.Fatal("a fault raised after the send held nothing") + } + // And a module's own, through judgeHealth. + at := since.Add(2 * time.Minute) + g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", + Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, + served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, + open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ + Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing", + First: since.Add(-time.Hour), Raised: since.Add(time.Minute), + Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}} + if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") { + t.Fatalf("a module's own fault there at the send: %s", why) + } + g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second) + if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") { + t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why) + } +} + +// Only an undecided part holds a condition that names it; a condition about another part clears, and a +// statement unknown as a whole holds every part (review of PR 179, A4). Pure. +func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}, + inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}), + "laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}), + "spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}), + "other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}), + }) + u := undecidedParts(f) + if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] || + !u["other"][link.PartTunnel] || u["other"]["*"] { + t.Fatalf("undecided: %v", u) + } + about := func(machine, said string, also ...string) conditions.Condition { + return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, + Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}} + } + for _, c := range []struct { + c conditions.Condition + held bool + }{ + {about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true}, + {about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false}, + {about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false}, + {about("spare", "route since …: no default route"), true}, + {about("hub", "anchor: names: refused", "anchor"), true}, + {about("hub", "anchor: tunnel: no handshake", "anchor"), false}, + } { + if got := heldUndecided(c.c, u); got != c.held { + t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held) + } + } +} diff --git a/cmd/mesh-controller/issue348_test.go b/cmd/mesh-controller/issue348_test.go index 7c0bd4a7..1f8544c4 100644 --- a/cmd/mesh-controller/issue348_test.go +++ b/cmd/mesh-controller/issue348_test.go @@ -1,15 +1,9 @@ package main import ( - "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 348: on 2026-10-09 the control node's resolver stopped answering on its private address @@ -20,192 +14,9 @@ import ( // builds failed their gate at 11:10 with "raised since it was sent" and were put back, for a fault that // began before they were sent. -// namesRefused is the control node's names part as its node-engine said it in the outage: its own -// resolver, at its own address, refusing. -func namesRefused(state string, streak int) link.NetworkPart { - p := link.NetworkPart{Part: link.PartNames, State: state, Since: h0, Streak: streak} - if state == link.StateUnhealthy { - p.Reason = "1 of its 2 resolvers do not answer as the mesh's do" - p.Said = "10.77.0.1 — anchor.internal (IPv4): read udp 10.77.0.1:35244->10.77.0.1:53: read: connection refused" - p.Toward = []string{"10.77.0.1"} - } - return p -} - -func networkSaying(parts ...link.NetworkPart) *link.NetworkHealth { - state := link.StateHealthy - for _, p := range parts { - switch { - case p.State == link.StateUnhealthy: - state = link.StateUnhealthy - case p.State == link.StateUnknown && state == link.StateHealthy: - state = link.StateUnknown - } - } - return &link.NetworkHealth{State: state, Since: h0, Parts: parts} -} - // TestReplay348 replays the statements of the outage: the condition raised before the send is not // cleared by the restarted engine's first, undecided statement, and the gate does not count it against // the builds sent after it began. -func TestReplay348(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "anchor", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - network := func() (conditions.Condition, bool) { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key == "machine.anchor.network" { - return c, true - } - } - return conditions.Condition{}, false - } - - // 10:58:45 — the second failing look: raised. - say(h0, networkSaying(namesRefused(link.StateUnhealthy, 2))) - raised, ok := network() - if !ok { - t.Fatal("the resolver refusing on the control node raised nothing") - } - time.Sleep(5 * time.Millisecond) - sent := time.Now().UTC() - time.Sleep(5 * time.Millisecond) - - // 10:59:42 — the restarted engine's first statement: one look failed, a second decides. - say(h0.Add(time.Minute), networkSaying(namesRefused(link.StateUnknown, 1))) - if _, ok := network(); !ok { - t.Fatal("a statement that judged nothing yet cleared the condition: the restarted engine's first look " + - "said the fault was gone while it still refused") - } - // 11:00:23 — its second look: unhealthy again, the same raising. - say(h0.Add(2*time.Minute), networkSaying(namesRefused(link.StateUnhealthy, 2))) - again, ok := network() - if !ok || !again.Raised.Equal(raised.Raised) || again.Count != 1 { - t.Fatalf("the same raising was not kept: raised %s (first %s), count %d", again.Raised, raised.Raised, again.Count) - } - - // The gate on the control node, for a build sent after the fault began. - open2, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - f := gateFacts{judged: true, open: open2} - if w := aboutTheMachine("anchor", []string{"mesh-host"}, sent, f); w.whole != "" || len(w.on) != 0 { - t.Fatalf("a fault from before the send held the build: %+v", w) - } - - // Decided healthy: cleared. - say(h0.Add(3*time.Minute), networkSaying(link.NetworkPart{Part: link.PartNames, State: link.StateHealthy, Since: h0})) - if c, ok := network(); ok { - t.Fatalf("a statement that decides the names healthy left %s open", c.Key) - } -} - -// A fault there at the send, cleared and reopened after it, is not raised since the send; one that cleared -// before the send and came back after it is — a send that breaks a recovered machine fails its gate (review -// of mesh-controller PR 179, A2). Read through OpenAt, by both of the gate's readings. -func TestAFaultThatFlappedAfterTheSendIsNotTheSendsAndOneThatRecoveredBeforeItIs(t *testing.T) { - since := h0 - network := func(first time.Time, gaps ...conditions.Gap) conditions.Condition { - raised := since.Add(time.Minute) - if len(gaps) > 0 { - raised = gaps[len(gaps)-1].Reopened - } - return conditions.Condition{Key: "machine.anchor.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor"}, - Summary: "anchor's network is not healthy", Source: sourceNetwork, First: first, Gaps: gaps, Raised: raised} - } - held := func(c conditions.Condition) bool { - return aboutTheMachine("anchor", []string{"mesh-controller"}, since, gateFacts{judged: true, - open: []conditions.Condition{c}}).whole != "" - } - // The day's case: raised before the send, cleared 8 s after it, reopened 49 s after it. - flapped := network(since.Add(-49*time.Second), - conditions.Gap{Cleared: since.Add(8 * time.Second), Reopened: since.Add(49 * time.Second)}) - if held(flapped) { - t.Fatal("a fault there at the send, flapping after it, held the machine") - } - // Recovered before the send, broken again after it: the send's. - recovered := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-30 * time.Second), Reopened: since.Add(20 * time.Second)}) - if !held(recovered) { - t.Fatal("a machine recovered at the send and broken after it passed the gate") - } - // An older gap, before the send, and the fault there at the send: not the send's. - twice := network(since.Add(-time.Hour), - conditions.Gap{Cleared: since.Add(-50 * time.Minute), Reopened: since.Add(-45 * time.Minute)}, - conditions.Gap{Cleared: since.Add(10 * time.Second), Reopened: since.Add(30 * time.Second)}) - if held(twice) { - t.Fatal("a fault there at the send, with an older gap, held the machine") - } - // Raised after the send, never cleared: the send's. - if !held(network(time.Time{})) { - t.Fatal("a fault raised after the send held nothing") - } - // And a module's own, through judgeHealth. - at := since.Add(2 * time.Minute) - g := gateFacts{judged: true, now: at, reports: map[string]inventory.Reported{"anchor": {Node: "anchor", - Outcome: inventory.OutcomeApplied, At: &at, Current: true}}, engines: map[string]string{}, - served: map[string]served{}, rolledBack: map[string][]lease.Rollback{}, - open: []conditions.Condition{{Key: "provider.app.anchor.x.failing", Subject: conditions.Subject{ - Scope: conditions.ScopeProvider, ID: "app.anchor.x", Machine: "anchor"}, Summary: "failing", - First: since.Add(-time.Hour), Raised: since.Add(time.Minute), - Gaps: []conditions.Gap{{Cleared: since.Add(5 * time.Second), Reopened: since.Add(time.Minute)}}}}} - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault there at the send: %s", why) - } - g.open[0].Gaps[0].Cleared = since.Add(-5 * time.Second) - if _, why := judgeHealth("app", "", catalogue.Manifest{Module: "app"}, "anchor", since, g); !strings.HasPrefix(why, "raised since it was sent") { - t.Fatalf("a module's own fault, recovered at the send and back after it, was not counted: %s", why) - } -} - -// Only an undecided part holds a condition that names it; a condition about another part clears, and a -// statement unknown as a whole holds every part (review of PR 179, A4). Pure. -func TestAnUndecidedPartHoldsOnlyWhatNamesIt(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnknown, inventory.NetworkPart{Part: link.PartNames, State: link.StateUnknown, Streak: 1}, - inventory.NetworkPart{Part: link.PartRoute, State: link.StateHealthy}), - "laptop": aNetwork(link.StateHealthy, inventory.NetworkPart{Part: link.PartNames, State: link.StateHealthy}), - "spare": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateHealthy}), - "other": aNetwork(link.StateStarting, inventory.NetworkPart{Part: link.PartTunnel, State: link.StateStarting}), - }) - u := undecidedParts(f) - if !u["anchor"][link.PartNames] || u["anchor"][link.PartRoute] || u["laptop"] != nil || !u["spare"]["*"] || - !u["other"][link.PartTunnel] || u["other"]["*"] { - t.Fatalf("undecided: %v", u) - } - about := func(machine, said string, also ...string) conditions.Condition { - return conditions.Condition{Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: machine, - Machine: machine, Also: also}, Evidence: []conditions.Evidence{{Said: said}}} - } - for _, c := range []struct { - c conditions.Condition - held bool - }{ - {about("anchor", "names since 2026-10-09 10:58:45 UTC: 10.77.0.1 — refused"), true}, - {about("anchor", "route since 2026-10-09 10:58:45 UTC: no default route"), false}, - {about("laptop", "names since 2026-10-09 10:58:45 UTC: refused"), false}, - {about("spare", "route since …: no default route"), true}, - {about("hub", "anchor: names: refused", "anchor"), true}, - {about("hub", "anchor: tunnel: no handshake", "anchor"), false}, - } { - if got := heldUndecided(c.c, u); got != c.held { - t.Errorf("%s %q held %v, want %v", c.c.Subject.Machine, c.c.Evidence[0].Said, got, c.held) - } - } -} // A release walks its modules without a record per module: D10 counts what its tier names as rolling, // so the node-engine a release walks is not "behind, and no plan is rolling it out" on its first machine. diff --git a/cmd/mesh-controller/issue352_integration_test.go b/cmd/mesh-controller/issue352_integration_test.go new file mode 100644 index 00000000..7f604afe --- /dev/null +++ b/cmd/mesh-controller/issue352_integration_test.go @@ -0,0 +1,227 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestReplay352(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) + t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) +} + +func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied + // 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on. + carried, _, _ := inv.SentBuilds(ctx, "anchor") + if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil { + t.Fatal(err) + } + reports, _ := inv.LastReports(ctx) + for _, r := range reports { + if r.Node == "anchor" && r.Current { + t.Fatal("the fixture's newer send reads as reported") + } + } + gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here + for i := 0; i < 4; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") { + t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note) + } + if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") { + t.Fatalf("the gate does not keep what it sent: %+v", g) + } + if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("app was put back to %s", current["app"].Commit) + } +} + +func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + advancePlans(ctx, b.open) // the release judges app c2 on anchor + _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) + if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { + t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) + } + if len(b.sent) != 1 { + t.Fatalf("sent %v", b.sent) + } +} + +func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + advancePlans(ctx, g.open) // anchor is sent app c2 first + if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, g.open) + p := g.plan(t) + if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { + t.Fatalf("the plan is %s: %s", p.State, p.Note) + } + // Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no + // rollback (a release may walk what the other send left waiting on anchor; that is not a put-back). + if r := p.Modules["app"].Gate.Rollback; r != "" { + t.Fatalf("a superseded judging made a rollback: %q", r) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("app was put back to %s", current["app"].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-2"); failed { + t.Fatal("a superseded build was marked failed") + } +} + +func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) + // Another send moves app on anchor to a build this gate does not judge. + carried, _, _ := inv.SentBuilds(ctx, "anchor") + carried["app"] = "c3" + if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, b.open) + p := b.release(t) + if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { + t.Fatalf("the release is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent) + } + if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found { + t.Fatal("a superseded judging kept a verdict") + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("app was put back to %s", current["app"].Commit) + } +} + +func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + keeper, _ := withConditionsInMemory(t) + told := &conditions.Told{} + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: keeper, teller: told} + t.Cleanup(func() { doctorFrom = was }) + wasSend := sendRollout + var sent [][]string + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + sent = append(sent, names) + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + build := func(id, commit, digest string, asked time.Time) inventory.Build { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit}) + b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".", + Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle, + Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit}, + inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + return b + } + previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour)) + failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute)) + if versionOfBuild(previous) != strings.Repeat("1", 12) { + t.Fatalf("the build's version is %q", versionOfBuild(previous)) + } + applied, err := inv.SchemaApplied(ctx) + if err != nil || applied < 87 { + t.Fatalf("the store's schema reaches %d (%v)", applied, err) + } + // The build before never recorded what it reads: not proved, refused. + if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") { + t.Fatalf("an unknown reach: %v %q", ok, why) + } + // It reads less than the store has: refused, naming both. + if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil { + t.Fatal(err) + } + if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") { + t.Fatalf("a reach behind the store: %v %q", ok, why) + } + // Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent. + at := time.Now().Add(-5 * time.Minute) + state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at} + p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at, + State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}} + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply") + if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") { + t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note) + } + if len(sent) != 0 { + t.Fatalf("sent %v: nothing is put back", sent) + } + if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit { + t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit) + } + if marked, _ := inv.GateFailed(ctx, failed.ID); !marked { + t.Fatal("the failed build is not marked failed at its gate") + } + open2, _ := keeper.Open(ctx) + var found bool + for _, c := range open2 { + if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") { + found = true + } + } + if !found { + t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2) + } + // Reaching the store: allowed. + if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil { + t.Fatal(err) + } + if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { + t.Fatalf("a build that reads the whole schema was refused: %q", why) + } + // A build with no bundle to know it by: refused. + if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") { + t.Fatalf("a build without a bundle: %v %q", ok, why) + } +} diff --git a/cmd/mesh-controller/issue352_test.go b/cmd/mesh-controller/issue352_test.go index e3a6f7e1..3baa9b57 100644 --- a/cmd/mesh-controller/issue352_test.go +++ b/cmd/mesh-controller/issue352_test.go @@ -1,16 +1,11 @@ package main import ( - "context" - "encoding/json" - "errors" - "reflect" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" ) @@ -24,121 +19,10 @@ import ( // TestReplay352 replays the walk on the backlog fixture: the release sends anchor and anchor reports; // another send reaches anchor, unreported; the gate still passes. And a send that moves a judged module // to another build supersedes the judging: no verdict, nothing put back. -func TestReplay352(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - t.Run("a newer send to the judged machine does not unreport the gate's", testANewerSendDoesNotUnreportTheGatesOwn) - t.Run("a send that moves the module supersedes the judging", testASendThatMovesTheModuleSupersedesTheJudging) -} - -func testANewerSendDoesNotUnreportTheGatesOwn(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // anchor is sent, and the fixture reports it applied - // 16:31:37 — another plan sends anchor a newer declaration, which it has not reported on. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-newer", carried); err != nil { - t.Fatal(err) - } - reports, _ := inv.LastReports(ctx) - for _, r := range reports { - if r.Node == "anchor" && r.Current { - t.Fatal("the fixture's newer send reads as reported") - } - } - gateEvery, gateBound = 0, 0 // past the bound at once: before the fix, "has not reported" fails it here - for i := 0; i < 4; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State == inventory.PlanFailed || strings.Contains(p.Note, "has not reported") { - t.Fatalf("the release failed on the newer send: %s %s", p.State, p.Note) - } - if g := p.Release.Gate; g != nil && (g.Sent == nil || g.Sent["anchor"].Digest == "") { - t.Fatalf("the gate does not keep what it sent: %+v", g) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app's pass on anchor was not kept: %+v %v %v", v, found, err) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} // A plan's own first send waits while a release judges the same module on that machine with another build. -func TestAPlansFirstSendWaitsForAReleaseJudgingTheModuleThere(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - advancePlans(ctx, b.open) // the release judges app c2 on anchor - _, _, _, err := gatedSend(ctx, b.open, "anchor", []inventory.CarriedMove{{Module: "app", Node: "anchor", From: "c2", To: "c3", Build: "build-app-c3"}}) - if !errors.Is(err, errWalkedElsewhere) || !strings.Contains(err.Error(), "release-") { - t.Fatalf("a newer build of a judged module was sent under the release's gate: %v", err) - } - if len(b.sent) != 1 { - t.Fatalf("sent %v", b.sent) - } -} // A merge plan's judging is superseded the same way: another send moved its module on the first machine. -func TestAPlansJudgingIsSupersededByASendThatMovesItsModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - advancePlans(ctx, g.open) // anchor is sent app c2 first - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-c3", map[string]string{"app": "c3"}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, g.open) - p := g.plan(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - // Nothing put back: the registered build stands, the build is not marked, and the plan's gate made no - // rollback (a release may walk what the other send left waiting on anchor; that is not a put-back). - if r := p.Modules["app"].Gate.Rollback; r != "" { - t.Fatalf("a superseded judging made a rollback: %q", r) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a superseded build was marked failed") - } -} - -func testASendThatMovesTheModuleSupersedesTheJudging(t *testing.T) { - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) - // Another send moves app on anchor to a build this gate does not judge. - carried, _, _ := inv.SentBuilds(ctx, "anchor") - carried["app"] = "c3" - if err := inv.RecordSent(ctx, nodeID(t, b.open, "anchor"), "d-anchor-c3", carried); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, b.open) - p := b.release(t) - if p.State != inventory.PlanSuperseded || !strings.Contains(p.Note, "superseded") || !strings.Contains(p.Note, "c3") { - t.Fatalf("the release is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(b.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: a superseded judging puts nothing back", b.sent) - } - if _, found, _ := inv.GateOf(ctx, "build-app-c2"); found { - t.Fatal("a superseded judging kept a verdict") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("app was put back to %s", current["app"].Commit) - } -} // A report is on the gate's own send: the declaration itself, or one sequenced after it; a gate kept // without its send reads the report against the send made last, as before. Pure. @@ -254,101 +138,6 @@ func TestAWalkWaitsForAJudgingOfTheSameModuleOnThatMachine(t *testing.T) { // The controller is never put back to a build that reaches less of the store's schema than the store // has, or to one that never said what it reaches: the current build is kept, and the condition says so. -func TestTheControllerIsNotPutBackToABuildOlderThanTheStoresSchema(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - keeper, _ := withConditionsInMemory(t) - told := &conditions.Told{} - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: keeper, teller: told} - t.Cleanup(func() { doctorFrom = was }) - wasSend := sendRollout - var sent [][]string - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, names) - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - build := func(id, commit, digest string, asked time.Time) inventory.Build { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "mesh-controller", Version: commit}) - b := inventory.Build{ID: id, Module: "mesh-controller", Commit: commit, Repository: "novox/mesh-controller", Path: ".", - Manifest: manifest, Asked: asked, At: asked, Made: []inventory.Artifact{{Name: "controller", Kind: catalogue.ArtifactBundle, - Reference: "mesh-artifact://mesh-controller/controller/blobs/sha256:" + digest}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-controller", Version: commit}, - inventory.Source{Repository: "novox/mesh-controller", Seat: "git", Path: ".", BuiltFrom: commit, Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - previous := build("build-old", "ef26d4cb", strings.Repeat("1", 64), time.Now().Add(-2*time.Hour)) - failed := build("build-new", "e6b00e2e", strings.Repeat("2", 64), time.Now().Add(-time.Minute)) - if versionOfBuild(previous) != strings.Repeat("1", 12) { - t.Fatalf("the build's version is %q", versionOfBuild(previous)) - } - applied, err := inv.SchemaApplied(ctx) - if err != nil || applied < 87 { - t.Fatalf("the store's schema reaches %d (%v)", applied, err) - } - // The build before never recorded what it reads: not proved, refused. - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "never recorded") { - t.Fatalf("an unknown reach: %v %q", ok, why) - } - // It reads less than the store has: refused, naming both. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied-1); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); ok || !strings.Contains(why, "is at") { - t.Fatalf("a reach behind the store: %v %q", ok, why) - } - // Through the gate: the failed build is marked, nothing is put back, nothing is sent, the condition is urgent. - at := time.Now().Add(-5 * time.Minute) - state := &inventory.PlanModule{Build: failed.ID, Commit: failed.Commit, Previous: previous.Commit, First: []string{"anchor"}, FirstAt: &at} - p := inventory.Plan{ID: "plan-352", Repository: "novox/mesh-controller", Branch: "main", Commit: failed.Commit, Created: at, - State: inventory.PlanRolling, Tiers: [][]string{{"mesh-controller"}}, Modules: map[string]*inventory.PlanModule{"mesh-controller": state}} - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - gateFailed(ctx, open, &p, "mesh-controller", state, []string{"anchor"}, "not healthy within 10m0s of its apply") - if state.Gate.Rollback != inventory.NotRolledBack || !strings.Contains(p.Note, "NOT put back") || !strings.Contains(p.Note, "the current build is kept") { - t.Fatalf("rollback %q: %s", state.Gate.Rollback, p.Note) - } - if len(sent) != 0 { - t.Fatalf("sent %v: nothing is put back", sent) - } - if current, _ := inv.CurrentBuilds(ctx); current["mesh-controller"].Commit != failed.Commit { - t.Fatalf("the module was put back to %s", current["mesh-controller"].Commit) - } - if marked, _ := inv.GateFailed(ctx, failed.ID); !marked { - t.Fatal("the failed build is not marked failed at its gate") - } - open2, _ := keeper.Open(ctx) - var found bool - for _, c := range open2 { - if c.Kind == kindRollbackFailed && c.Severity == conditions.Urgent && strings.Contains(c.Summary, "current build is kept") { - found = true - } - } - if !found { - t.Fatalf("no urgent rollback-failed condition saying the current build is kept: %+v", open2) - } - // Reaching the store: allowed. - if err := inv.RecordSchemaReach(ctx, versionOfBuild(previous), applied); err != nil { - t.Fatal(err) - } - if why, ok := controllerSchemaAllows(ctx, inv, previous); !ok { - t.Fatalf("a build that reads the whole schema was refused: %q", why) - } - // A build with no bundle to know it by: refused. - if why, ok := controllerSchemaAllows(ctx, inv, inventory.Build{Commit: "x"}); ok || !strings.Contains(why, "names no bundle") { - t.Fatalf("a build without a bundle: %v %q", ok, why) - } -} // The lease's holder names the build the declaration told it it is, and the version stamp only without one. func TestTheHolderNamesTheBuildTheDeclarationToldIt(t *testing.T) { diff --git a/cmd/mesh-controller/issue459_integration_test.go b/cmd/mesh-controller/issue459_integration_test.go new file mode 100644 index 00000000..14c9acc0 --- /dev/null +++ b/cmd/mesh-controller/issue459_integration_test.go @@ -0,0 +1,293 @@ +//go:build integration + +package main + +import ( + "errors" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 || !ws[0].Open() { + t.Fatalf("no open walk to stop: %+v", ws) + } + stopped := ws[0] + p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || + !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { + t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) + } + if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { + t.Fatalf("the stop answered %+v", said) + } + again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) + if err != nil || len(again) != 1 || again[0].Commit != one.Commit { + t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) + } + if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) + } + cutAt(t, open, t0.Add(4*time.Minute)) + fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) + if err != nil || fresh.Batch() || !fresh.Open() { + t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) + } + if _, in := fresh.Modules["one"]; !in { + t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) + } + if _, in := fresh.Modules["app"]; in { + t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) + } + if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { + t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) + } + // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). + cutAt(t, open, t0.Add(5*time.Minute)) + if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { + t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) + } +} + +func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + first := catalogueMerge("cat1aaaaaaaa", "app", t0) + later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) + one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head) + hear(t, open, first, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + hear(t, open, later, t0.Add(11*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + + if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { + t.Fatal("a stop leaving out a merge the walk does not answer was taken") + } + if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { + t.Fatalf("a refused stop ended the walk: %s", p.State) + } + + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) + if err != nil { + t.Fatal(err) + } + want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, + {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} + if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { + t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) + } + if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { + t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) + } + again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) + if len(again) != 1 || again[0].Commit != one.Commit { + t.Fatalf("walked again: %+v; want the other repository's merge alone", again) + } + + // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. + cutAt(t, open, t0.Add(4*time.Minute)) + fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) + q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", + []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) + if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { + t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) + } + if _, bs := walks(t, open); len(bs) != 0 { + t.Fatalf("a batch of nothing was made: %+v", bs) + } +} + +func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) + hear(t, open, one2, t0.Add(3*time.Minute)) + _, bs := walks(t, open) + if len(bs) != 1 { + t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) + } + waiting := bs[0] + p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { + t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) + } + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batches; want the open one alone", len(bs)) + } + b, _ := open.inventory.PlanByID(ctx, waiting.ID) + if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { + t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) + } + // Its window closed by the stop, not by the clock: cut at once. + cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) + fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) + if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { + t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) + } + var answered []string + for _, m := range fresh.Delivery.Merges { + answered = append(answered, m.Commit+">"+m.Carried) + } + if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { + t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) + } +} + +func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { + open := windowed(t) + groupsWaiting(t, "pr/cat1aaaaaaaa") + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one1, t0.Add(6*time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + walk := ws[0] + sent := t0.Add(150 * time.Second) + for name, m := range walk.Modules { + if name == "app" || name == "one" { + m.First, m.FirstAt = []string{"anchor"}, &sent + } + } + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) + hear(t, open, cat2, t0.Add(3*time.Minute)) + _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { + t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) + } + p, _ := open.inventory.PlanByID(ctx, walk.ID) + if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { + t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) + } + if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { + t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) + } + if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { + t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) + } +} + +func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + cat := catalogueMerge("cat1aaaaaaaa", "app", t0) + one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) + hear(t, open, cat, t0.Add(time.Second)) + hear(t, open, one, t0.Add(6*time.Second)) + _, bs := walks(t, open) + b := bs[0] + r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", + []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) + if err != nil { + t.Fatal(err) + } + if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { + t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) + } + if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { + t.Fatalf("the stopped record holds %+v", left) + } + if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || + stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { + t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) + } + _ = r + cutAt(t, open, t0.Add(3*time.Minute)) + w, _ := open.inventory.PlanByID(ctx, b.ID) + if _, in := w.Modules["app"]; in || w.Batch() { + t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) + } + // A batch whose every merge is left out is stopped itself. + hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) + _, bs = walks(t, open) + last := bs[0] + if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", + []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { + t.Fatalf("a batch left with nothing: %+v %v", said, err) + } + if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { + t.Fatalf("the emptied batch is %s", p.State) + } +} + +func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + endsItsLine(t) + ctx := t.Context() + hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) + _, bs := walks(t, open) + stale := bs[0] + moved := stale + moved.Note = "written by somebody else" + if err := open.inventory.SavePlan(ctx, &moved); err != nil { + t.Fatal(err) + } + fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, + State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} + err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, + []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) + if !errors.Is(err, inventory.ErrPlanMoved) { + t.Fatalf("a stale plan in the act: %v", err) + } + if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { + t.Fatal("the new plan was written though the act was refused") + } + if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { + t.Fatalf("the merge moved to %q though the act was refused", m.Plan) + } +} diff --git a/cmd/mesh-controller/issue459_test.go b/cmd/mesh-controller/issue459_test.go index 7fec24d4..9c79df8c 100644 --- a/cmd/mesh-controller/issue459_test.go +++ b/cmd/mesh-controller/issue459_test.go @@ -2,10 +2,8 @@ package main import ( "encoding/json" - "errors" "fmt" "reflect" - "slices" "strings" "testing" "time" @@ -121,297 +119,24 @@ func TestDeliveryStopTakesTheMergesToLeaveOut(t *testing.T) { // A walk of the catalogue's merge and another repository's is stopped without the catalogue's: it ends stopped, // names the batch that walks the other merge again and the merge it left out, and that batch is cut into a walk // that builds the other repository's module alone. -func TestAStopWithoutAMergeWalksTheOthersAgain(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 || !ws[0].Open() { - t.Fatalf("no open walk to stop: %+v", ws) - } - stopped := ws[0] - p, said, err := stopWalkWithout(ctx, open.inventory, stopped.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || p.Delivery.Stopped == "" || p.Delivery.WalkedAgainBy == "" || - !reflect.DeepEqual(p.Delivery.LeftOut, []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: cat.Commit}}) { - t.Fatalf("the stopped walk reads %s %+v", p.State, p.Delivery) - } - if said.WalkedAgainBy != p.Delivery.WalkedAgainBy || !reflect.DeepEqual(said.WalkedAgain, []string{"novox/one@" + one.Commit}) { - t.Fatalf("the stop answered %+v", said) - } - again, err := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if err != nil || len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("the fresh batch answers %+v (%v); want the other repository's merge alone", again, err) - } - if left, _ := open.inventory.MergesOf(ctx, stopped.ID); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped walk still answers %+v; want the left-out merge alone", left) - } - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, err := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - if err != nil || fresh.Batch() || !fresh.Open() { - t.Fatalf("the fresh batch was not cut into a walk: %s %v", fresh.State, err) - } - if _, in := fresh.Modules["one"]; !in { - t.Fatalf("the fresh walk does not build one: %v", fresh.Modules) - } - if _, in := fresh.Modules["app"]; in { - t.Fatalf("the fresh walk builds app, whose merge was left out: %v", fresh.Modules) - } - if len(fresh.Delivery.Merges) != 1 || fresh.Delivery.Merges[0].Commit != one.Commit { - t.Fatalf("the fresh walk answers %+v", fresh.Delivery.Merges) - } - // A stopped walk starts no search of its own (ADR 0276 decision 3 is for a failed one). - cutAt(t, open, t0.Add(5*time.Minute)) - if alone, _ := open.inventory.AloneMerges(ctx); len(alone) != 0 { - t.Fatalf("a merge of the stopped walk waits to be walked alone: %+v", alone) - } -} // A later merge of the stopped merge's repository contains it: it is left out too, named with the merge it // contains, and only the other repository's merge is walked again. With nothing left to walk, the walk is only // ended. A merge the walk does not answer is refused, and nothing ends. -func TestAStopWithoutAMergeEndsTheMergesThatContainIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - first := catalogueMerge("cat1aaaaaaaa", "app", t0) - later := catalogueMerge("cat2cccccccc", "notes", t0.Add(10*time.Second)) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), first.Head) - hear(t, open, first, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - hear(t, open, later, t0.Add(11*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - - if _, _, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@ffffffffffff"}, t0.Add(3*time.Minute)); err == nil { - t.Fatal("a stop leaving out a merge the walk does not answer was taken") - } - if p, _ := open.inventory.PlanByID(ctx, walk.ID); !p.Open() { - t.Fatalf("a refused stop ended the walk: %s", p.State) - } - - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + first.Commit}, t0.Add(3*time.Minute)) - if err != nil { - t.Fatal(err) - } - want := []inventory.PlanLeftOut{{Repository: "novox/mesh-catalog", Commit: first.Commit}, - {Repository: "novox/mesh-catalog", Commit: later.Commit, Contains: first.Commit}} - if !reflect.DeepEqual(p.Delivery.LeftOut, want) || !reflect.DeepEqual(said.LeftOut, want) { - t.Fatalf("left out %+v, said %+v; want %+v", p.Delivery.LeftOut, said.LeftOut, want) - } - if !strings.Contains(p.Note, later.Commit[:8]) || !strings.Contains(p.Note, p.Delivery.WalkedAgainBy) { - t.Fatalf("the stopped walk's note does not name the merge ended with it and the fresh batch: %q", p.Note) - } - again, _ := open.inventory.MergesOf(ctx, p.Delivery.WalkedAgainBy) - if len(again) != 1 || again[0].Commit != one.Commit { - t.Fatalf("walked again: %+v; want the other repository's merge alone", again) - } - - // Nothing left once the other repository's merge is left out too: the fresh walk is stopped, and nothing more. - cutAt(t, open, t0.Add(4*time.Minute)) - fresh, _ := open.inventory.PlanByID(ctx, p.Delivery.WalkedAgainBy) - q, said, err := stopWalkWithout(ctx, open.inventory, fresh.ID, "mesh-delivery for jochen", "that one too", - []string{"novox/one@" + one.Commit}, t0.Add(5*time.Minute)) - if err != nil || q.State != inventory.PlanFailed || q.Delivery.WalkedAgainBy != "" || said.WalkedAgainBy != "" { - t.Fatalf("a stop leaving nothing to walk: %s %+v %+v %v", q.State, q.Delivery, said, err) - } - if _, bs := walks(t, open); len(bs) != 0 { - t.Fatalf("a batch of nothing was made: %+v", bs) - } -} // A merge heard while the walk ran waits in the open batch, at a later commit of the same repository. The merge // walked again joins that batch, never a batch of its own that would walk the older commit after the newer one // (review of #232): its window closes, and its walk is at the newer commit, answering both. -func TestAStopWalksTheOthersAgainInTheOpenBatch(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - one2 := repoMerge("one", "one2cccccccc", t0.Add(3*time.Minute)) - hear(t, open, one2, t0.Add(3*time.Minute)) - _, bs := walks(t, open) - if len(bs) != 1 { - t.Fatalf("the merge heard while the walk ran is in %d batches", len(bs)) - } - waiting := bs[0] - p, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if p.Delivery.WalkedAgainBy != waiting.ID || said.WalkedAgainBy != waiting.ID { - t.Fatalf("walked again by %q (said %q); want the open batch %s", p.Delivery.WalkedAgainBy, said.WalkedAgainBy, waiting.ID) - } - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batches; want the open one alone", len(bs)) - } - b, _ := open.inventory.PlanByID(ctx, waiting.ID) - if b.Delivery.Batch == nil || !b.Delivery.Batch.Closed || !slices.Contains(b.Delivery.Batch.Withheld, "app") { - t.Fatalf("the open batch reads %+v; want its window closed and app withheld", b.Delivery.Batch) - } - // Its window closed by the stop, not by the clock: cut at once. - cutAt(t, open, t0.Add(3*time.Minute+11*time.Second)) - fresh, _ := open.inventory.PlanByID(ctx, waiting.ID) - if fresh.Batch() || fresh.CommitOf("novox/one") != one2.Commit { - t.Fatalf("the batch is %s at %q; want it cut, walking the newer commit", fresh.State, fresh.CommitOf("novox/one")) - } - var answered []string - for _, m := range fresh.Delivery.Merges { - answered = append(answered, m.Commit+">"+m.Carried) - } - if !slices.Contains(answered, one1.Commit+">"+one2.Commit) || !slices.Contains(answered, one2.Commit+">") { - t.Fatalf("the walk answers %v; want the earlier merge carried by the later", answered) - } -} // A stop names what the stopped walk already sent and nothing walks again, on which machines; and a later merge of // the stopped merge's repository that waits in a batch, which will deliver the stopped change. A stopped walk // that walked its merges again is refused a retry, saying why. -func TestAStopNamesWhatStaysOnTheMachinesAndWhatStillCarriesIt(t *testing.T) { - open := windowed(t) - groupsWaiting(t, "pr/cat1aaaaaaaa") - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one1 := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one1, t0.Add(6*time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - walk := ws[0] - sent := t0.Add(150 * time.Second) - for name, m := range walk.Modules { - if name == "app" || name == "one" { - m.First, m.FirstAt = []string{"anchor"}, &sent - } - } - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - cat2 := catalogueMerge("cat2dddddddd", "notes", t0.Add(3*time.Minute)) - hear(t, open, cat2, t0.Add(3*time.Minute)) - _, said, err := stopWalkWithout(ctx, open.inventory, walk.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(3*time.Minute+10*time.Second)) - if err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(said.Kept, []inventory.PlanKept{{Module: "app", Machines: []string{"anchor"}}}) { - t.Fatalf("kept %+v; want app on anchor alone (one is walked again)", said.Kept) - } - p, _ := open.inventory.PlanByID(ctx, walk.ID) - if !reflect.DeepEqual(p.Delivery.Kept, said.Kept) || !strings.Contains(p.Note, "app on anchor") { - t.Fatalf("the stopped walk's record does not name what stays: %+v %q", p.Delivery.Kept, p.Note) - } - if len(said.StillCarriedBy) != 1 || !strings.Contains(said.StillCarriedBy[0], cat2.Commit) { - t.Fatalf("still carried by %v; want the later catalogue merge waiting in the batch", said.StillCarriedBy) - } - if _, err := retryPlan(ctx, open, walk.ID); err == nil || !strings.Contains(err.Error(), "was stopped without") { - t.Fatalf("a walk stopped without some merges was retried, or refused for another reason: %v", err) - } -} // A batch not yet cut is stopped without a merge: the merge goes to a stopped record of its own, the batch goes on // with the others and withholds the stopped merge's modules; with every merge left out, the batch is stopped. -func TestAStopWithoutAMergeLeavesItOutOfABatch(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - cat := catalogueMerge("cat1aaaaaaaa", "app", t0) - one := fromHead(repoMerge("one", "one1bbbbbbbb", t0.Add(5*time.Second)), cat.Head) - hear(t, open, cat, t0.Add(time.Second)) - hear(t, open, one, t0.Add(6*time.Second)) - _, bs := walks(t, open) - b := bs[0] - r, said, err := stopWalkWithout(ctx, open.inventory, b.ID, "mesh-delivery for jochen", "its group stopped", - []string{"novox/mesh-catalog@" + cat.Commit}, t0.Add(10*time.Second)) - if err != nil { - t.Fatal(err) - } - if said.WalkedAgainBy != b.ID || said.Stopped == b.ID { - t.Fatalf("said %+v; want a stopped record of its own, and the batch walking the others", said) - } - if left, _ := open.inventory.MergesOf(ctx, said.Stopped); len(left) != 1 || left[0].Commit != cat.Commit { - t.Fatalf("the stopped record holds %+v", left) - } - if stopped, _ := open.inventory.PlanByID(ctx, said.Stopped); stopped.State != inventory.PlanFailed || - stopped.Delivery.Stopped == "" || stopped.Delivery.WalkedAgainBy != b.ID { - t.Fatalf("the stopped record reads %s %+v", stopped.State, stopped.Delivery) - } - _ = r - cutAt(t, open, t0.Add(3*time.Minute)) - w, _ := open.inventory.PlanByID(ctx, b.ID) - if _, in := w.Modules["app"]; in || w.Batch() { - t.Fatalf("the batch's walk is %s and builds %v; want it cut without app", w.State, w.Modules) - } - // A batch whose every merge is left out is stopped itself. - hear(t, open, catalogueMerge("cat3eeeeeeee", "notes", t0.Add(4*time.Minute)), t0.Add(4*time.Minute)) - _, bs = walks(t, open) - last := bs[0] - if _, said, err := stopWalkWithout(ctx, open.inventory, last.ID, "mesh-delivery for jochen", "x", - []string{"novox/mesh-catalog@cat3eeeeeeee"}, t0.Add(4*time.Minute+time.Second)); err != nil || said.Stopped != last.ID { - t.Fatalf("a batch left with nothing: %+v %v", said, err) - } - if p, _ := open.inventory.PlanByID(ctx, last.ID); p.State != inventory.PlanFailed || p.Batch() { - t.Fatalf("the emptied batch is %s", p.State) - } -} // The stopped walk, the batch and the merges moved are one act (ADR 0299 decision 7): a plan written by somebody // else since it was read refuses all of it, and no merge moves, no plan is written. -func TestSavingPlansAndMovingMergesIsOneAct(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - endsItsLine(t) - ctx := t.Context() - hear(t, open, catalogueMerge("cat1aaaaaaaa", "app", t0), t0.Add(time.Second)) - _, bs := walks(t, open) - stale := bs[0] - moved := stale - moved.Note = "written by somebody else" - if err := open.inventory.SavePlan(ctx, &moved); err != nil { - t.Fatal(err) - } - fresh := inventory.Plan{ID: "plan-fresh", Repository: "novox/mesh-catalog", Branch: "main", Commit: "x", Created: t0, - State: inventory.PlanAssembling, Tiers: [][]string{}, Modules: map[string]*inventory.PlanModule{}} - err := open.inventory.SavePlansMoving(ctx, []*inventory.Plan{&fresh, &stale}, - []inventory.MergeMove{{Repository: "novox/mesh-catalog", Commit: "cat1aaaaaaaa", Plan: "plan-fresh"}}) - if !errors.Is(err, inventory.ErrPlanMoved) { - t.Fatalf("a stale plan in the act: %v", err) - } - if _, err := open.inventory.PlanByID(ctx, "plan-fresh"); err == nil { - t.Fatal("the new plan was written though the act was refused") - } - if m, _, _ := open.inventory.MergeOf(ctx, "novox/mesh-catalog", "cat1aaaaaaaa"); m.Plan != stale.ID { - t.Fatalf("the merge moved to %q though the act was refused", m.Plan) - } -} // What a stopped walk leaves on the machines names every machine its sends reached (review of #232): a module sent // with no first machine reached every machine running it; one sent first reached those, and the rest it was sent diff --git a/cmd/mesh-controller/issue611_integration_test.go b/cmd/mesh-controller/issue611_integration_test.go new file mode 100644 index 00000000..31aa2acb --- /dev/null +++ b/cmd/mesh-controller/issue611_integration_test.go @@ -0,0 +1,54 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", + Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), + Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { + t.Fatal(err) + } + readme := repoMerge("one", "one1aaaaaaaa", t0) + readme.Paths = []string{"README.md"} + hear(t, open, readme, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("walks: %+v", ws) + } + w := ws[0] + if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { + t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) + } + if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") { + t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) + } + if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") { + t.Errorf("the walk's note does not name it: %q", w.Note) + } + + // A merge that changes the build source is walked, and says nothing unchanged. + code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute)) + hear(t, open, code, t0.Add(3*time.Minute+time.Second)) + cutAt(t, open, t0.Add(6*time.Minute)) + ws, _ = walks(t, open) + if len(ws) != 2 { + t.Fatalf("walks: %+v", ws) + } + if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 { + t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery) + } +} diff --git a/cmd/mesh-controller/issue611_test.go b/cmd/mesh-controller/issue611_test.go index 2b2a8e7b..71235868 100644 --- a/cmd/mesh-controller/issue611_test.go +++ b/cmd/mesh-controller/issue611_test.go @@ -5,7 +5,6 @@ import ( "sort" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -95,48 +94,6 @@ func TestAWalkSaysUnchangedOnlyWhatItDoesNotBuild(t *testing.T) { // Against the store, the path a real merge takes: a merge whose module recorded a build source that holds none // of its files is cut into a walk done with nothing to walk, whose record names the module unchanged and why. -func TestAWalkThatMovedNothingSaysWhichModulesMatchedWhatRuns(t *testing.T) { - open := windowed(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { - t.Fatal(err) - } - readme := repoMerge("one", "one1aaaaaaaa", t0) - readme.Paths = []string{"README.md"} - hear(t, open, readme, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "holds none of the 1 changed file(s)") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) - } - if !strings.Contains(w.Note, "unchanged") || !strings.Contains(w.Note, "one") { - t.Errorf("the walk's note does not name it: %q", w.Note) - } - - // A merge that changes the build source is walked, and says nothing unchanged. - code := repoMerge("one", "one2bbbbbbbb", t0.Add(3*time.Minute)) - hear(t, open, code, t0.Add(3*time.Minute+time.Second)) - cutAt(t, open, t0.Add(6*time.Minute)) - ws, _ = walks(t, open) - if len(ws) != 2 { - t.Fatalf("walks: %+v", ws) - } - if _, built := ws[0].Modules["one"]; !built || ws[0].Delivery == nil || len(ws[0].Delivery.Unchanged) != 0 { - t.Fatalf("the walk of a change to main.go reads modules %v, delivery %+v", ws[0].Modules, ws[0].Delivery) - } -} func sortedOf(names []string) []string { out := append([]string{}, names...) diff --git a/cmd/mesh-controller/issue648_test.go b/cmd/mesh-controller/issue648_test.go index 0557e774..89fc9bb5 100644 --- a/cmd/mesh-controller/issue648_test.go +++ b/cmd/mesh-controller/issue648_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/issue_204_integration_test.go b/cmd/mesh-controller/issue_204_integration_test.go new file mode 100644 index 00000000..666519c8 --- /dev/null +++ b/cmd/mesh-controller/issue_204_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { + inv := inventory.ForTest(t) + ctx, cancel := context.WithCancel(t.Context()) + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + cancel() // the sender is going away: its context is cancelled between the send and the record + body := []byte(`{"declaration":1,"resources":[]}`) + digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) + if err != nil { + // NodeByName on the cancelled context may itself refuse; the record must still be possible + // through the detached context, so look the node up again on a live one. + t.Fatalf("recording a send after cancellation failed: %v", err) + } + outstanding, err := inv.Outstanding(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + if outstanding != digest || digest != digestOf(body) { + t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest) + } +} diff --git a/cmd/mesh-controller/issue_204_test.go b/cmd/mesh-controller/issue_204_test.go index 1779292a..950fd9cb 100644 --- a/cmd/mesh-controller/issue_204_test.go +++ b/cmd/mesh-controller/issue_204_test.go @@ -3,8 +3,6 @@ package main import ( "context" "testing" - - "github.com/novox/mesh-controller/internal/inventory" ) // A declaration composed earlier is numbered lower than one composed later, whatever order the two @@ -69,25 +67,3 @@ func TestTheNumberIsTakenBeforeComposingAndItsFailureIsARefusal(t *testing.T) { // What was sent is written down even when the sender's context is already cancelled (issue 204): a // controller replaced mid-send had told the machine and never recorded it, so status read "applied, // current" over a machine that had just been sent something else. -func TestASendIsRecordedEvenWhenTheSenderIsBeingCancelled(t *testing.T) { - inv := inventory.ForTest(t) - ctx, cancel := context.WithCancel(t.Context()) - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - cancel() // the sender is going away: its context is cancelled between the send and the record - body := []byte(`{"declaration":1,"resources":[]}`) - digest, err := recordSent(ctx, inv, "anchor", body, nil, 0, sentRecord{sender: "a test"}) - if err != nil { - // NodeByName on the cancelled context may itself refuse; the record must still be possible - // through the detached context, so look the node up again on a live one. - t.Fatalf("recording a send after cancellation failed: %v", err) - } - outstanding, err := inv.Outstanding(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - if outstanding != digest || digest != digestOf(body) { - t.Fatalf("the send was not recorded: outstanding %q, sent %q", outstanding, digest) - } -} diff --git a/cmd/mesh-controller/issued_integration_test.go b/cmd/mesh-controller/issued_integration_test.go new file mode 100644 index 00000000..023be0c0 --- /dev/null +++ b/cmd/mesh-controller/issued_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/secrets" +) + +func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) { + var r issuedRecorder + words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" { + t.Fatalf("the provider's value was not kept: %d kept", len(r.kept)) + } + if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" { + t.Fatalf("the holders' machines were not sent: %v", r.sent) + } + if len(r.acts) != 1 || r.acts[0].Verb != handActIssued { + t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts) + } + for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} { + if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) { + t.Fatalf("the value or its fingerprint was said: %q", said) + } + } + if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") { + t.Fatalf("the answer does not name the holders: %q", words) + } +} + +func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) { + r := issuedRecorder{again: true} + words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.sent) != 0 || len(r.acts) != 0 { + t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts) + } + if !strings.Contains(words, "nothing was sent") { + t.Fatalf("the answer does not say nothing was sent: %q", words) + } +} + +func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) { + other, _, _ := secrets.Keypair() + elsewhere, _ := secrets.Seal(other, []byte("somebody's value")) + raw, _ := json.Marshal(map[string]string{"sealed": elsewhere}) + plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"}) + for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){ + "sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil }, + "answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil }, + "refused": func(issuedProvider, map[string]any) (link.Answer, error) { + return link.Answer{Error: "the controller alone"}, nil + }, + "not reached": func(issuedProvider, map[string]any) (link.Answer, error) { + return link.Answer{}, errors.New("503 no responders") + }, + "sealed empty": issuedSealed(t, " "), + } { + var r issuedRecorder + if _, err := r.taker(t, ask).take("plex-api"); err == nil { + t.Errorf("%s: taken", name) + } + if len(r.kept) != 0 || len(r.sent) != 0 { + t.Errorf("%s: something was kept or sent", name) + } + } +} + +func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) { + asked := false + taker := issuedTaker{ + providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil }, + ask: func(issuedProvider, map[string]any) (link.Answer, error) { + asked = true + return link.Answer{}, nil + }, + } + if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") { + t.Fatalf("expected a refusal naming the record, got %v", err) + } + if asked { + t.Fatal("a provider was asked for a provision no offer says its software issues") + } +} diff --git a/cmd/mesh-controller/issued_test.go b/cmd/mesh-controller/issued_test.go index f2b9da84..f2819a76 100644 --- a/cmd/mesh-controller/issued_test.go +++ b/cmd/mesh-controller/issued_test.go @@ -2,7 +2,6 @@ package main import ( "encoding/json" - "errors" "strings" "testing" @@ -100,90 +99,12 @@ func (r *issuedRecorder) taker(t *testing.T, ask func(issuedProvider, map[string } // The value the provider's tool sealed to the call is kept, recorded and sent, and the answer never says it. -func TestAnIssuedValueIsTakenSealedFromTheProviderAndSent(t *testing.T) { - var r issuedRecorder - words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") - if err != nil { - t.Fatal(err) - } - if len(r.kept) != 1 || r.kept[0] != "the-servers-own-token" { - t.Fatalf("the provider's value was not kept: %d kept", len(r.kept)) - } - if len(r.sent) != 1 || strings.Join(r.sent[0], ",") != "ace" { - t.Fatalf("the holders' machines were not sent: %v", r.sent) - } - if len(r.acts) != 1 || r.acts[0].Verb != handActIssued { - t.Fatalf("the change was not recorded in the hand-act log: %+v", r.acts) - } - for _, said := range []string{words, r.acts[0].Why, strings.Join(r.acts[0].Args, " ")} { - if strings.Contains(said, "the-servers-own-token") || strings.Contains(said, inventory.IssuedStamp("the-servers-own-token")) { - t.Fatalf("the value or its fingerprint was said: %q", said) - } - } - if !strings.Contains(words, "kometa on ace") || !strings.Contains(words, "tautulli on ace") { - t.Fatalf("the answer does not name the holders: %q", words) - } -} // The same value again writes nothing, records nothing and sends nobody (ADR 0305 §5). -func TestAnIssuedValueEveryHolderHoldsSendsNobody(t *testing.T) { - r := issuedRecorder{again: true} - words, err := r.taker(t, issuedSealed(t, "the-servers-own-token")).take("plex-api") - if err != nil { - t.Fatal(err) - } - if len(r.sent) != 0 || len(r.acts) != 0 { - t.Fatalf("an unchanged value was sent or recorded: %v %v", r.sent, r.acts) - } - if !strings.Contains(words, "nothing was sent") { - t.Fatalf("the answer does not say nothing was sent: %q", words) - } -} // A value not sealed to the call's key is not taken, nor is a refusal, nor an empty answer (ADR 0305 §2). -func TestOnlyAValueSealedToTheCallIsTaken(t *testing.T) { - other, _, _ := secrets.Keypair() - elsewhere, _ := secrets.Seal(other, []byte("somebody's value")) - raw, _ := json.Marshal(map[string]string{"sealed": elsewhere}) - plain, _ := json.Marshal(map[string]string{"value": "the-servers-own-token"}) - for name, ask := range map[string]func(issuedProvider, map[string]any) (link.Answer, error){ - "sealed to another key": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: raw}, nil }, - "answered in the clear": func(issuedProvider, map[string]any) (link.Answer, error) { return link.Answer{Result: plain}, nil }, - "refused": func(issuedProvider, map[string]any) (link.Answer, error) { - return link.Answer{Error: "the controller alone"}, nil - }, - "not reached": func(issuedProvider, map[string]any) (link.Answer, error) { - return link.Answer{}, errors.New("503 no responders") - }, - "sealed empty": issuedSealed(t, " "), - } { - var r issuedRecorder - if _, err := r.taker(t, ask).take("plex-api"); err == nil { - t.Errorf("%s: taken", name) - } - if len(r.kept) != 0 || len(r.sent) != 0 { - t.Errorf("%s: something was kept or sent", name) - } - } -} // A provision no offer says its software issues is refused before anything is asked. -func TestAProvisionNoOfferSaysIsIssuedIsRefused(t *testing.T) { - asked := false - taker := issuedTaker{ - providers: func(string) ([]issuedProvider, bool, error) { return nil, false, nil }, - ask: func(issuedProvider, map[string]any) (link.Answer, error) { - asked = true - return link.Answer{}, nil - }, - } - if _, err := taker.take("postgres-database"); err == nil || !strings.Contains(err.Error(), "ADR 0305") { - t.Fatalf("expected a refusal naming the record, got %v", err) - } - if asked { - t.Fatal("a provider was asked for a provision no offer says its software issues") - } -} // Which assignments are asked: the modules on the providing machine whose offer says so, with their tool. func TestTheProvidersAskedAreTheOffersThatSaySo(t *testing.T) { diff --git a/cmd/mesh-controller/issued_to_vault_test.go b/cmd/mesh-controller/issued_to_vault_test.go index d2525d65..6f518918 100644 --- a/cmd/mesh-controller/issued_to_vault_test.go +++ b/cmd/mesh-controller/issued_to_vault_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/join_through_the_tunnel_test.go b/cmd/mesh-controller/join_through_the_tunnel_test.go index 5782644b..19cc37b2 100644 --- a/cmd/mesh-controller/join_through_the_tunnel_test.go +++ b/cmd/mesh-controller/join_through_the_tunnel_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/last_send_integration_test.go b/cmd/mesh-controller/last_send_integration_test.go new file mode 100644 index 00000000..4cb33fb7 --- /dev/null +++ b/cmd/mesh-controller/last_send_integration_test.go @@ -0,0 +1,51 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + record, err := open.inventory.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, + Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", + Modules: []string{"docker", "sudo"}}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, + Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { + t.Fatal(err) + } + shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) + for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", + "docker, sudo", "unknown field \"generation\""} { + if !strings.Contains(shown, want) { + t.Errorf("node show laptop does not say %q:\n%s", want, shown) + } + } + body := printed(t, func() error { return statusFor(ctx, open, true) }) + var got struct { + LastSent []struct { + Node string `json:"node"` + Sender string `json:"sender"` + Answer string `json:"answer"` + } `json:"lastSent"` + } + if err := json.Unmarshal([]byte(body), &got); err != nil { + t.Fatalf("%v:\n%s", err, body) + } + if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || + got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { + t.Fatalf("status --json's last sends read %+v", got.LastSent) + } +} diff --git a/cmd/mesh-controller/last_send_test.go b/cmd/mesh-controller/last_send_test.go index 0fecd717..33c329f7 100644 --- a/cmd/mesh-controller/last_send_test.go +++ b/cmd/mesh-controller/last_send_test.go @@ -119,42 +119,3 @@ func TestStatusJSONCarriesEachMachinesLastSend(t *testing.T) { // And through the store: a send recorded and refused by the machine is what `node show` and `status --json` // read back. -func TestNodeShowAndStatusReadTheLastSendFromTheStore(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - record, err := open.inventory.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RecordSentWith(ctx, record.ID, "d12", nil, 57, 40, inventory.Send{Sequence: 12, - Epoch: 57, Sender: "a one-shot push by jochen at a shell on anchor", Generation: 40, Digest: "d12", - Modules: []string{"docker", "sudo"}}); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.RecordDoing(ctx, record.ID, inventory.Doing{Outcome: inventory.OutcomeRefused, - Declared: "d12", Refused: "unknown field \"generation\""}); err != nil { - t.Fatal(err) - } - shown := printed(t, func() error { return showNode(ctx, open, "laptop") }) - for _, want := range []string{"a one-shot push by jochen at a shell on anchor", "lease epoch 57", "generation 40", - "docker, sudo", "unknown field \"generation\""} { - if !strings.Contains(shown, want) { - t.Errorf("node show laptop does not say %q:\n%s", want, shown) - } - } - body := printed(t, func() error { return statusFor(ctx, open, true) }) - var got struct { - LastSent []struct { - Node string `json:"node"` - Sender string `json:"sender"` - Answer string `json:"answer"` - } `json:"lastSent"` - } - if err := json.Unmarshal([]byte(body), &got); err != nil { - t.Fatalf("%v:\n%s", err, body) - } - if len(got.LastSent) != 1 || got.LastSent[0].Node != "laptop" || got.LastSent[0].Answer != "refused" || - got.LastSent[0].Sender != "a one-shot push by jochen at a shell on anchor" { - t.Fatalf("status --json's last sends read %+v", got.LastSent) - } -} diff --git a/cmd/mesh-controller/lean_walk_integration_test.go b/cmd/mesh-controller/lean_walk_integration_test.go new file mode 100644 index 00000000..bc5ebd47 --- /dev/null +++ b/cmd/mesh-controller/lean_walk_integration_test.go @@ -0,0 +1,141 @@ +//go:build integration + +package main + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, n := range []string{"desk", "laptop", "anchor"} { + if !slices.Contains(*said, "send "+n) { + t.Fatalf("%s was not sent: %v", n, *said) + } + } + if slices.Contains(*said, "judge") { + t.Fatalf("a machine was judged by its health: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + b.broken["desk"] = true // desk fails what it is sent + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the others have no verdict of their own: %+v", p.Release) + } + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { + t.Fatalf("desk was put back: %v", sent) + } +} + +func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gates["anchor"] + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). + if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/lean_walk_test.go b/cmd/mesh-controller/lean_walk_test.go index eced96cf..5f6ad330 100644 --- a/cmd/mesh-controller/lean_walk_test.go +++ b/cmd/mesh-controller/lean_walk_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "slices" "strings" "testing" @@ -53,140 +52,15 @@ func withTheGate(t *testing.T) { // Every machine heard from is sent at once and passed on its applied report, with no judging (novox/hq ADR 0360 §7); // before, the first machine was judged before the rest were sent (ADR 0353 §1). -func TestAReleaseSendsEveryMachineAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, n := range []string{"desk", "laptop", "anchor"} { - if !slices.Contains(*said, "send "+n) { - t.Fatalf("%s was not sent: %v", n, *said) - } - } - if slices.Contains(*said, "judge") { - t.Fatalf("a machine was judged by its health: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} // A machine that reports its apply failed is named and fails the release, with nothing put back, and every other // machine goes on to its own report (novox/hq ADR 0360 §5, §7); before, it was put back (ADR 0353 §3). -func TestAMachineThatFailsItsApplyIsNamedAndNothingIsPutBack(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - b.broken["desk"] = true // desk fails what it is sent - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.Contains(p.Release.Done, "laptop") || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the others have no verdict of their own: %+v", p.Release) - } - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c2" { - t.Fatalf("desk was put back: %v", sent) - } -} // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8), with every machine sent at once (ADR 0360 §7). -func TestAReleaseSendKeptAndMadeIsNotMadeAgainAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gates["anchor"] == nil || !p.Release.Gates["anchor"].Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if n := len(slices.DeleteFunc(slices.Clone(b.sent), func(s []string) bool { return !slices.Equal(s, []string{"anchor"}) })); n != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gates["anchor"] - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4); its step also sent anchor, so the release is done, not failed (ADR 0360 §7). -func TestAMachineSilentPastItsBoundIsLeftAndSaidAtOnce(t *testing.T) { - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - // anchor applied in the same step, so the release is done, laptop named silent (novox/hq ADR 0360 §7). - if p.State != inventory.PlanDone || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} // Issue 840 (novox/hq ADR 0362 §1-5): a walk's end counts a machine's send of a provisioned module applied only once // the engine and the machine's provisioner client both said so. diff --git a/cmd/mesh-controller/lease_command_test.go b/cmd/mesh-controller/lease_command_test.go index af20840e..f8c0c811 100644 --- a/cmd/mesh-controller/lease_command_test.go +++ b/cmd/mesh-controller/lease_command_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/lease_integration_test.go b/cmd/mesh-controller/lease_integration_test.go new file mode 100644 index 00000000..4c7e980f --- /dev/null +++ b/cmd/mesh-controller/lease_integration_test.go @@ -0,0 +1,154 @@ +//go:build integration + +package main + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) { + t.Helper() + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + js, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + _ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket) + if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil { + t.Fatal(err) + } + kv, err := js.KeyValue(t.Context(), broker.LeaseBucket) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) }) + return url, kv +} + +func TestTwoControllersOneActs(t *testing.T) { + url, kv := aBusForTheLease(t) + inv := inventory.ForTest(t) + ctx := t.Context() + + // Each renews five times and looks again ten times as often as a serving controller does, so the + // handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what + // is checked is the order of events, which does not change with the interval. Each renewal still has a + // serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease. + const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10 + + // Controller A takes the lease. + a := &actor{renewEvery: renewEvery, poll: poll} + aCtx, stopA := context.WithCancel(ctx) + defer stopA() + lostA, err := a.serveUnderTheLease(aCtx, inv, url) + if err != nil { + t.Fatal(err) + } + epochA, err := a.epoch(ctx) + if err != nil || epochA == 0 { + t.Fatalf("A holds no epoch: %d, %v", epochA, err) + } + + // Controller B starts while A holds it, and waits — acting on nothing meanwhile. + b := &actor{renewEvery: renewEvery, poll: poll} + bCtx, stopB := context.WithCancel(ctx) + defer stopB() + tookB := make(chan (<-chan struct{}), 1) + go func() { + lost, err := b.serveUnderTheLease(bCtx, inv, url) + if err != nil { + t.Errorf("B: %v", err) + close(tookB) + return + } + tookB <- lost + }() + // B looks fifteen times, across one of A's renewals at least. + select { + case <-tookB: + t.Fatal("B took the lease while A held it") + case <-time.After(15 * poll): + } + if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { + t.Fatalf("B, waiting, may act: %v", err) + } + + // A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch. + stopA() + a.release() + var lostB <-chan struct{} + select { + case lostB = <-tookB: + case <-time.After(10 * time.Second): + t.Fatal("B did not take the lease A gave back") + } + select { + case <-lostA: + default: + t.Fatal("A, having given the lease back, is not told it no longer holds it") + } + epochB, err := b.epoch(ctx) + if err != nil || epochB <= epochA { + t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err) + } + if _, err := a.epoch(ctx); err == nil { + t.Fatal("A acts after giving the lease back") + } + ea, _, _ := inv.EpochOf(ctx, epochA) + eb, _, _ := inv.EpochOf(ctx, epochB) + if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance { + t.Fatalf("the record of the handover reads %+v then %+v", ea, eb) + } + + // Something else writes the lease's key — a third controller on a clock that read it as expired: + // B's next renewal is refused, and B stops acting at once. + if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil { + t.Fatal(err) + } + select { + case <-lostB: + case <-time.After(2 * renewEvery): + t.Fatal("B was not told it lost the lease") + } + if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { + t.Fatalf("B acts after losing the lease: %v", err) + } + // Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition + // is not raised. + inv.ActsUnder(b.epoch) + if _, err := inv.AddNode(ctx, "anchor"); err != nil { + t.Fatal(err) + } + saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), + State: inventory.PlanBuilding} + if err := inv.SavePlan(ctx, &saved); err == nil { + t.Fatal("B wrote a plan after losing the lease") + } + store := conditions.NewInMemory() + keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, + Epoch: func() (uint64, error) { return b.epoch(ctx) }}) + defer keeper.Close(context.Background()) + if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x", + Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil { + t.Fatal("B raised a condition after losing the lease") + } + if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost { + t.Fatalf("B's epoch does not say it was lost: %+v", ended) + } +} diff --git a/cmd/mesh-controller/lease_test.go b/cmd/mesh-controller/lease_test.go index 5356a116..2aa9114d 100644 --- a/cmd/mesh-controller/lease_test.go +++ b/cmd/mesh-controller/lease_test.go @@ -4,17 +4,8 @@ import ( "context" "errors" "testing" - "time" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/lease" - - "github.com/novox/mesh-controller/internal/testbus" ) // Two controllers at once (novox/hq to-be 45 §6, issue 204; the half of replay R1 that lives here): two @@ -26,141 +17,6 @@ import ( // MESH_TEST_POSTGRES=… go test ./cmd/mesh-controller/ -run Controllers (each test on a bus of its own: internal/testbus) // aBusForTheLease is the test bus with the controller's lease bucket new. -func aBusForTheLease(t *testing.T) (string, jetstream.KeyValue) { - t.Helper() - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - js, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - _ = js.DeleteKeyValue(t.Context(), broker.LeaseBucket) - if err := broker.EnsureLeaseBucket(t.Context(), js); err != nil { - t.Fatal(err) - } - kv, err := js.KeyValue(t.Context(), broker.LeaseBucket) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.DeleteKeyValue(context.Background(), broker.LeaseBucket) }) - return url, kv -} - -func TestTwoControllersOneActs(t *testing.T) { - url, kv := aBusForTheLease(t) - inv := inventory.ForTest(t) - ctx := t.Context() - - // Each renews five times and looks again ten times as often as a serving controller does, so the - // handover and the loss below are seen in a second, not in a renewal of five (novox/hq issue 591): what - // is checked is the order of events, which does not change with the interval. Each renewal still has a - // serving controller's five seconds to answer (acting.go), so a slow bus does not lose A the lease. - const renewEvery, poll = lease.RenewEvery / 5, lease.Poll / 10 - - // Controller A takes the lease. - a := &actor{renewEvery: renewEvery, poll: poll} - aCtx, stopA := context.WithCancel(ctx) - defer stopA() - lostA, err := a.serveUnderTheLease(aCtx, inv, url) - if err != nil { - t.Fatal(err) - } - epochA, err := a.epoch(ctx) - if err != nil || epochA == 0 { - t.Fatalf("A holds no epoch: %d, %v", epochA, err) - } - - // Controller B starts while A holds it, and waits — acting on nothing meanwhile. - b := &actor{renewEvery: renewEvery, poll: poll} - bCtx, stopB := context.WithCancel(ctx) - defer stopB() - tookB := make(chan (<-chan struct{}), 1) - go func() { - lost, err := b.serveUnderTheLease(bCtx, inv, url) - if err != nil { - t.Errorf("B: %v", err) - close(tookB) - return - } - tookB <- lost - }() - // B looks fifteen times, across one of A's renewals at least. - select { - case <-tookB: - t.Fatal("B took the lease while A held it") - case <-time.After(15 * poll): - } - if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { - t.Fatalf("B, waiting, may act: %v", err) - } - - // A hands over, as a controller being replaced does: B takes the lease at once, at a higher epoch. - stopA() - a.release() - var lostB <-chan struct{} - select { - case lostB = <-tookB: - case <-time.After(10 * time.Second): - t.Fatal("B did not take the lease A gave back") - } - select { - case <-lostA: - default: - t.Fatal("A, having given the lease back, is not told it no longer holds it") - } - epochB, err := b.epoch(ctx) - if err != nil || epochB <= epochA { - t.Fatalf("B acts as epoch %d after A's %d (%v): an epoch only grows", epochB, epochA, err) - } - if _, err := a.epoch(ctx); err == nil { - t.Fatal("A acts after giving the lease back") - } - ea, _, _ := inv.EpochOf(ctx, epochA) - eb, _, _ := inv.EpochOf(ctx, epochB) - if ea.How != inventory.EpochReleased || eb.Ended != nil || eb.Instance != instance { - t.Fatalf("the record of the handover reads %+v then %+v", ea, eb) - } - - // Something else writes the lease's key — a third controller on a clock that read it as expired: - // B's next renewal is refused, and B stops acting at once. - if _, err := kv.Put(ctx, lease.Key, []byte(`{"instance":"a third controller","epoch":1}`)); err != nil { - t.Fatal(err) - } - select { - case <-lostB: - case <-time.After(2 * renewEvery): - t.Fatal("B was not told it lost the lease") - } - if _, err := b.epoch(ctx); !errors.Is(err, lease.ErrNotHeld) { - t.Fatalf("B acts after losing the lease: %v", err) - } - // Nothing B does is written: a declaration's number is not taken, a plan is not saved, a condition - // is not raised. - inv.ActsUnder(b.epoch) - if _, err := inv.AddNode(ctx, "anchor"); err != nil { - t.Fatal(err) - } - saved := inventory.Plan{ID: "plan-after-loss", Repository: "novox/app", Commit: "c0ffee00", Created: time.Now(), - State: inventory.PlanBuilding} - if err := inv.SavePlan(ctx, &saved); err == nil { - t.Fatal("B wrote a plan after losing the lease") - } - store := conditions.NewInMemory() - keeper := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, - Epoch: func() (uint64, error) { return b.epoch(ctx) }}) - defer keeper.Close(context.Background()) - if _, err := keeper.Observe(ctx, conditions.Observation{Scope: conditions.ScopeCore, ID: "x", Kind: "x", - Severity: conditions.Warning, Summary: "x", Source: "test"}); err == nil { - t.Fatal("B raised a condition after losing the lease") - } - if ended, _, _ := inv.EpochOf(ctx, epochB); ended.How != inventory.EpochLost { - t.Fatalf("B's epoch does not say it was lost: %+v", ended) - } -} // A controller whose bus refuses it the lease's key, with nobody holding it, serves without the lease: // it acts with no epoch — refused by no node-engine — says so, and takes the lease once it can. diff --git a/cmd/mesh-controller/left_out_integration_test.go b/cmd/mesh-controller/left_out_integration_test.go new file mode 100644 index 00000000..feae280f --- /dev/null +++ b/cmd/mesh-controller/left_out_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) { + plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil) + withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil + }}) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"} + key := "module.nfs-server.anchor.left-out" + openOnes := func() map[string]conditions.Condition { + t.Helper() + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range open { + out[c.Key] = c + } + return out + } + + d.runOnce(t.Context(), "a test") + c, raised := openOnes()[key] + if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning { + t.Fatalf("a module left out raised %+v", openOnes()) + } + // A statement from the machine that says nothing of it — the module runs nothing there — leaves it open. + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, + time.Now().Add(72*time.Hour)); err != nil { + t.Fatal(err) + } + if _, still := openOnes()[key]; !still { + t.Fatal("a health statement that says nothing of the module cleared its left-out condition") + } + d.runOnce(t.Context(), "a test") + if c := openOnes()[key]; c.Severity != conditions.Warning { + t.Fatalf("after a health statement and days, it is %+v", openOnes()) + } + + // The setting given: it composes, and the condition clears. + settings = sharesGiven("library=/srv/library") + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("composed again, still open: %+v", openOnes()) + } + + // Left out again, then unassigned: no longer in the machine's set, and it clears. + settings = nil + d.runOnce(t.Context(), "a test") + if _, raised := openOnes()[key]; !raised { + t.Fatal("left out again and not raised") + } + plan = catalogue.Resolution{} + d.runOnce(t.Context(), "a test") + if _, still := openOnes()[key]; still { + t.Fatalf("unassigned, still open: %+v", openOnes()) + } +} + +func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) { + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false)) + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting, + Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}} + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil { + t.Fatal(err) + } + waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out" + open := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator { + t.Fatalf("the wait raised %+v", open()) + } + if c := open()[leftKey]; c.Kind != kindNeedsOperator { + t.Fatalf("the left-out condition is not open as needs-operator: %+v", open()) + } + // D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait, + // which the machine's statement raised. + if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; !still { + t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open()) + } + if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + if _, still := open()[waitKey]; still { + t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open()) + } + if _, still := open()[leftKey]; !still { + t.Fatalf("the left-out condition was cleared by a health statement: %+v", open()) + } +} diff --git a/cmd/mesh-controller/left_out_test.go b/cmd/mesh-controller/left_out_test.go index 7c75d307..795bef55 100644 --- a/cmd/mesh-controller/left_out_test.go +++ b/cmd/mesh-controller/left_out_test.go @@ -1,18 +1,14 @@ package main import ( - "context" "encoding/json" "os" "slices" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 380: nfs-server was assigned to the home server for days and every send left it out — "nfs-server @@ -95,67 +91,6 @@ func TestAModuleLeftOutForAnotherCauseIsAWarningWithTheReasonAsEvidence(t *testi // The self-check raises it, keeps it a warning however long it stands, and clears it when the module composes // again or is no longer assigned; a machine's health statement neither clears nor raises it. -func TestALeftOutModulesConditionClearsWhenItComposesAgainOrIsUnassigned(t *testing.T) { - plan, settings := leftOutNFS(t), catalogue.SettingsBy(nil) - withProbes(t, probe{ID: probeLeftOutID, Asserts: "the test's", Kind: kindLeftOut, Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - return leftOutObservations("anchor", leftOutOf(plan, settings, false)), nil - }}) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - d := &doctor{keeper: k, teller: &conditions.Told{}, host: "anchor"} - key := "module.nfs-server.anchor.left-out" - openOnes := func() map[string]conditions.Condition { - t.Helper() - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range open { - out[c.Key] = c - } - return out - } - - d.runOnce(t.Context(), "a test") - c, raised := openOnes()[key] - if !raised || c.Kind != kindNeedsOperator || c.Severity != conditions.Warning { - t.Fatalf("a module left out raised %+v", openOnes()) - } - // A statement from the machine that says nothing of it — the module runs nothing there — leaves it open. - if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, - time.Now().Add(72*time.Hour)); err != nil { - t.Fatal(err) - } - if _, still := openOnes()[key]; !still { - t.Fatal("a health statement that says nothing of the module cleared its left-out condition") - } - d.runOnce(t.Context(), "a test") - if c := openOnes()[key]; c.Severity != conditions.Warning { - t.Fatalf("after a health statement and days, it is %+v", openOnes()) - } - - // The setting given: it composes, and the condition clears. - settings = sharesGiven("library=/srv/library") - d.runOnce(t.Context(), "a test") - if _, still := openOnes()[key]; still { - t.Fatalf("composed again, still open: %+v", openOnes()) - } - - // Left out again, then unassigned: no longer in the machine's set, and it clears. - settings = nil - d.runOnce(t.Context(), "a test") - if _, raised := openOnes()[key]; !raised { - t.Fatal("left out again and not raised") - } - plan = catalogue.Resolution{} - d.runOnce(t.Context(), "a test") - if _, still := openOnes()[key]; still { - t.Fatalf("unassigned, still open: %+v", openOnes()) - } -} func TestTheLeftOutProbeIsInTheRegistry(t *testing.T) { for _, p := range probeRegistry { @@ -207,53 +142,3 @@ func TestStatusAndNodeListAModuleAssignedAndNotApplied(t *testing.T) { // The skip is this probe's alone (review of #223): a part waiting for the operator (ADR 0283) keeps its own // needs-operator condition, `module...needs-operator`, which still clears on the first statement that no // longer names it — beside a left-out condition on the same machine, which stays. -func TestAWaitsNeedsOperatorStillClearsWhenItsStatementStopsNamingItBesideALeftOutOne(t *testing.T) { - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - left := leftOutObservations("anchor", leftOutOf(leftOutNFS(t), nil, false)) - if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { - t.Fatal(err) - } - waiting := map[string][]inventory.ResourceHealth{"notes": {{Module: "notes", Resource: "server", State: link.StateWaiting, - Waits: []inventory.Wait{{Setting: "domain", What: "the domain it serves"}}}}} - if err := judgeModuleHealth(t.Context(), nil, k, "anchor", waiting, map[string]int{"notes": 2}, time.Now()); err != nil { - t.Fatal(err) - } - waitKey, leftKey := needsOperatorKey("notes", "anchor"), "module.nfs-server.anchor.left-out" - open := func() map[string]conditions.Condition { - t.Helper() - list, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range list { - out[c.Key] = c - } - return out - } - if c, raised := open()[waitKey]; !raised || c.Kind != kindNeedsOperator { - t.Fatalf("the wait raised %+v", open()) - } - if c := open()[leftKey]; c.Kind != kindNeedsOperator { - t.Fatalf("the left-out condition is not open as needs-operator: %+v", open()) - } - // D1 runs again and still finds nfs-server left out: its reconcile clears only what D1 raised, never the wait, - // which the machine's statement raised. - if err := k.Reconcile(t.Context(), probeLeftOutID, left); err != nil { - t.Fatal(err) - } - if _, still := open()[waitKey]; !still { - t.Fatalf("D1's reconcile cleared the wait's needs-operator: %+v", open()) - } - if err := judgeModuleHealth(t.Context(), nil, k, "anchor", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - if _, still := open()[waitKey]; still { - t.Fatalf("the statement no longer names the wait, and its needs-operator is still open: %+v", open()) - } - if _, still := open()[leftKey]; !still { - t.Fatalf("the left-out condition was cleared by a health statement: %+v", open()) - } -} diff --git a/cmd/mesh-controller/lone_merge_test.go b/cmd/mesh-controller/lone_merge_test.go index 344c12a5..4c9eb8f6 100644 --- a/cmd/mesh-controller/lone_merge_test.go +++ b/cmd/mesh-controller/lone_merge_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( @@ -99,16 +101,19 @@ func TestARelatedMergeJoinsTheBatchAndWaitsForTheWindow(t *testing.T) { func TestAMergeOfAGroupWaitsEvenAlone(t *testing.T) { open := windowed(t) asksWithPaths(t) - groupsWaiting(t, "feat/x") + states := map[string]string{"novox/one": "ready", "novox/two": "ready", "novox/elsewhere": "ready"} + groupStates(t, "feat/x", states) hear(t, open, fromHead(repoMerge("one", "c1", t0), "feat/x"), t0) ws, bs := walks(t, open) if len(ws) != 0 || len(bs) != 1 || bs[0].Delivery.Batch.CutAtOnce || windowClosed(bs[0], t0) { t.Fatalf("a group's merge alone: %d walk(s), batches %+v", len(ws), bs) } - if words := batchWords(bs[0], t0); !strings.HasPrefix(words, "assembling: 90 s left") || - !strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready)") { + if words := batchWords(bs[0], t0); !strings.Contains(words, "group feat/x has a member still to merge: novox/elsewhere (ready), novox/two (ready)") { t.Fatalf("it reads %q", words) } + // Its members merge: the batch is held for the last, and the window runs again from that merge. + delete(states, "novox/elsewhere") + states["novox/two"] = "merged" hear(t, open, fromHead(repoMerge("two", "c2", t0.Add(5*time.Second)), "feat/x"), t0.Add(5*time.Second)) if _, bs := walks(t, open); len(bs) != 1 || len(bs[0].Delivery.Merges) != 2 { t.Fatalf("the group's second member did not join the first: %+v", bs) @@ -331,70 +336,3 @@ func TestALoneMergeRelatedToAWalkTheCutFoldsJoinsTheBatch(t *testing.T) { t.Fatalf("a lone merge related to the waiting walk: batches %+v", bs) } } - -// A delivery group with a member of each kind (review of #250): a merge on the controller's own path and a -// catalogue merge never share a batch, so the later member cannot join the earlier one's. Kept in a batch of the -// other kind, the earlier member keeps the later one waiting, whatever mesh-delivery says yet — never cut at once -// to walk before it. The two batches are then cut in the order their windows close (cutBatchesHeld: the oldest -// closed batch first, an own-path one first at one look, and the other queued behind a walk that started): the -// earlier member's window closes first, so it walks first, in both orders. -func TestAGroupAcrossBothKindsKeepsItsOrder(t *testing.T) { - for _, ownFirst := range []bool{true, false} { - name := "catalogue member first" - if ownFirst { - name = "controller member first" - } - t.Run(name, func(t *testing.T) { - open := windowed(t) - asksWithPaths(t) - ctx := t.Context() - was := deliveryGroupOf - // mesh-delivery still says both members ready: it marks one merged only once its walk opened. - deliveryGroupOf = func(context.Context, bool, string) ([]groupMember, bool, error) { - return []groupMember{{Repository: "novox/mesh-controller", State: "ready"}, - {Repository: "novox/mesh-catalog", State: "ready"}}, true, nil - } - t.Cleanup(func() { deliveryGroupOf = was }) - own := fromHead(repoMerge("mesh-controller", "k1", t0), "feat/both-kinds") - cat := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), "feat/both-kinds") - first, second := cat, own - if ownFirst { - first, second = own, cat - } - second.MergedAt = t0.Add(10 * time.Second).Format(time.RFC3339Nano) - hear(t, open, first, t0) - hear(t, open, second, t0.Add(10*time.Second)) - ws, bs := walks(t, open) - if len(ws) != 0 || len(bs) != 2 { - t.Fatalf("both members heard: %d walk(s), %d batch(es); want two batches, nothing cut", len(ws), len(bs)) - } - later := bs[0] // newest first - if later.Delivery.Batch.CutAtOnce || !strings.Contains(later.Delivery.Batch.Waits, - "has a member waiting in a batch of the other kind: novox/"+first.Repo+"@") { - t.Fatalf("the later member's batch reads %+v; want it waiting for the earlier member", later.Delivery.Batch) - } - // The earlier member's window closes first: its batch alone is cut. - cutAt(t, open, t0.Add(90*time.Second)) - ws, bs = walks(t, open) - if len(ws) != 1 || !ws[0].Open() || ws[0].CommitOf("novox/"+first.Repo) != first.Commit || len(bs) != 1 || - bs[0].ID != later.ID { - t.Fatalf("at the earlier window: walks %+v, batches %+v; want the earlier member alone cut", ws, bs) - } - earlier := ws[0] - // The later member's window closes while the earlier walk runs: it queues behind it. - cutAt(t, open, t0.Add(100*time.Second)) - if _, bs := walks(t, open); len(bs) != 1 || bs[0].State != inventory.PlanQueued || bs[0].Delivery.Batch.Behind != earlier.ID { - t.Fatalf("the later member's batch does not queue behind the earlier walk: %+v", bs) - } - earlier.State = inventory.PlanDone - if err := open.inventory.SavePlan(ctx, &earlier); err != nil { - t.Fatal(err) - } - cutAt(t, open, t0.Add(101*time.Second)) - got, err := open.inventory.PlanByID(ctx, later.ID) - if err != nil || got.Batch() || got.CommitOf("novox/"+second.Repo) != second.Commit { - t.Fatalf("the later member was not walked after the earlier: %s %v", got.State, err) - } - }) - } -} diff --git a/cmd/mesh-controller/machine_network_integration_test.go b/cmd/mesh-controller/machine_network_integration_test.go new file mode 100644 index 00000000..0a18cf2d --- /dev/null +++ b/cmd/mesh-controller/machine_network_integration_test.go @@ -0,0 +1,266 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + say := func(at time.Time, n *link.NetworkHealth) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { + t.Fatal(err) + } + } + openKeys := func() []string { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + var keys []string + for _, c := range list { + keys = append(keys, c.Key) + } + return keys + } + healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} + rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ + {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, + Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, + {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, + Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} + + say(h0, healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("a healthy network raised %v", keys) + } + say(h0.Add(time.Minute), rewritten) + if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("the rewrite raised %v", keys) + } + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { + t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || + !strings.Contains(lines, "unhealthy resolv-conf") { + t.Fatalf("node show says:\n%s", lines) + } + say(h0.Add(2*time.Minute), healthy) + if keys := openKeys(); len(keys) != 0 { + t.Fatalf("written back, still open: %v", keys) + } + // An engine older than the judging says no network: nothing raised, and node show says it is not known. + say(h0.Add(3*time.Minute), nil) + kept, had, _ = inv.HealthOf(ctx, "laptop") + if keys := openKeys(); len(keys) != 0 || kept.Network != nil { + t.Fatalf("an older engine: %v %+v", keys, kept.Network) + } + if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { + t.Fatalf("node show says:\n%s", lines) + } +} + +func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { + raw, err := os.ReadFile("testdata/network-drill.json") + if err != nil { + t.Fatal(err) + } + var said []link.Health + if err := json.Unmarshal(raw, &said); err != nil { + t.Fatal(err) + } + open := aMesh(t) + ctx := t.Context() + k := conditionsFrom + var raisedAt []int + for i, h := range said { + if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { + t.Fatal(err) + } + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + for _, c := range list { + if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { + t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) + } + raisedAt = append(raisedAt, i) + } + } + // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. + if len(raisedAt) != 1 || raisedAt[0] != 3 { + t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) + } +} + +func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { + for i := range parts { + if parts[i].State == "" { + parts[i].State = link.StateUnhealthy + } + parts[i].Since = h0 + } + return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} +} + +func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { + t.Fatalf("want one finding, the rewrite, got %v", keys) + } + o := obs[0] + for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", + "next reconcile"} { + if !strings.Contains(o.Summary, want) { + t.Errorf("the summary does not say %q: %s", want, o.Summary) + } + } + if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { + t.Errorf("an address or a path reached the summary: %s", o.Summary) + } + if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { + t.Errorf("the evidence or the severity is wrong: %+v", o) + } +} + +func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateHealthy), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("want the laptop's own, got %v", keys) + } + if obs[0].Severity != conditions.Warning { + t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) + } +} + +func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + "spare": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { + t.Fatalf("want one condition at the hub, got %v", keys) + } + o := obs[0] + if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) + f.silent["anchor"] = true + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) + } +} + +func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, noRoute), + "laptop": aNetwork(link.StateUnhealthy, tunnelDown), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { + t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) + } + if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { + t.Fatalf("the hub's condition is %+v", o) + } +} + +func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { + silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", + Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} + obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { + t.Fatalf("one machine alone: want its own, got %v", keys) + } + obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ + "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), + "laptop": aNetwork(link.StateUnhealthy, silentResolver), + })) + if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { + t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) + } + if !strings.Contains(obs[0].Summary, "its resolver") { + t.Fatalf("the resolver's machine is said %s", obs[0].Summary) + } +} + +func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the control node's rewritten file: %+v", obs) + } + bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} + f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) + if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { + t.Fatalf("the bus unreachable from the laptop: %+v", obs) + } +} + +func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { + f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) + if obs := networkObservations(f); len(obs) != 0 { + t.Fatalf("got %v", keysOf(obs)) + } +} + +// The gate: a resolver file another program rewrote waits the judging rather than failing it at the +// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own +// network fault holds the machine as a whole, as before. +func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { + since := h0 + rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, + Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} + f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} + if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { + t.Fatalf("a rewrite the send did not make: %+v", w) + } + if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || + w.on["letta"] != "" || w.waiting != "" { + t.Fatalf("a rewrite of the file a moved module owns: %+v", w) + } + own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { + t.Fatalf("the machine's own network: %+v", w) + } + unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, + Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} + if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, + open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { + t.Fatalf("a machine that cannot reach the hub: %+v", w) + } +} diff --git a/cmd/mesh-controller/machine_network_test.go b/cmd/mesh-controller/machine_network_test.go index 20bea9dc..355f384f 100644 --- a/cmd/mesh-controller/machine_network_test.go +++ b/cmd/mesh-controller/machine_network_test.go @@ -1,12 +1,6 @@ package main import ( - "encoding/json" - "os" - "strings" - "testing" - "time" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" @@ -19,16 +13,6 @@ import ( // urgent; an engine that says nothing of its network raises nothing; and the gate waits on what is shown to // be another's. -func aNetwork(state string, parts ...inventory.NetworkPart) *inventory.NetworkHealth { - for i := range parts { - if parts[i].State == "" { - parts[i].State = link.StateUnhealthy - } - parts[i].Since = h0 - } - return &inventory.NetworkHealth{State: state, Since: h0, Parts: parts} -} - func netFacts(healths map[string]*inventory.NetworkHealth) networkFacts { f := networkFacts{healths: map[string]inventory.NodeHealth{}, hub: "anchor", control: "anchor", byAddress: map[string]string{"10.77.0.1": "anchor", "10.77.0.2": "laptop", "10.77.0.3": "spare"}, @@ -57,250 +41,11 @@ func keysOf(obs []conditions.Observation) []string { return keys } -func TestAResolverFileRewrittenIsOneFindingNamingItsWriterAndWhatItCosts(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, rewrittenByVPN, namesThroughVPN), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("want one finding, the rewrite, got %v", keys) - } - o := obs[0] - for _, want := range []string{"laptop", "rewritten by another program (FortiClient)", "mesh names do not resolve", - "next reconcile"} { - if !strings.Contains(o.Summary, want) { - t.Errorf("the summary does not say %q: %s", want, o.Summary) - } - } - if strings.Contains(o.Summary, "172.16.") || strings.Contains(o.Summary, "/etc/") { - t.Errorf("an address or a path reached the summary: %s", o.Summary) - } - if !strings.Contains(o.Said, "172.16.5.5") || o.Severity != conditions.Warning || o.Machine != "laptop" { - t.Errorf("the evidence or the severity is wrong: %+v", o) - } -} - -func TestOneMachineFailingTowardAHealthyHubIsItsOwn(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateHealthy), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("want the laptop's own, got %v", keys) - } - if obs[0].Severity != conditions.Warning { - t.Fatalf("a laptop's own tunnel is a warning, got %s", obs[0].Severity) - } -} - -func TestTwoMachinesThatCannotReachTheHubAreSaidOnceAtTheHub(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - "spare": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.unreachable" { - t.Fatalf("want one condition at the hub, got %v", keys) - } - o := obs[0] - if o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop, spare") || len(o.Also) != 2 { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestWhatPointsAtASilentHubIsHeldUnderItsSilence(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, tunnelDown)}) - f.silent["anchor"] = true - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("the hub's silence says it; got %v", keysOf(obs)) - } -} - -func TestAHubWhoseOwnNetworkIsUnhealthyListsWhoCannotReachIt(t *testing.T) { - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, noRoute), - "laptop": aNetwork(link.StateUnhealthy, tunnelDown), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.anchor.network" { - t.Fatalf("want the hub's own, holding the laptop's, got %v", keys) - } - if o := obs[0]; o.Severity != conditions.Urgent || !strings.Contains(o.Summary, "laptop cannot reach it") { - t.Fatalf("the hub's condition is %+v", o) - } -} - -func TestOneMachineFailingAHealthyMeshResolverIsItsOwnAndTwoAreTheResolvers(t *testing.T) { - silentResolver := inventory.NetworkPart{Part: link.PartNames, Reason: "1 of its 2 resolvers do not answer as the mesh's do", - Said: "10.77.0.3 — no answer within 1s", Toward: []string{"10.77.0.3"}} - obs := networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateHealthy), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.laptop.network" { - t.Fatalf("one machine alone: want its own, got %v", keys) - } - obs = networkObservations(netFacts(map[string]*inventory.NetworkHealth{ - "anchor": aNetwork(link.StateUnhealthy, silentResolver), "spare": aNetwork(link.StateHealthy), - "laptop": aNetwork(link.StateUnhealthy, silentResolver), - })) - if keys := keysOf(obs); len(keys) != 1 || keys[0] != "machine.spare.unreachable" { - t.Fatalf("two machines: want it said once at the resolver's machine, got %v", keys) - } - if !strings.Contains(obs[0].Summary, "its resolver") { - t.Fatalf("the resolver's machine is said %s", obs[0].Summary) - } -} - -func TestTheControlNodeAndTheBusAreUrgent(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"anchor": aNetwork(link.StateUnhealthy, rewrittenByVPN)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the control node's rewritten file: %+v", obs) - } - bus := inventory.NetworkPart{Part: link.PartBus, Reason: "the bus cannot be reached", Said: "no link"} - f = netFacts(map[string]*inventory.NetworkHealth{"laptop": aNetwork(link.StateUnhealthy, bus, noRoute)}) - if obs := networkObservations(f); len(obs) != 1 || obs[0].Severity != conditions.Urgent { - t.Fatalf("the bus unreachable from the laptop: %+v", obs) - } -} - -func TestAnEngineThatSaysNothingOfItsNetworkRaisesNothing(t *testing.T) { - f := netFacts(map[string]*inventory.NetworkHealth{"laptop": nil, "anchor": aNetwork(link.StateHealthy)}) - if obs := networkObservations(f); len(obs) != 0 { - t.Fatalf("got %v", keysOf(obs)) - } -} - // Through the store and the keeper: the statement kept, the rewrite raised from it, cleared when the file // is written back, and node show saying it. -func TestARewrittenResolverFileIsRaisedFromTheStatementAndClearedWhenWrittenBack(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - say := func(at time.Time, n *link.NetworkHealth) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, Network: n}, at); err != nil { - t.Fatal(err) - } - } - openKeys := func() []string { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - var keys []string - for _, c := range list { - keys = append(keys, c.Key) - } - return keys - } - healthy := &link.NetworkHealth{State: link.StateHealthy, Since: h0, Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateHealthy, Since: h0}}} - rewritten := &link.NetworkHealth{State: link.StateUnhealthy, Since: h0.Add(time.Minute), Parts: []link.NetworkPart{ - {Part: link.PartResolvConf, State: link.StateUnhealthy, Reason: rewrittenByVPN.Reason, Said: rewrittenByVPN.Said, - Writer: "FortiClient", Owner: "networkmanager", Since: h0.Add(time.Minute)}, - {Part: link.PartNames, State: link.StateUnhealthy, Reason: namesThroughVPN.Reason, Said: namesThroughVPN.Said, - Toward: namesThroughVPN.Toward, Since: h0.Add(time.Minute)}}} - - say(h0, healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("a healthy network raised %v", keys) - } - say(h0.Add(time.Minute), rewritten) - if keys := openKeys(); len(keys) != 1 || keys[0] != "machine.laptop.networkmanager.rewritten" { - t.Fatalf("the rewrite raised %v", keys) - } - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Network == nil || kept.Network.Parts[0].Writer != "FortiClient" { - t.Fatalf("the statement's network was not kept: %+v %v", kept.Network, err) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "FortiClient") || - !strings.Contains(lines, "unhealthy resolv-conf") { - t.Fatalf("node show says:\n%s", lines) - } - say(h0.Add(2*time.Minute), healthy) - if keys := openKeys(); len(keys) != 0 { - t.Fatalf("written back, still open: %v", keys) - } - // An engine older than the judging says no network: nothing raised, and node show says it is not known. - say(h0.Add(3*time.Minute), nil) - kept, had, _ = inv.HealthOf(ctx, "laptop") - if keys := openKeys(); len(keys) != 0 || kept.Network != nil { - t.Fatalf("an older engine: %v %+v", keys, kept.Network) - } - if lines := strings.Join(networkLines(kept, had, h0), "\n"); !strings.Contains(lines, "older than that judging") { - t.Fatalf("node show says:\n%s", lines) - } -} - -// The gate: a resolver file another program rewrote waits the judging rather than failing it at the -// bound; the same, when the send moved the module whose file it is, is that module's; a machine's own -// network fault holds the machine as a whole, as before. -func TestTheGateWaitsOnARewriteItDidNotMakeAndHoldsTheOwnerOnOneItMoved(t *testing.T) { - since := h0 - rewrite := conditions.Condition{Key: "machine.laptop.networkmanager.rewritten", Kind: kindNetworkRewritten, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop.networkmanager", Machine: "laptop"}, - Summary: "the resolver file was rewritten", Raised: since.Add(time.Minute), Source: sourceNetwork} - f := gateFacts{judged: true, open: []conditions.Condition{rewrite}} - if w := aboutTheMachine("laptop", []string{"letta"}, since, f); w.waiting == "" || w.whole != "" || len(w.on) != 0 { - t.Fatalf("a rewrite the send did not make: %+v", w) - } - if w := aboutTheMachine("laptop", []string{"networkmanager", "letta"}, since, f); w.on["networkmanager"] == "" || - w.on["letta"] != "" || w.waiting != "" { - t.Fatalf("a rewrite of the file a moved module owns: %+v", w) - } - own := conditions.Condition{Key: "machine.laptop.network", Kind: kindMachineNetwork, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's network is not healthy", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{own}}); w.whole == "" || w.waiting != "" { - t.Fatalf("the machine's own network: %+v", w) - } - unreachable := conditions.Condition{Key: "machine.anchor.unreachable", Kind: kindNetworkUnreachable, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "anchor", Machine: "anchor", Also: []string{"laptop", "spare"}}, - Summary: "anchor cannot be reached", Raised: since.Add(time.Minute), Source: sourceNetwork} - if w := aboutTheMachine("laptop", []string{"letta"}, since, gateFacts{judged: true, - open: []conditions.Condition{unreachable}}); w.waiting == "" || w.whole != "" { - t.Fatalf("a machine that cannot reach the hub: %+v", w) - } -} // TestTheDrillsStatementsRaiseAndClearTheRewrite replays the drill of ADR 0241 (mesh-host // internal/network TestDrill…): what a node-engine said in a throwaway container while its resolver file // was declared, rewritten as a VPN client rewrites it, and written back — recorded, with the mesh's names // and addresses replaced by this test mesh's. Healthy raises nothing; the rewrite, on its second look, is // raised as one finding naming the writer; written back, it clears. -func TestTheDrillsStatementsRaiseAndClearTheRewrite(t *testing.T) { - raw, err := os.ReadFile("testdata/network-drill.json") - if err != nil { - t.Fatal(err) - } - var said []link.Health - if err := json.Unmarshal(raw, &said); err != nil { - t.Fatal(err) - } - open := aMesh(t) - ctx := t.Context() - k := conditionsFrom - var raisedAt []int - for i, h := range said { - if err := stateHealth(ctx, open.inventory, k, "laptop", h, h.At); err != nil { - t.Fatal(err) - } - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - for _, c := range list { - if c.Key != "machine.laptop.networkmanager.rewritten" || !strings.Contains(c.Summary, "(FortiClient)") { - t.Fatalf("statement %d raised %s: %s", i, c.Key, c.Summary) - } - raisedAt = append(raisedAt, i) - } - } - // Five statements: healthy, healthy, one failing look (still healthy), unhealthy, written back. - if len(raisedAt) != 1 || raisedAt[0] != 3 { - t.Fatalf("the rewrite was open after statements %v; want after the fourth alone", raisedAt) - } -} diff --git a/cmd/mesh-controller/machine_units_integration_test.go b/cmd/mesh-controller/machine_units_integration_test.go new file mode 100644 index 00000000..fd9017e0 --- /dev/null +++ b/cmd/mesh-controller/machine_units_integration_test.go @@ -0,0 +1,125 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv, k := open.inventory, conditionsFrom + openConditions := func() map[string]conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + out := map[string]conditions.Condition{} + for _, c := range list { + out[c.Key] = c + } + return out + } + say := func(at time.Time, h link.Health) { + t.Helper() + if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { + t.Fatal(err) + } + } + + say(h0, desktop(h0)) + got := openConditions() + machine, raised := got["machine.laptop.units"] + if !raised || len(got) != 1 { + t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) + } + for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { + if !strings.Contains(machine.Summary, unit) { + t.Errorf("the finding names %s: %s", unit, machine.Summary) + } + } + if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { + t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) + } + + say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) + got = openConditions() + module, raised := got["module.openrazer.laptop.unhealthy"] + if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || + !strings.Contains(module.Summary, "account's own service manager") { + t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) + } + + kept, had, err := inv.HealthOf(ctx, "laptop") + if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { + t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) + } + lines := strings.Join(unitsLines(kept, had, h0), "\n") + for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", + "openrazer-daemon.service — openrazer's"} { + if !strings.Contains(lines, want) { + t.Errorf("node show does not say %q:\n%s", want, lines) + } + } + + // An older engine says nothing of its units: nothing cleared on its word. + older := desktop(h0.Add(2 * time.Minute)) + older.Units = nil + say(h0.Add(2*time.Minute), older) + if _, still := openConditions()["machine.laptop.units"]; !still { + t.Fatal("a statement saying nothing of units cleared the machine's finding") + } + + // The operator mends the mounts and removes the leftover link; the module's unit still fails. + mended := desktop(h0.Add(3 * time.Minute)) + mended.Units.Failed = []link.FailedUnit{} + say(h0.Add(3*time.Minute), mended) + got = openConditions() + if _, still := got["machine.laptop.units"]; still { + t.Fatalf("no failed unit of the machine's own, still open: %v", got) + } + if _, still := got["module.openrazer.laptop.unhealthy"]; !still { + t.Fatalf("the module's unit still fails, and its condition stands: %v", got) + } + + // And the module's unit mended: running, nothing open. + well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), + Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} + say(h0.Add(4*time.Minute), well) + if got = openConditions(); len(got) != 0 { + t.Fatalf("a running service manager leaves nothing open: %v", got) + } +} + +// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged +// user unit, the module's; two mounts of the machine's own table and a unit a removed package left +// behind, the machine's. +func desktop(at time.Time) link.Health { + return link.Health{Contract: link.ReadinessContract, At: at, + Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, + Target: "openrazer-daemon.service", State: link.StateUnhealthy, + Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, + Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ + {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, + {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, + {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, + }}} +} + +func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { + since := h0 + units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, + Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, + Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} + if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, + open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { + t.Fatalf("the machine's own failed units held a send: %+v", w) + } +} diff --git a/cmd/mesh-controller/machine_units_test.go b/cmd/mesh-controller/machine_units_test.go index 162a7ffa..e6e8146f 100644 --- a/cmd/mesh-controller/machine_units_test.go +++ b/cmd/mesh-controller/machine_units_test.go @@ -1,128 +1,6 @@ package main -import ( - "strings" - "testing" - "time" - - "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" -) - // A failed unit is never silent (novox/hq issue 315, "how it is checked"): a module's failed unit raises // the module's condition on the second statement, naming the unit; the machine's own failed units raise // one finding for the machine, listing them, cleared when none is listed; an engine that says nothing of // its units raises and clears nothing; and the gate never holds a send on the machine's own. - -// desktop is the workstation as found on 2026-10-08, with the mesh's names: the Razer daemon's packaged -// user unit, the module's; two mounts of the machine's own table and a unit a removed package left -// behind, the machine's. -func desktop(at time.Time) link.Health { - return link.Health{Contract: link.ReadinessContract, At: at, - Resources: []link.ResourceHealth{{Module: "openrazer", Resource: "openrazer.daemon", Kind: link.KindUnit, - Target: "openrazer-daemon.service", State: link.StateUnhealthy, - Reason: "failed in the account's own service manager (exit-code)", Since: h0, Streak: 2}}, - Units: &link.UnitsHealth{State: link.UnitsDegraded, Failed: []link.FailedUnit{ - {Unit: "mnt-recalbox.mount", Scope: "system", Load: "loaded", Result: "exit-code", Since: h0}, - {Unit: "storage-media.mount", Scope: "system", Load: "loaded", Result: "timeout", Since: h0}, - {Unit: "greenclip.service", Scope: "user", Load: "not-found", Result: "start-limit-hit", Since: h0}, - }}} -} - -func TestAFailedUnitIsTheModulesConditionAndTheMachinesOwnAreOneFinding(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv, k := open.inventory, conditionsFrom - openConditions := func() map[string]conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - out := map[string]conditions.Condition{} - for _, c := range list { - out[c.Key] = c - } - return out - } - say := func(at time.Time, h link.Health) { - t.Helper() - if err := stateHealth(ctx, inv, k, "laptop", h, at); err != nil { - t.Fatal(err) - } - } - - say(h0, desktop(h0)) - got := openConditions() - machine, raised := got["machine.laptop.units"] - if !raised || len(got) != 1 { - t.Fatalf("the machine's own failed units are one finding at once (the engine looked twice): %v", got) - } - for _, unit := range []string{"mnt-recalbox.mount", "storage-media.mount", "greenclip.service"} { - if !strings.Contains(machine.Summary, unit) { - t.Errorf("the finding names %s: %s", unit, machine.Summary) - } - } - if strings.Contains(machine.Summary, "openrazer") || machine.Severity != conditions.Warning { - t.Errorf("the module's unit is the module's, and the machine's a warning: %+v", machine) - } - - say(h0.Add(time.Minute), desktop(h0.Add(time.Minute))) - got = openConditions() - module, raised := got["module.openrazer.laptop.unhealthy"] - if !raised || !strings.Contains(module.Summary, "openrazer-daemon.service") || - !strings.Contains(module.Summary, "account's own service manager") { - t.Fatalf("the module's failed unit raises its condition on the second statement, naming the unit: %+v", got) - } - - kept, had, err := inv.HealthOf(ctx, "laptop") - if err != nil || !had || kept.Units == nil || len(kept.Units.Failed) != 3 { - t.Fatalf("the statement's units were not kept: %+v %v", kept.Units, err) - } - lines := strings.Join(unitsLines(kept, had, h0), "\n") - for _, want := range []string{"degraded", "storage-media.mount (system) — no module places it", - "openrazer-daemon.service — openrazer's"} { - if !strings.Contains(lines, want) { - t.Errorf("node show does not say %q:\n%s", want, lines) - } - } - - // An older engine says nothing of its units: nothing cleared on its word. - older := desktop(h0.Add(2 * time.Minute)) - older.Units = nil - say(h0.Add(2*time.Minute), older) - if _, still := openConditions()["machine.laptop.units"]; !still { - t.Fatal("a statement saying nothing of units cleared the machine's finding") - } - - // The operator mends the mounts and removes the leftover link; the module's unit still fails. - mended := desktop(h0.Add(3 * time.Minute)) - mended.Units.Failed = []link.FailedUnit{} - say(h0.Add(3*time.Minute), mended) - got = openConditions() - if _, still := got["machine.laptop.units"]; still { - t.Fatalf("no failed unit of the machine's own, still open: %v", got) - } - if _, still := got["module.openrazer.laptop.unhealthy"]; !still { - t.Fatalf("the module's unit still fails, and its condition stands: %v", got) - } - - // And the module's unit mended: running, nothing open. - well := link.Health{Contract: link.ReadinessContract, At: h0.Add(4 * time.Minute), - Units: &link.UnitsHealth{State: link.UnitsRunning, Failed: []link.FailedUnit{}}} - say(h0.Add(4*time.Minute), well) - if got = openConditions(); len(got) != 0 { - t.Fatalf("a running service manager leaves nothing open: %v", got) - } -} - -func TestTheGateNeverHoldsASendOnTheMachinesOwnUnits(t *testing.T) { - since := h0 - units := conditions.Condition{Key: "machine.laptop.units", Kind: kindMachineUnits, - Subject: conditions.Subject{Scope: conditions.ScopeMachine, ID: "laptop", Machine: "laptop"}, - Summary: "laptop's service manager is degraded", Raised: since.Add(time.Minute), Source: sourceUnits} - if w := aboutTheMachine("laptop", []string{"openrazer"}, since, gateFacts{judged: true, - open: []conditions.Condition{units}}); w.whole != "" || w.waiting != "" || len(w.on) != 0 { - t.Fatalf("the machine's own failed units held a send: %+v", w) - } -} diff --git a/cmd/mesh-controller/merge_check_nested_test.go b/cmd/mesh-controller/merge_check_nested_test.go index e8888954..7dd78da6 100644 --- a/cmd/mesh-controller/merge_check_nested_test.go +++ b/cmd/mesh-controller/merge_check_nested_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/merge_gate_handing_integration_test.go b/cmd/mesh-controller/merge_gate_handing_integration_test.go new file mode 100644 index 00000000..74720f4c --- /dev/null +++ b/cmd/mesh-controller/merge_gate_handing_integration_test.go @@ -0,0 +1,63 @@ +//go:build integration + +package main + +import ( + "os" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + said := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) + for node, contract := range map[string]int{"anchor": link.HandedContract, "laptop": link.HandedContract - 1} { + if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: contract, + SaidAt: said, HeardAt: said}); err != nil { + t.Fatal(err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + if len(f.Machines) != 2 { + t.Fatalf("the snapshot holds %d machines", len(f.Machines)) + } + want := map[string]bool{} + for _, m := range f.Machines { + if m.Contract == 0 { + t.Fatalf("%s: the snapshot carries no contract", m.Name) + } + want[m.Name] = m.Contract >= link.HandedContract + } + handing := 0 + for _, hands := range want { + if hands { + handing++ + } + } + if handing != 1 { + t.Fatalf("one machine takes handed files in the snapshot, got %v", want) + } + + raised, drop, err := throwawayStores(ctx, os.Getenv("MESH_TEST_POSTGRES")) + if err != nil { + t.Fatal(err) + } + defer drop() + if _, err := raiseFromFacts(ctx, raised, f, map[string]catalogue.Manifest{}, map[string]inventory.Source{}); err != nil { + t.Fatal(err) + } + for name, hands := range want { + got, err := engineTakesHanded(ctx, raised.inventory, name) + if err != nil || got != hands { + t.Errorf("%s: the gate raises it as taking handed files = %v (%v), the mesh says %v", name, got, err, hands) + } + } +} diff --git a/cmd/mesh-controller/merge_gate_handing_test.go b/cmd/mesh-controller/merge_gate_handing_test.go index a5840284..4dfa0c91 100644 --- a/cmd/mesh-controller/merge_gate_handing_test.go +++ b/cmd/mesh-controller/merge_gate_handing_test.go @@ -1,13 +1,10 @@ package main import ( - "os" "testing" "time" - "github.com/novox/mesh-controller/internal/catalogue" snapshot "github.com/novox/mesh-controller/internal/facts" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -34,52 +31,3 @@ func TestIssue969TheGateGivesAMachineItsContractSoItTakesHandedFiles(t *testing. // gate raises it as taking handed files, and one whose statement is below it is raised as not taking them. Run in // the merge check with the other tests that need a store; the composition itself, handed here and refused there, is // the catalogue's TestAHandedModuleNamingARunTimeBoundKeyComposesOnlyWhereItIsHanded. -func TestIssue969TheGateRaisesEachMachineAsItsEngineStatedItself(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - said := time.Date(2026, 10, 16, 12, 0, 0, 0, time.UTC) - for node, contract := range map[string]int{"anchor": link.HandedContract, "laptop": link.HandedContract - 1} { - if _, err := open.inventory.RecordHealth(ctx, inventory.NodeHealth{Node: node, Contract: contract, - SaidAt: said, HeardAt: said}); err != nil { - t.Fatal(err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - if len(f.Machines) != 2 { - t.Fatalf("the snapshot holds %d machines", len(f.Machines)) - } - want := map[string]bool{} - for _, m := range f.Machines { - if m.Contract == 0 { - t.Fatalf("%s: the snapshot carries no contract", m.Name) - } - want[m.Name] = m.Contract >= link.HandedContract - } - handing := 0 - for _, hands := range want { - if hands { - handing++ - } - } - if handing != 1 { - t.Fatalf("one machine takes handed files in the snapshot, got %v", want) - } - - raised, drop, err := throwawayStores(ctx, os.Getenv("MESH_TEST_POSTGRES")) - if err != nil { - t.Fatal(err) - } - defer drop() - if _, err := raiseFromFacts(ctx, raised, f, map[string]catalogue.Manifest{}, map[string]inventory.Source{}); err != nil { - t.Fatal(err) - } - for name, hands := range want { - got, err := engineTakesHanded(ctx, raised.inventory, name) - if err != nil || got != hands { - t.Errorf("%s: the gate raises it as taking handed files = %v (%v), the mesh says %v", name, got, err, hands) - } - } -} diff --git a/cmd/mesh-controller/merge_gate_integration_test.go b/cmd/mesh-controller/merge_gate_integration_test.go new file mode 100644 index 00000000..57ade01d --- /dev/null +++ b/cmd/mesh-controller/merge_gate_integration_test.go @@ -0,0 +1,528 @@ +//go:build integration + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "objects": `{"module":"objects","version":"1", + "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], + "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, + "resolver": `{"module":"resolver","version":"1", + "provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`, + "networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`, + "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + } + for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"}, + {"laptop", "album"}, {"laptop", "lemurs"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict { + t.Helper() + admin := os.Getenv("MESH_TEST_POSTGRES") + in := mergeCheckInput{facts: f, admin: admin, changed: changed} + if tree != "" { + in.repository, in.tree = "novox/mesh-catalog", tree + } + v, err := judgeChange(t.Context(), in) + if err != nil { + t.Fatal(err) + } + return v +} + +func TestTheMeshAsItIsPassesTheGate(t *testing.T) { + f, manifests := catalogueMesh(t) + for _, m := range f.Machines { + if !m.Declaration.Composes { + t.Fatalf("the mesh itself does not compose: %+v", m.Declaration) + } + } + v := gateJudged(t, f, aTree(t, manifests)) + if v.Verdict != "pass" { + t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report()) + } + for _, m := range v.Machines { + if !m.Base.Composes || !m.Change.Composes { + t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change) + } + } +} + +func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) { + f, manifests := catalogueMesh(t) + tree := aTree(t, withEdit(manifests, "networkmanager", + `{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`)) + v := gateJudged(t, f, tree) + if v.Verdict != "fail" { + t.Fatalf("the overflow passed the gate:\n%s", v.Report()) + } + report := v.Report() + if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") { + t.Errorf("the refusal does not name the module and the provision:\n%s", report) + } +} + +func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) { + f, manifests := catalogueMesh(t) + broken := `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}` + v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") { + t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report()) + } + laptop := snapshot.Pseudonym("machine", "laptop") + if !strings.Contains(v.Report(), laptop) { + t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report()) + } + + // And as a new module, which no machine runs: tried on one that could. + newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`) + newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service") + v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") { + t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report()) + } +} + +func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) { + f, manifests := catalogueMesh(t) + v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", + `{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") { + t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report()) + } +} + +func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) { + f, manifests := catalogueMesh(t) + v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", ""))) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") { + t.Fatalf("removing a module a machine runs passed:\n%s", v.Report()) + } +} + +func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) { + f, manifests := catalogueMesh(t) + was := wideRebuild + wideRebuild = 2 + t.Cleanup(func() { wideRebuild = was }) + for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} { + v := gateJudged(t, f, aTree(t, manifests), file) + if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" { + t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict) + } + if !strings.Contains(v.Report(), "read by no module's build") { + t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report()) + } + } + v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go") + if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" { + t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict) + } + // With the reference module's definition in the tree, its directory is a module, held or not. + withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`) + v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts") + if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 { + t.Fatalf("a file of a module nobody holds reads %+v", v.Width) + } + v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json") + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" { + t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict) + } +} + +func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) { + f, manifests := catalogueMeshWithAnApp(t) + needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`) + catTree := aTree(t, needsTheApp) + alone := gateJudged(t, f, catTree, "modules/album/module.json") + if alone.Verdict != "fail" { + t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report()) + } + app := t.TempDir() + if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1", + "provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil { + t.Fatal(err) + } + in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog", + tree: catTree, changed: []string{"modules/album/module.json"}, + group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}} + together, err := judgeChange(t.Context(), in) + if err != nil { + t.Fatal(err) + } + if together.Verdict == "fail" { + t.Fatalf("the group composed together fails:\n%s", together.Report()) + } + if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" { + t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules) + } +} + +func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "objects": `{"module":"objects","version":"1", + "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], + "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, + "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + } + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +func busMesh(t *testing.T) snapshot.Facts { + t.Helper() + open := aMesh(t) + ctx := t.Context() + m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1", + "own-secrets":{"broker":"/var/lib/speaker/broker"}}`)) + if err != nil { + t.Fatal(err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "speaker"); err != nil { + t.Fatal(err) + } + user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username() + password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, + Node: "laptop", Module: "speaker"}) + if err != nil { + t.Fatal(err) + } + if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker", + `{"user":"`+user+`","password":"`+password+`"}`); err != nil { + t.Fatal(err) + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + for _, mc := range f.Machines { + if !mc.Declaration.Composes { + t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration) + } + } + return f +} + +func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) { + f := busMesh(t) + v := gateJudged(t, f, "") + if v.Verdict != "pass" { + t.Fatalf("the mesh as it is does not pass:\n%s", v.Report()) + } + for _, m := range v.Machines { + if !m.Base.Composes { + t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems) + } + } + if !strings.Contains(v.Summary, "2 of 2 compose") { + t.Errorf("the summary does not say every machine composes: %s", v.Summary) + } +} + +func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) { + f := busMesh(t) + for i := range f.Machines { + // A fact the snapshot does not carry: the module's credential, gone from the laptop's. + var kept []snapshot.Accepted + for _, a := range f.Machines[i].Accepted { + if a.Name != "broker" { + kept = append(kept, a) + } + } + f.Machines[i].Accepted = kept + } + v := gateJudged(t, f, "") + if v.Verdict != "error" { + t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report()) + } + if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") { + t.Errorf("the error does not name what could not be raised: %v", v.Errors) + } +} + +func TestAGroupsMoveOfAModuleIsNeitherTwoDefinitionsNorARemoval(t *testing.T) { + const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" + built := func(repo, path string) inventory.Source { + return inventory.Source{Repository: "http://forge.internal:20000/" + repo + ".git", Path: path, BuiltFrom: "c0ffee"} + } + sources := map[string]inventory.Source{"mover": built(catalogueRepo, "modules/mover")} + raw := `{"module":"mover","version":"1"}` + base := map[string]catalogue.Manifest{} + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatal(err) + } + base["mover"] = m + facts := snapshot.Facts{Machines: []snapshot.Machine{{Name: "anchor", Assigned: []string{"mover"}}}} + + catalogueWithout := aTree(t, map[string]string{"other": `{"module":"other","version":"1"}`}) + controllerWith := func() string { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "modules", "mover"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "modules", "mover", "module.json"), []byte(raw), 0o644); err != nil { + t.Fatal(err) + } + return dir + }() + removes := groupChange{Repository: catalogueRepo, Tree: catalogueWithout, Changed: []string{"modules/mover/module.json"}} + defines := groupChange{Repository: controllerRepo, Tree: controllerWith, Changed: []string{"modules/mover/module.json"}} + + lay := func(heads ...groupChange) (map[string]catalogue.Manifest, []string, map[string]string) { + t.Helper() + moves, err := movesInGroup(heads, sources) + if err != nil { + t.Fatal(err) + } + at := sources + if len(moves) > 0 { + at = map[string]inventory.Source{} + for k, v := range sources { + at[k] = v + } + for name, mv := range moves { + o := at[name] + o.Repository, o.Path = mv.To, mv.Dir + at[name] = o + } + } + shelf, said := base, map[string]string{} + var failures []string + for _, h := range heads { + var f []string + if shelf, f, err = shelfWithChange(shelf, at, facts, h.Repository, h.Tree, said, nil); err != nil { + t.Fatal(err) + } + failures = append(failures, f...) + } + for name := range moves { + said[name] = "moved" + } + return shelf, failures, said + } + for _, order := range [][]groupChange{{removes, defines}, {defines, removes}} { + shelf, failures, said := lay(order...) + if len(failures) > 0 { + t.Fatalf("a move of a module machines run, laid %s first, was refused: %v", order[0].Repository, failures) + } + if said["mover"] != "moved" { + t.Errorf("the move reads %v, not moved", said) + } + if _, held := shelf["mover"]; !held { + t.Errorf("the moved module is not on the shelf after laying %s first", order[0].Repository) + } + } + // Alone, a head is still refused: a second definition (issue 239), and a removal of what runs (ADR 0236). + if _, failures, _ := lay(defines); len(failures) != 1 || !strings.Contains(failures[0], "issue 239") { + t.Errorf("a definition of a module another repository builds, alone: %v", failures) + } + if _, failures, _ := lay(removes); len(failures) != 1 || !strings.Contains(failures[0], "ADR 0236") { + t.Errorf("the removal of a module machines run, alone: %v", failures) + } +} + +func TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate(t *testing.T) { + f, manifests := catalogueMeshWithAnApp(t) + catTree := aTree(t, withEdit(manifests, "album", "")) + app := t.TempDir() + // The app module is built from this repository's root, and stays there: only album moves. + if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1"}`), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(app, "modules", "album"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(app, "modules", "album", "module.json"), []byte(manifests["album"]), 0o644); err != nil { + t.Fatal(err) + } + removes := groupChange{Repository: "novox/mesh-catalog", Tree: catTree, Changed: []string{"modules/album/module.json"}} + defines := groupChange{Repository: "novox/app", Tree: app, Changed: []string{"modules/album/module.json"}} + judge := func(first groupChange, rest ...groupChange) mergeVerdict { + v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), + repository: first.Repository, tree: first.Tree, changed: first.Changed, group: rest}) + if err != nil { + t.Fatal(err) + } + return v + } + for _, v := range []mergeVerdict{judge(removes, defines), judge(defines, removes)} { + if v.Verdict == "fail" || v.Modules["album"] != "moved" { + t.Fatalf("a group moving a module machines run: %v\n%s", v.Modules, v.Report()) + } + } + if v := judge(defines); v.Verdict != "fail" { + t.Errorf("a definition of a module another repository builds passed alone:\n%s", v.Report()) + } + if v := judge(removes); v.Verdict != "fail" { + t.Errorf("the removal of a module machines run passed alone:\n%s", v.Report()) + } +} + +func TestTheGateRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) { + withContext := func(repo string) string { + return `{"module":"proxy","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile",` + + `"context":{"seat":"git","repository":"` + repo + `","ref":"main"}}]}}` + } + judge := func(repo string) []string { + t.Helper() + tree := aTree(t, map[string]string{"proxy": withContext(repo)}) + _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, map[string]inventory.Source{}, + snapshot.Facts{}, "novox/mesh-controller", tree, map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return failures + } + const want = "proxy builds artifact server from novox/mesh-catalog: a module lives whole in one repository (novox/hq ADR 0420 §1)" + failures := judge("novox/mesh-catalog") + if len(failures) != 1 || !strings.Contains(failures[0], want) { + t.Errorf("a context naming another repository: %v, want it to say %q", failures, want) + } + for _, f := range judge("novox/mesh-controller") { + if strings.Contains(f, "ADR 0420 §1") { + t.Errorf("a context naming the module's own repository was refused: %s", f) + } + } +} + +func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) { + const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" + raw := `{"module":"mover","version":"1"}` + sources := map[string]inventory.Source{"mover": {Repository: "http://forge.internal:20000/" + catalogueRepo + ".git", + Path: "modules/mover", BuiltFrom: "c0ffee"}} + // A tree is written into dir: defining mover, or holding only another module. + write := func(dir string, defines bool) string { + name, body := "other", `{"module":"other","version":"1"}` + if defines { + name, body = "mover", raw + } + if err := os.MkdirAll(filepath.Join(dir, "modules", name), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "modules", name, "module.json"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return dir + } + defining := func() string { return write(t.TempDir(), true) } + without := func() string { return write(t.TempDir(), false) } + // beside lays out a MESH_CHECK_BESIDE directory holding the named repositories' trees, each defining mover or not. + beside := func(trees map[string]bool) string { + root := t.TempDir() + for name, defines := range trees { + write(filepath.Join(root, name), defines) + } + return root + } + moves := func(repository, tree, besideRoot string) map[string]moveOfModule { + t.Helper() + got, err := movesOfChange(mergeCheckInput{repository: repository, tree: tree, beside: besideRoot}, sources) + if err != nil { + t.Fatal(err) + } + return got + } + // The controller's member: defines mover; the catalogue beside has removed it. + if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false})); got["mover"].To != controllerRepo { + t.Errorf("the controller's change defining mover, the catalogue beside without it: moves %v", got) + } + // The sources the gate then judges with: built from the controller, the definition is not a second one (issue 239); + // unmoved, it still is. + judge := func(at map[string]inventory.Source) []string { + t.Helper() + _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, at, snapshot.Facts{}, controllerRepo, defining(), + map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return failures + } + mv := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false}))["mover"] + moved := map[string]inventory.Source{"mover": {Repository: mv.To, Path: mv.Dir, BuiltFrom: "c0ffee"}} + if f := judge(moved); len(f) != 0 { + t.Errorf("a definition of a module the catalogue beside removed was refused: %v", f) + } + if f := judge(sources); len(f) != 1 || !strings.Contains(f[0], "issue 239") { + t.Errorf("with the catalogue beside at main, the definition: %v, want the issue 239 refusal", f) + } + if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": true})); len(got) != 0 { + t.Errorf("the catalogue beside at main still defines mover, yet moves %v", got) + } + // The catalogue's member: removes mover; the controller beside defines it. + if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": true})); got["mover"].To != controllerRepo { + t.Errorf("the catalogue's change removing mover, the controller beside defining it: moves %v", got) + } + if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": false})); len(got) != 0 { + t.Errorf("the controller beside at main does not define mover, yet moves %v", got) + } + if got := moves(catalogueRepo, without(), ""); len(got) != 0 { + t.Errorf("nothing beside, yet moves %v", got) + } +} diff --git a/cmd/mesh-controller/merge_gate_test.go b/cmd/mesh-controller/merge_gate_test.go index 11c304c7..7960ea21 100644 --- a/cmd/mesh-controller/merge_gate_test.go +++ b/cmd/mesh-controller/merge_gate_test.go @@ -5,11 +5,6 @@ import ( "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" - snapshot "github.com/novox/mesh-controller/internal/facts" - "github.com/novox/mesh-controller/internal/inventory" ) // The merge gate (novox/hq to-be 45 §9): a change judged against every machine of the snapshot, by the @@ -19,43 +14,6 @@ import ( // catalogueMesh is two machines and a catalogue repository: a resolver and an object store on the // anchor, and on the laptop a network manager requiring the resolver, an album requiring the object // store, and a login manager. Every module is registered from novox/mesh-catalog, as the mesh's are. -func catalogueMesh(t *testing.T) (snapshot.Facts, map[string]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - manifests := map[string]string{ - "objects": `{"module":"objects","version":"1", - "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], - "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, - "resolver": `{"module":"resolver","version":"1", - "provides":[{"name":"wildcard-resolution","scope":"mesh","identity":false}]}`, - "networkmanager": `{"module":"networkmanager","version":"1","requires":["wildcard-resolution"]}`, - "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, - "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], - "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, - } - for name, raw := range manifests { - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - } - for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "resolver"}, {"laptop", "networkmanager"}, - {"laptop", "album"}, {"laptop", "lemurs"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - return f, manifests -} // aTree is a checkout of the catalogue repository holding these manifests. func aTree(t *testing.T, manifests map[string]string) string { @@ -73,20 +31,6 @@ func aTree(t *testing.T, manifests map[string]string) string { return dir } -func gateJudged(t *testing.T, f snapshot.Facts, tree string, changed ...string) mergeVerdict { - t.Helper() - admin := os.Getenv("MESH_TEST_POSTGRES") - in := mergeCheckInput{facts: f, admin: admin, changed: changed} - if tree != "" { - in.repository, in.tree = "novox/mesh-catalog", tree - } - v, err := judgeChange(t.Context(), in) - if err != nil { - t.Fatal(err) - } - return v -} - func withEdit(manifests map[string]string, name, raw string) map[string]string { out := map[string]string{} for k, v := range manifests { @@ -101,271 +45,42 @@ func withEdit(manifests map[string]string, name, raw string) map[string]string { } // The mesh as it is passes: every machine composes in the gate's store as the controller composed it. -func TestTheMeshAsItIsPassesTheGate(t *testing.T) { - f, manifests := catalogueMesh(t) - for _, m := range f.Machines { - if !m.Declaration.Composes { - t.Fatalf("the mesh itself does not compose: %+v", m.Declaration) - } - } - v := gateJudged(t, f, aTree(t, manifests)) - if v.Verdict != "pass" { - t.Fatalf("an unchanged catalogue does not pass:\n%s", v.Report()) - } - for _, m := range v.Machines { - if !m.Base.Composes || !m.Change.Composes { - t.Errorf("%s does not compose in the gate's store as it does on the mesh: %+v / %+v", m.Described, m.Base, m.Change) - } - } -} // **Issue 263**: a change makes the network manager require the object store's provision; on a machine // whose name is six characters its identity is 26 against a bound of 20. Refused in the pull request, // naming the module, and not on the anchor after it merged. -func TestIssue263AnIdentityARealMachineNameOverflowsFailsThePullRequest(t *testing.T) { - f, manifests := catalogueMesh(t) - tree := aTree(t, withEdit(manifests, "networkmanager", - `{"module":"networkmanager","version":"1","requires":["wildcard-resolution","s3-bucket"]}`)) - v := gateJudged(t, f, tree) - if v.Verdict != "fail" { - t.Fatalf("the overflow passed the gate:\n%s", v.Report()) - } - report := v.Report() - if !strings.Contains(report, "networkmanager") || !strings.Contains(report, "s3-bucket") { - t.Errorf("the refusal does not name the module and the provision:\n%s", report) - } -} // **Issue 236**: a login manager's service that omits its state passed the catalogue check and was // refused whole by the node-engine on the first machine. A change to a module a machine runs, and a // module nobody runs yet, are both refused before merge, naming the machine and the module. -func TestIssue236AManifestTheNodeEngineRefusesFailsThePullRequest(t *testing.T) { - f, manifests := catalogueMesh(t) - broken := `{"module":"lemurs","version":"1","capabilities":["systemd"], - "resources":[{"id":"service","type":"service","unit":"lemurs.service","boot":"enabled"}]}` - v := gateJudged(t, f, aTree(t, withEdit(manifests, "lemurs", broken))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "lemurs") { - t.Fatalf("a service the node-engine refuses passed the gate:\n%s", v.Report()) - } - laptop := snapshot.Pseudonym("machine", "laptop") - if !strings.Contains(v.Report(), laptop) { - t.Errorf("the refusal does not name the machine it would be refused on:\n%s", v.Report()) - } - - // And as a new module, which no machine runs: tried on one that could. - newcomer := strings.ReplaceAll(broken, `"lemurs"`, `"greeter"`) - newcomer = strings.ReplaceAll(newcomer, "lemurs.service", "greeter.service") - v = gateJudged(t, f, aTree(t, withEdit(manifests, "greeter", newcomer))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "greeter, assigned to") { - t.Fatalf("a new module the node-engine would refuse passed the gate:\n%s", v.Report()) - } -} // **Version skew**: a manifest the controller judging it cannot read — the one the mesh runs, for a // catalogue change — fails here, not at registration after the merge. -func TestAManifestTheRunningControllerCannotReadFailsThePullRequest(t *testing.T) { - f, manifests := catalogueMesh(t) - v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", - `{"module":"album","version":"1","requires":["s3-bucket"],"a-field-of-a-newer-controller":true}`))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "refuses it") { - t.Fatalf("a manifest this controller cannot read passed:\n%s", v.Report()) - } -} // A module a machine runs, removed from its source, fails until it is unassigned (ADR 0236). -func TestAModuleAMachineRunsRemovedFromItsSourceFails(t *testing.T) { - f, manifests := catalogueMesh(t) - v := gateJudged(t, f, aTree(t, withEdit(manifests, "album", ""))) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "album is removed") { - t.Fatalf("removing a module a machine runs passed:\n%s", v.Report()) - } -} // **Issues 278 and 280**: a file in no held module's directory read as shared code, and a merge rebuilt the // catalogue. A file is a module's only by being inside it — no build reads one outside — so it rebuilds // nothing, and the gate says why; a merge that does rebuild widely is warned of. -func TestIssue278AWideRebuildIsSaidBeforeTheMerge(t *testing.T) { - f, manifests := catalogueMesh(t) - was := wideRebuild - wideRebuild = 2 - t.Cleanup(func() { wideRebuild = was }) - for _, file := range []string{"modules/showcase/index.ts", "merge-check.sh", "README.md"} { - v := gateJudged(t, f, aTree(t, manifests), file) - if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 1 || v.Verdict != "pass" { - t.Fatalf("%s, read by no build, reads %+v, %s", file, v.Width, v.Verdict) - } - if !strings.Contains(v.Report(), "read by no module's build") { - t.Errorf("why %s rebuilds nothing is not said:\n%s", file, v.Report()) - } - } - v := gateJudged(t, f, aTree(t, manifests), "modules/album/x.ts", "modules/objects/x.go", "modules/resolver/x.go") - if v.Width == nil || len(v.Width.Modules) < 3 || v.Verdict != "warning" { - t.Fatalf("a rebuild wider than the bound is not warned of: %+v, %s", v.Width, v.Verdict) - } - // With the reference module's definition in the tree, its directory is a module, held or not. - withShowcase := withEdit(manifests, "showcase", `{"module":"showcase","version":"1"}`) - v = gateJudged(t, f, aTree(t, withShowcase), "modules/showcase/index.ts") - if v.Width == nil || len(v.Width.Modules) != 0 || len(v.Width.Unread) != 0 { - t.Fatalf("a file of a module nobody holds reads %+v", v.Width) - } - v = gateJudged(t, f, aTree(t, manifests), "modules/album/module.json") - if v.Width == nil || strings.Join(v.Width.Modules, ",") != "album" || v.Verdict != "pass" { - t.Fatalf("a change to one module's directory reads %+v, %s", v.Width, v.Verdict) - } -} // **A delivery group is judged as one future state** (novox/hq ADR 0239): a catalogue change that needs a // provision only another repository's change adds fails alone and passes composed with it; and a group // whose other head breaks what the first needs fails, naming it. -func TestADeliveryGroupsHeadsAreComposedTogether(t *testing.T) { - f, manifests := catalogueMeshWithAnApp(t) - needsTheApp := withEdit(manifests, "album", `{"module":"album","version":"1","requires":["s3-bucket","app-api"]}`) - catTree := aTree(t, needsTheApp) - alone := gateJudged(t, f, catTree, "modules/album/module.json") - if alone.Verdict != "fail" { - t.Fatalf("a requirement nothing provides passed alone:\n%s", alone.Report()) - } - app := t.TempDir() - if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1", - "provides":[{"name":"app-api","scope":"mesh","identity":false}]}`), 0o644); err != nil { - t.Fatal(err) - } - in := mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), repository: "novox/mesh-catalog", - tree: catTree, changed: []string{"modules/album/module.json"}, - group: []groupChange{{Repository: "novox/app", Tree: app, Changed: []string{"module.json"}}}} - together, err := judgeChange(t.Context(), in) - if err != nil { - t.Fatal(err) - } - if together.Verdict == "fail" { - t.Fatalf("the group composed together fails:\n%s", together.Report()) - } - if together.Modules["app"] != "changed" || together.Modules["album"] != "changed" { - t.Fatalf("the group's heads were not both laid over the mesh: %v", together.Modules) - } -} // catalogueMeshWithAnApp is catalogueMesh with an application built from a repository of its own, at its // root, on the anchor. -func catalogueMeshWithAnApp(t *testing.T) (snapshot.Facts, map[string]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - manifests := map[string]string{ - "objects": `{"module":"objects","version":"1", - "provides":[{"name":"s3-bucket","scope":"mesh","identity":{"max":20,"in":"an S3 access key"}}], - "receives":{"s3-bucket":"/var/lib/mesh/objects/mesh.json"}}`, - "album": `{"module":"album","version":"1","requires":["s3-bucket"]}`, - } - for name, raw := range manifests { - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - } - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/app", BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - for _, a := range [][2]string{{"anchor", "objects"}, {"anchor", "app"}, {"laptop", "album"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - return f, manifests -} // busMesh is aMesh with a module on the bus on the laptop, its account issued as `module issue` issues // one: minted, and its credential held as the module's own secret named broker. -func busMesh(t *testing.T) snapshot.Facts { - t.Helper() - open := aMesh(t) - ctx := t.Context() - m, err := catalogue.ParseManifest([]byte(`{"module":"speaker","version":"1", - "own-secrets":{"broker":"/var/lib/speaker/broker"}}`)) - if err != nil { - t.Fatal(err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/speaker", BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "speaker"); err != nil { - t.Fatal(err) - } - user := broker.Principal{Kind: broker.KindModule, Node: "laptop", Module: "speaker"}.Username() - password, err := open.inventory.MintBusPassword(ctx, inventory.BusUser{Username: user, Kind: inventory.BusModule, - Node: "laptop", Module: "speaker"}) - if err != nil { - t.Fatal(err) - } - if err := open.inventory.AcceptSecretForModule(ctx, "laptop", "speaker", "broker", - `{"user":"`+user+`","password":"`+password+`"}`); err != nil { - t.Fatal(err) - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - for _, mc := range f.Machines { - if !mc.Declaration.Composes { - t.Fatalf("the mesh itself does not compose: %+v", mc.Declaration) - } - } - return f -} // **Issue 285**: every machine running a module on the bus failed to compose in the gate's store, with // the change and without — the store held the module's credential and no account for it, which // composition refuses (issue 203) — and the gate passed every change, "0 of 4 compose". The account the // mesh issued is raised with its credential, so the machine composes in the gate as on the mesh. -func TestIssue285AModuleOnTheBusComposesInTheGate(t *testing.T) { - f := busMesh(t) - v := gateJudged(t, f, "") - if v.Verdict != "pass" { - t.Fatalf("the mesh as it is does not pass:\n%s", v.Report()) - } - for _, m := range v.Machines { - if !m.Base.Composes { - t.Errorf("%s composes on the mesh and not in the gate: %v", m.Described, m.Base.Problems) - } - } - if !strings.Contains(v.Summary, "2 of 2 compose") { - t.Errorf("the summary does not say every machine composes: %s", v.Summary) - } -} // **Issue 285**: a machine the mesh composes that the gate cannot raise as it is leaves the change judged // against a machine that is not the mesh's — broken against broken, which passes whatever the change does. // That is an error, never a pass. -func TestIssue285AMachineTheGateCannotRaiseIsAnErrorNeverAPass(t *testing.T) { - f := busMesh(t) - for i := range f.Machines { - // A fact the snapshot does not carry: the module's credential, gone from the laptop's. - var kept []snapshot.Accepted - for _, a := range f.Machines[i].Accepted { - if a.Name != "broker" { - kept = append(kept, a) - } - } - f.Machines[i].Accepted = kept - } - v := gateJudged(t, f, "") - if v.Verdict != "error" { - t.Fatalf("a gate whose mesh does not compose said %s:\n%s", v.Verdict, v.Report()) - } - if len(v.Errors) != 1 || !strings.Contains(v.Errors[0], "speaker") { - t.Errorf("the error does not name what could not be raised: %v", v.Errors) - } -} // A path the snapshot withheld is given a path of its own where an access or a place needs one: a bare // "withheld" is no absolute path, and a machine whose setting composes on the mesh would not in the gate. @@ -439,231 +154,14 @@ func sdkPinnedByGoMod(t *testing.T) string { // neither a second definition of one module name (issue 239) nor the removal of a module machines run // (ADR 0236). Alone, each head is still refused. This lays the heads over the shelf as judgeChange does, // which needs no database. -func TestAGroupsMoveOfAModuleIsNeitherTwoDefinitionsNorARemoval(t *testing.T) { - const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" - built := func(repo, path string) inventory.Source { - return inventory.Source{Repository: "http://forge.internal:20000/" + repo + ".git", Path: path, BuiltFrom: "c0ffee"} - } - sources := map[string]inventory.Source{"mover": built(catalogueRepo, "modules/mover")} - raw := `{"module":"mover","version":"1"}` - base := map[string]catalogue.Manifest{} - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatal(err) - } - base["mover"] = m - facts := snapshot.Facts{Machines: []snapshot.Machine{{Name: "anchor", Assigned: []string{"mover"}}}} - - catalogueWithout := aTree(t, map[string]string{"other": `{"module":"other","version":"1"}`}) - controllerWith := func() string { - dir := t.TempDir() - if err := os.MkdirAll(filepath.Join(dir, "modules", "mover"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "modules", "mover", "module.json"), []byte(raw), 0o644); err != nil { - t.Fatal(err) - } - return dir - }() - removes := groupChange{Repository: catalogueRepo, Tree: catalogueWithout, Changed: []string{"modules/mover/module.json"}} - defines := groupChange{Repository: controllerRepo, Tree: controllerWith, Changed: []string{"modules/mover/module.json"}} - - lay := func(heads ...groupChange) (map[string]catalogue.Manifest, []string, map[string]string) { - t.Helper() - moves, err := movesInGroup(heads, sources) - if err != nil { - t.Fatal(err) - } - at := sources - if len(moves) > 0 { - at = map[string]inventory.Source{} - for k, v := range sources { - at[k] = v - } - for name, mv := range moves { - o := at[name] - o.Repository, o.Path = mv.To, mv.Dir - at[name] = o - } - } - shelf, said := base, map[string]string{} - var failures []string - for _, h := range heads { - var f []string - if shelf, f, err = shelfWithChange(shelf, at, facts, h.Repository, h.Tree, said, nil); err != nil { - t.Fatal(err) - } - failures = append(failures, f...) - } - for name := range moves { - said[name] = "moved" - } - return shelf, failures, said - } - for _, order := range [][]groupChange{{removes, defines}, {defines, removes}} { - shelf, failures, said := lay(order...) - if len(failures) > 0 { - t.Fatalf("a move of a module machines run, laid %s first, was refused: %v", order[0].Repository, failures) - } - if said["mover"] != "moved" { - t.Errorf("the move reads %v, not moved", said) - } - if _, held := shelf["mover"]; !held { - t.Errorf("the moved module is not on the shelf after laying %s first", order[0].Repository) - } - } - // Alone, a head is still refused: a second definition (issue 239), and a removal of what runs (ADR 0236). - if _, failures, _ := lay(defines); len(failures) != 1 || !strings.Contains(failures[0], "issue 239") { - t.Errorf("a definition of a module another repository builds, alone: %v", failures) - } - if _, failures, _ := lay(removes); len(failures) != 1 || !strings.Contains(failures[0], "ADR 0236") { - t.Errorf("the removal of a module machines run, alone: %v", failures) - } -} // A group that moves a module machines run passes the gate whole, in either order of its heads, and says the // module moved (novox/hq ADR 0420 §6, issue 1025); a head alone is still refused. Needs the test store. -func TestADeliveryGroupMovingAModuleMachinesRunPassesTheGate(t *testing.T) { - f, manifests := catalogueMeshWithAnApp(t) - catTree := aTree(t, withEdit(manifests, "album", "")) - app := t.TempDir() - // The app module is built from this repository's root, and stays there: only album moves. - if err := os.WriteFile(filepath.Join(app, "module.json"), []byte(`{"module":"app","version":"1"}`), 0o644); err != nil { - t.Fatal(err) - } - if err := os.MkdirAll(filepath.Join(app, "modules", "album"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(app, "modules", "album", "module.json"), []byte(manifests["album"]), 0o644); err != nil { - t.Fatal(err) - } - removes := groupChange{Repository: "novox/mesh-catalog", Tree: catTree, Changed: []string{"modules/album/module.json"}} - defines := groupChange{Repository: "novox/app", Tree: app, Changed: []string{"modules/album/module.json"}} - judge := func(first groupChange, rest ...groupChange) mergeVerdict { - v, err := judgeChange(t.Context(), mergeCheckInput{facts: f, admin: os.Getenv("MESH_TEST_POSTGRES"), - repository: first.Repository, tree: first.Tree, changed: first.Changed, group: rest}) - if err != nil { - t.Fatal(err) - } - return v - } - for _, v := range []mergeVerdict{judge(removes, defines), judge(defines, removes)} { - if v.Verdict == "fail" || v.Modules["album"] != "moved" { - t.Fatalf("a group moving a module machines run: %v\n%s", v.Modules, v.Report()) - } - } - if v := judge(defines); v.Verdict != "fail" { - t.Errorf("a definition of a module another repository builds passed alone:\n%s", v.Report()) - } - if v := judge(removes); v.Verdict != "fail" { - t.Errorf("the removal of a module machines run passed alone:\n%s", v.Report()) - } -} // **A module lives whole in one repository** (novox/hq ADR 0420 §1, issue 1019): an artifact whose build // context names another repository is refused by the gate, and one naming its own passes it. -func TestTheGateRefusesAnArtifactContextNamingAnotherRepository(t *testing.T) { - withContext := func(repo string) string { - return `{"module":"proxy","version":"1","build":{"artifacts":[{"name":"server","kind":"image","from":"Dockerfile",` + - `"context":{"seat":"git","repository":"` + repo + `","ref":"main"}}]}}` - } - judge := func(repo string) []string { - t.Helper() - tree := aTree(t, map[string]string{"proxy": withContext(repo)}) - _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, map[string]inventory.Source{}, - snapshot.Facts{}, "novox/mesh-controller", tree, map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - return failures - } - const want = "proxy builds artifact server from novox/mesh-catalog: a module lives whole in one repository (novox/hq ADR 0420 §1)" - failures := judge("novox/mesh-catalog") - if len(failures) != 1 || !strings.Contains(failures[0], want) { - t.Errorf("a context naming another repository: %v, want it to say %q", failures, want) - } - for _, f := range judge("novox/mesh-controller") { - if strings.Contains(f, "ADR 0420 §1") { - t.Errorf("a context naming the module's own repository was refused: %s", f) - } - } -} // **A member of a delivery group judged alone sees a move** through the core clones beside its check // (novox/hq ADR 0420 §6, issue 1043): the controller's own gate, with the catalogue beside having removed the // module, is not refused for a second definition (issue 239); the catalogue's own gate, with the controller // beside defining it, is not refused for a removal (ADR 0236). A clone beside at main finds no move. -func TestAMemberJudgedAloneSeesAMoveThroughTheClonesBeside(t *testing.T) { - const catalogueRepo, controllerRepo = "novox/mesh-catalog", "novox/mesh-controller" - raw := `{"module":"mover","version":"1"}` - sources := map[string]inventory.Source{"mover": {Repository: "http://forge.internal:20000/" + catalogueRepo + ".git", - Path: "modules/mover", BuiltFrom: "c0ffee"}} - // A tree is written into dir: defining mover, or holding only another module. - write := func(dir string, defines bool) string { - name, body := "other", `{"module":"other","version":"1"}` - if defines { - name, body = "mover", raw - } - if err := os.MkdirAll(filepath.Join(dir, "modules", name), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, "modules", name, "module.json"), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - return dir - } - defining := func() string { return write(t.TempDir(), true) } - without := func() string { return write(t.TempDir(), false) } - // beside lays out a MESH_CHECK_BESIDE directory holding the named repositories' trees, each defining mover or not. - beside := func(trees map[string]bool) string { - root := t.TempDir() - for name, defines := range trees { - write(filepath.Join(root, name), defines) - } - return root - } - moves := func(repository, tree, besideRoot string) map[string]moveOfModule { - t.Helper() - got, err := movesOfChange(mergeCheckInput{repository: repository, tree: tree, beside: besideRoot}, sources) - if err != nil { - t.Fatal(err) - } - return got - } - // The controller's member: defines mover; the catalogue beside has removed it. - if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false})); got["mover"].To != controllerRepo { - t.Errorf("the controller's change defining mover, the catalogue beside without it: moves %v", got) - } - // The sources the gate then judges with: built from the controller, the definition is not a second one (issue 239); - // unmoved, it still is. - judge := func(at map[string]inventory.Source) []string { - t.Helper() - _, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, at, snapshot.Facts{}, controllerRepo, defining(), - map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - return failures - } - mv := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": false}))["mover"] - moved := map[string]inventory.Source{"mover": {Repository: mv.To, Path: mv.Dir, BuiltFrom: "c0ffee"}} - if f := judge(moved); len(f) != 0 { - t.Errorf("a definition of a module the catalogue beside removed was refused: %v", f) - } - if f := judge(sources); len(f) != 1 || !strings.Contains(f[0], "issue 239") { - t.Errorf("with the catalogue beside at main, the definition: %v, want the issue 239 refusal", f) - } - if got := moves(controllerRepo, defining(), beside(map[string]bool{"mesh-catalog": true})); len(got) != 0 { - t.Errorf("the catalogue beside at main still defines mover, yet moves %v", got) - } - // The catalogue's member: removes mover; the controller beside defines it. - if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": true})); got["mover"].To != controllerRepo { - t.Errorf("the catalogue's change removing mover, the controller beside defining it: moves %v", got) - } - if got := moves(catalogueRepo, without(), beside(map[string]bool{"mesh-controller": false})); len(got) != 0 { - t.Errorf("the controller beside at main does not define mover, yet moves %v", got) - } - if got := moves(catalogueRepo, without(), ""); len(got) != 0 { - t.Errorf("nothing beside, yet moves %v", got) - } -} diff --git a/cmd/mesh-controller/merge_on_the_bus_test.go b/cmd/mesh-controller/merge_on_the_bus_test.go index cc84fc3f..df3825e2 100644 --- a/cmd/mesh-controller/merge_on_the_bus_test.go +++ b/cmd/mesh-controller/merge_on_the_bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/merge_window_bus.go b/cmd/mesh-controller/merge_window_bus.go index 84ff718e..5b4cb0e3 100644 --- a/cmd/mesh-controller/merge_window_bus.go +++ b/cmd/mesh-controller/merge_window_bus.go @@ -54,8 +54,8 @@ func windowClockOf() windowClock { // most, each at least a second (the least a key can live). Not armed for a batch whose window is closed already // (by a stop, by its time, or because it is cut at once) or that is no batch. func windowDeadlines(b inventory.Plan, now time.Time) (quiet, most time.Duration, armed bool) { - if b.Delivery == nil || b.Delivery.Batch == nil || windowClosed(b, now) { - return 0, 0, false + if b.Delivery == nil || b.Delivery.Batch == nil || windowClosed(b, now) || len(b.Delivery.Batch.Holds) > 0 { + return 0, 0, false // a held batch has no clock: its group's last member closes it (ADR 0418 clause 3) } // Rounded up to a whole second, at least one: the server rounds a key's time to live down to whole seconds, so // a fraction would close the key before the record's time and the look would cut nothing. diff --git a/cmd/mesh-controller/merge_window_bus_integration_test.go b/cmd/mesh-controller/merge_window_bus_integration_test.go new file mode 100644 index 00000000..5cdae1e8 --- /dev/null +++ b/cmd/mesh-controller/merge_window_bus_integration_test.go @@ -0,0 +1,90 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func windowOnABus(t *testing.T) (*busWindow, <-chan string) { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + api, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + heard := make(chan string, 16) + ctx, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch }) + if err != nil { + t.Fatal(err) + } + return w, heard +} + +func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) { + w, heard := windowOnABus(t) + if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil { + t.Fatal(err) + } + if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" { + t.Fatalf("heard %q (%v); want plan-1", b, ok) + } +} + +func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) { + w, heard := windowOnABus(t) + ctx := t.Context() + if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { + t.Fatal(err) + } + time.Sleep(1200 * time.Millisecond) + if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was + t.Fatal(err) + } + if b, ok := hears(t, heard, 1500*time.Millisecond); ok { + t.Fatalf("heard %q before the restarted window was up", b) + } + // Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s. + start := time.Now() + for time.Since(start) < 2500*time.Millisecond { + if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { + t.Fatal(err) + } + time.Sleep(700 * time.Millisecond) + } + if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" { + t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok) + } +} + +func TestForgettingABatchWakesNobody(t *testing.T) { + w, heard := windowOnABus(t) + if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil { + t.Fatal(err) + } + w.Forget(t.Context(), "plan-3") + if b, ok := hears(t, heard, 3*time.Second); ok { + t.Fatalf("a forgotten batch was heard expire: %q", b) + } +} diff --git a/cmd/mesh-controller/merge_window_bus_test.go b/cmd/mesh-controller/merge_window_bus_test.go index a2c24dc0..760a360e 100644 --- a/cmd/mesh-controller/merge_window_bus_test.go +++ b/cmd/mesh-controller/merge_window_bus_test.go @@ -1,47 +1,13 @@ package main import ( - "context" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/testbus" ) // windowOnABus is the merge window's clock on a real bus, and the batches its watcher heard expire. -func windowOnABus(t *testing.T) (*busWindow, <-chan string) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan string, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - w, err := newBusWindow(ctx, api, func(batch string) { heard <- batch }) - if err != nil { - t.Fatal(err) - } - return w, heard -} func hears(t *testing.T, heard <-chan string, within time.Duration) (string, bool) { t.Helper() @@ -55,55 +21,11 @@ func hears(t *testing.T, heard <-chan string, within time.Duration) (string, boo // A batch's quiet key expires when no merge came for the window, and the watcher names the batch (issue 973, // novox/hq ADR 0407 decision 1). -func TestTheQuietKeyExpiresAndTheWatcherNamesTheBatch(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-1", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - if b, ok := hears(t, heard, 10*time.Second); !ok || b != "plan-1" { - t.Fatalf("heard %q (%v); want plan-1", b, ok) - } -} // A merge into the batch makes the quiet key again, so the window restarts: it does not expire at the first // deadline, and the batch's most closes it all the same. -func TestAMergeRestartsTheQuietKeyAndTheMostKeyClosesAllTheSame(t *testing.T) { - w, heard := windowOnABus(t) - ctx := t.Context() - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { // a merge: quiet again, most as it was - t.Fatal(err) - } - if b, ok := hears(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("heard %q before the restarted window was up", b) - } - // Another merge keeps the window open; the most (5 s from the first) closes it at about 5 s. - start := time.Now() - for time.Since(start) < 2500*time.Millisecond { - if err := w.Arm(ctx, "plan-2", 2*time.Second, 5*time.Second); err != nil { - t.Fatal(err) - } - time.Sleep(700 * time.Millisecond) - } - if b, ok := hears(t, heard, 8*time.Second); !ok || b != "plan-2" { - t.Fatalf("the most did not close the batch: heard %q (%v)", b, ok) - } -} // A batch cut is forgotten: its keys are taken away by a delete, which is no expiry and wakes nobody. -func TestForgettingABatchWakesNobody(t *testing.T) { - w, heard := windowOnABus(t) - if err := w.Arm(t.Context(), "plan-3", time.Second, time.Minute); err != nil { - t.Fatal(err) - } - w.Forget(t.Context(), "plan-3") - if b, ok := hears(t, heard, 3*time.Second); ok { - t.Fatalf("a forgotten batch was heard expire: %q", b) - } -} // The deadlines a batch's keys are made with: what is left of its window and of its most, at least a second; // none for a batch whose window is closed or that is no batch. diff --git a/cmd/mesh-controller/mesh_for_test.go b/cmd/mesh-controller/mesh_for_test.go index eea81d13..aad02830 100644 --- a/cmd/mesh-controller/mesh_for_test.go +++ b/cmd/mesh-controller/mesh_for_test.go @@ -8,9 +8,10 @@ import ( "encoding/base64" "encoding/json" "fmt" - "github.com/novox/mesh-controller/internal/conditions" "testing" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/licences" diff --git a/cmd/mesh-controller/meshcli_integration_test.go b/cmd/mesh-controller/meshcli_integration_test.go new file mode 100644 index 00000000..94980ef7 --- /dev/null +++ b/cmd/mesh-controller/meshcli_integration_test.go @@ -0,0 +1,193 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) { + t.Setenv(echoEnvironment, "1") + t.Setenv("MESH_VERB", "leaked-from-the-serving-process") + // The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's. + t.Setenv(servedVar, "1") + ctx := context.Background() + + a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"}) + if a.Exit != 0 || a.Refused != "" || !a.Terminal { + t.Fatalf("the terminal's line did not run: %+v", a) + } + if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") || + !strings.Contains(got, "terminal=true") { + t.Fatalf("the terminal's line ran with %s", got) + } + + a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) + if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" { + t.Fatalf("an ordinary line did not run as one: %+v", a) + } + if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") { + t.Fatalf("an ordinary line ran with %s", got) + } +} + +func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) { + t.Setenv(echoEnvironment, "1") + ctx := context.Background() + ordinary := cliVerdict{why: "not the terminal"} + a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary) + if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" { + t.Fatalf("a repair without --why ran as an ordinary call: %+v", a) + } + a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary) + if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) { + t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a) + } + for _, server := range []string{"serve", "api", "board"} { + a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true}) + if a.Refused == "" || len(a.Stdout) != 0 { + t.Fatalf("%s was run for mesh-cli: %+v", server, a) + } + } + a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) + if a.Refused != "agent is not answered" || len(a.Stdout) != 0 { + t.Fatalf("a refused line ran: %+v", a) + } +} + +func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) { + t.Setenv("MESH_VERB", "assign") + ctx := context.Background() + if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil || + !strings.Contains(err.Error(), "terminal") { + t.Fatalf("assigning the controller through a verb was not refused: %v", err) + } + if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("assigning the controller through a verb was not refused: %v", err) + } + t.Setenv("MESH_VERB", "unassign") + if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { + t.Fatalf("unassigning the controller through a verb was not refused: %v", err) + } + // Another module through a verb, and the controller's at the terminal, are not refused for it. + if err := refusedMovingTheController([]string{"zsh"}); err != nil { + t.Fatalf("another module was refused: %v", err) + } + t.Setenv("MESH_VERB", "") + if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil { + t.Fatalf("the terminal was refused: %v", err) + } +} + +func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) { + t.Setenv(echoEnvironment, "1") + var said []string + was := cliJournal + cliJournal = func(line string) { said = append(said, line) } + t.Cleanup(func() { cliJournal = was }) + ctx := link.WithCallID(context.Background(), "call-1") + runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`), + cliVerdict{terminal: true, why: "the terminal"}) + runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) + all := strings.Join(said, "\n") + if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") || + !strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") { + t.Fatalf("the journal said %q", said) + } + if strings.Contains(all, "s3cret") { + t.Fatalf("the journal carries the line's values: %q", said) + } +} + +func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) { + t.Setenv(echoEnvironment, "1") + for _, line := range [][]string{ + {"node", "account", "control", "x"}, + {"node", "agent-account", "control", "x", "--clear"}, + {"token", "issue", "laptop"}, + {"secret", "export", "x"}, + {"operator", "key", "set", "x"}, + {"assign", "laptop", "zsh"}, + } { + if _, err := ordinaryLine(line); err == nil { + t.Errorf("%q composed as an ordinary line", line) + } + a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"}) + if a.Refused == "" || len(a.Stdout) != 0 { + t.Errorf("%q ran as an ordinary line: %+v", line, a) + } + } + if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" { + t.Fatalf("a read was refused: %v", err) + } +} + +func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer { + return link.CLIAnswer{Stdout: []byte("s3cret-join")} + }, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"}) + msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { + t.Fatalf("the asker was not given its answer: %s", msg.Data) + } + recent, _ := link.Calls.Recent() + var id string + for _, c := range recent { + if c.Seat == link.CLISeat { + id = c.ID + break + } + } + shown, err := callsAnswer(link.Calls, id) + if err != nil { + t.Fatal(err) + } + said, _ := json.Marshal(shown) + if strings.Contains(string(said), "s3cret-join") || + strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { + t.Fatalf("calls showed a mesh-cli line's answer: %s", said) + } +} + +func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) { + t.Setenv(echoEnvironment, "1") + t.Setenv(cliTerminalVar, "1") + t.Setenv(servedVar, "1") + for _, env := range [][]string{commandEnvironment("someone", "status", false)} { + for _, kv := range env { + if strings.HasPrefix(kv, cliTerminalVar+"=") { + t.Fatalf("a verb's line carries the terminal's mark: %s", kv) + } + } + } + a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) + if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) { + t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got) + } + a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true}) + if got := string(a.Stdout); !strings.Contains(got, "terminal=true") { + t.Fatalf("the terminal's line ran as %s", got) + } +} diff --git a/cmd/mesh-controller/meshcli_reads_integration_test.go b/cmd/mesh-controller/meshcli_reads_integration_test.go new file mode 100644 index 00000000..5a76dfd2 --- /dev/null +++ b/cmd/mesh-controller/meshcli_reads_integration_test.go @@ -0,0 +1,66 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestAReadCallWritesNoJournalLine(t *testing.T) { + said := journalled(t) + ctx := link.WithCallID(context.Background(), "call-1") + ordinary := cliVerdict{why: "not the terminal"} + for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"}, + {"settings", "show", "zsh"}, {"plans", "--json"}} { + if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" { + t.Fatalf("%q did not run: %+v", line, a) + } + } + runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"}) + if len(*said) != 0 { + t.Fatalf("read calls wrote journal lines: %q", *said) + } + runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"}) + if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) { + t.Fatalf("a call that changes something was not said: %q", *said) + } +} + +func TestAFailedOrRefusedReadIsSaid(t *testing.T) { + said := journalled(t) + runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"}) + t.Setenv(echoFails, "1") + a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"}) + if a.Exit == 0 { + t.Fatalf("the read did not fail: %+v", a) + } + if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") { + t.Fatalf("the journal said %q", *said) + } + if line := cliReads.said(readCallsEvery); line != "" { + t.Fatalf("a failed read was counted as a read: %q", line) + } +} + +func TestTheReadCallsAreCountedPerCaller(t *testing.T) { + said := journalled(t) + ordinary := cliVerdict{why: "not the terminal"} + for range 3 { + runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary) + } + runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary) + if len(*said) != 0 { + t.Fatalf("read calls wrote lines: %q", *said) + } + line := cliReads.said(readCallsEvery) + if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") { + t.Fatalf("the hourly line said %q", line) + } + if again := cliReads.said(readCallsEvery); again != "" { + t.Fatalf("an hour with no read calls said %q", again) + } +} diff --git a/cmd/mesh-controller/meshcli_reads_test.go b/cmd/mesh-controller/meshcli_reads_test.go index f56e2534..e4c82681 100644 --- a/cmd/mesh-controller/meshcli_reads_test.go +++ b/cmd/mesh-controller/meshcli_reads_test.go @@ -1,11 +1,7 @@ package main import ( - "context" - "strings" "testing" - - "github.com/novox/mesh-controller/internal/link" ) // echoFails makes the echoing test binary end with a failure, as a read that failed would. @@ -24,42 +20,8 @@ func journalled(t *testing.T) *[]string { } // **A call that only reads writes no journal line; one that changes something does** (novox/hq issue 762). -func TestAReadCallWritesNoJournalLine(t *testing.T) { - said := journalled(t) - ctx := link.WithCallID(context.Background(), "call-1") - ordinary := cliVerdict{why: "not the terminal"} - for _, line := range [][]string{{"plans"}, {"status"}, {"conditions"}, {"node", "list"}, {"builds"}, {"queue"}, - {"settings", "show", "zsh"}, {"plans", "--json"}} { - if a := runForMeshCLI(ctx, "g14", cliAsked("jochen", 1000, line...), ordinary); a.Exit != 0 || a.Refused != "" { - t.Fatalf("%q did not run: %+v", line, a) - } - } - runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "plans"), cliVerdict{terminal: true, why: "the terminal"}) - if len(*said) != 0 { - t.Fatalf("read calls wrote journal lines: %q", *said) - } - runForMeshCLI(ctx, "control", cliAsked("jochen", 1000, "assign", "g14", "zsh"), cliVerdict{terminal: true, why: "the terminal"}) - if len(*said) != 1 || !strings.Contains((*said)[0], `asked "assign"`) { - t.Fatalf("a call that changes something was not said: %q", *said) - } -} // **A refused or failed read keeps its line** (novox/hq issue 762). -func TestAFailedOrRefusedReadIsSaid(t *testing.T) { - said := journalled(t) - runForMeshCLI(context.Background(), "g14", cliAsked("root", 0, "plans"), cliVerdict{refused: "root is not answered"}) - t.Setenv(echoFails, "1") - a := runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), cliVerdict{why: "not the terminal"}) - if a.Exit == 0 { - t.Fatalf("the read did not fail: %+v", a) - } - if len(*said) != 2 || !strings.Contains((*said)[0], "refused") || !strings.Contains((*said)[1], "failed (exit 3)") { - t.Fatalf("the journal said %q", *said) - } - if line := cliReads.said(readCallsEvery); line != "" { - t.Fatalf("a failed read was counted as a read: %q", line) - } -} // **A verb is classed by what the call does, not its name** (novox/hq issue 762): the same command word reads with // some arguments and writes with others, and a line no reading form names is said. @@ -103,21 +65,3 @@ func TestAVerbIsClassedByItsArguments(t *testing.T) { } // **Once an hour the read calls are counted per caller, only if there were any** (novox/hq issue 762). -func TestTheReadCallsAreCountedPerCaller(t *testing.T) { - said := journalled(t) - ordinary := cliVerdict{why: "not the terminal"} - for range 3 { - runForMeshCLI(context.Background(), "g14", cliAsked("jochen", 1000, "plans"), ordinary) - } - runForMeshCLI(context.Background(), "ace", cliAsked("jochen", 1000, "status"), ordinary) - if len(*said) != 0 { - t.Fatalf("read calls wrote lines: %q", *said) - } - line := cliReads.said(readCallsEvery) - if !strings.Contains(line, "jochen on ace 1, jochen on g14 3") || !strings.Contains(line, "1h0m0s") { - t.Fatalf("the hourly line said %q", line) - } - if again := cliReads.said(readCallsEvery); again != "" { - t.Fatalf("an hour with no read calls said %q", again) - } -} diff --git a/cmd/mesh-controller/meshcli_stdin_test.go b/cmd/mesh-controller/meshcli_stdin_test.go index 78971630..78193011 100644 --- a/cmd/mesh-controller/meshcli_stdin_test.go +++ b/cmd/mesh-controller/meshcli_stdin_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/meshcli_test.go b/cmd/mesh-controller/meshcli_test.go index 7968f0c8..cc279de1 100644 --- a/cmd/mesh-controller/meshcli_test.go +++ b/cmd/mesh-controller/meshcli_test.go @@ -1,18 +1,11 @@ package main import ( - "context" - "encoding/base64" - "encoding/json" "strings" "testing" - "time" - - "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // echoEnvironment makes the test binary, run as a command line, say the verb and caller it was given (TestMain). @@ -63,82 +56,12 @@ func TestMeshCLIIsTheTerminalOnlyForTheControlNodesOperator(t *testing.T) { // The terminal runs its line without MESH_VERB, even where this process carries one; an ordinary call names // mesh-cli; both record who asked through mesh-cli. -func TestTheTerminalRunsWithoutAVerbAndAnOrdinaryCallNamesMeshCLI(t *testing.T) { - t.Setenv(echoEnvironment, "1") - t.Setenv("MESH_VERB", "leaked-from-the-serving-process") - // The serving controller marks itself (ADR 0266); its terminal line for mesh-cli is still the terminal's. - t.Setenv(servedVar, "1") - ctx := context.Background() - - a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true, why: "the terminal"}) - if a.Exit != 0 || a.Refused != "" || !a.Terminal { - t.Fatalf("the terminal's line did not run: %+v", a) - } - if got := string(a.Stdout); !strings.Contains(got, `verb=""`) || !strings.Contains(got, "operator through mesh-cli on control") || - !strings.Contains(got, "terminal=true") { - t.Fatalf("the terminal's line ran with %s", got) - } - - a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) - if a.Exit != 0 || a.Terminal || a.Why != "not the terminal" { - t.Fatalf("an ordinary line did not run as one: %+v", a) - } - if got := string(a.Stdout); !strings.Contains(got, `verb="mesh-cli"`) || !strings.Contains(got, "terminal=false") { - t.Fatalf("an ordinary line ran with %s", got) - } -} // An ordinary call meets every refusal of the generic command verb, and nothing runs; a server is never run. -func TestAnOrdinaryCallMeetsTheCommandVerbsRefusals(t *testing.T) { - t.Setenv(echoEnvironment, "1") - ctx := context.Background() - ordinary := cliVerdict{why: "not the terminal"} - a := runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "cleanup", "delete", "x"), ordinary) - if a.Refused == "" || len(a.Stdout) != 0 || a.Exit != 1 || a.Why != "not the terminal" { - t.Fatalf("a repair without --why ran as an ordinary call: %+v", a) - } - a = runForMeshCLI(ctx, "laptop", cliAsked("operator", 1000, "settings", "set", "claude-code", "{}"), ordinary) - if a.Refused != "" || !strings.Contains(string(a.Stdout), `verb="mesh-cli"`) { - t.Fatalf("an ordinary settings set did not run through the settings verb's path with MESH_VERB set: %+v", a) - } - for _, server := range []string{"serve", "api", "board"} { - a := runForMeshCLI(ctx, "control", cliAsked("operator", 1000, server), cliVerdict{terminal: true}) - if a.Refused == "" || len(a.Stdout) != 0 { - t.Fatalf("%s was run for mesh-cli: %+v", server, a) - } - } - a = runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) - if a.Refused != "agent is not answered" || len(a.Stdout) != 0 { - t.Fatalf("a refused line ran: %+v", a) - } -} // **Which node is the terminal does not follow a verb** (review of ADR 0272): the controller's module is assigned // and unassigned at the terminal alone, so no caller of `assign` can move the terminal to a node of its choosing. // Asked through the acts themselves, as the verbs ask them; refused before any store is touched (none is given). -func TestTheControllersModuleIsMovedAtTheTerminalAlone(t *testing.T) { - t.Setenv("MESH_VERB", "assign") - ctx := context.Background() - if _, err := assignWith(ctx, nil, "laptop", assignOptions{}, "zsh", "mesh-controller"); err == nil || - !strings.Contains(err.Error(), "terminal") { - t.Fatalf("assigning the controller through a verb was not refused: %v", err) - } - if _, err := assign(ctx, nil, "laptop", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { - t.Fatalf("assigning the controller through a verb was not refused: %v", err) - } - t.Setenv("MESH_VERB", "unassign") - if _, err := unassign(ctx, nil, "control", "mesh-controller"); err == nil || !strings.Contains(err.Error(), "terminal") { - t.Fatalf("unassigning the controller through a verb was not refused: %v", err) - } - // Another module through a verb, and the controller's at the terminal, are not refused for it. - if err := refusedMovingTheController([]string{"zsh"}); err != nil { - t.Fatalf("another module was refused: %v", err) - } - t.Setenv("MESH_VERB", "") - if err := refusedMovingTheController([]string{"mesh-controller"}); err != nil { - t.Fatalf("the terminal was refused: %v", err) - } -} // An ordinary `settings set|clear` goes down the settings verb's own path: composed as that verb composes it, and // run with MESH_VERB set, so its refusals — the terminal-only keys among them — are the settings command's own, @@ -177,117 +100,17 @@ func TestAnOrdinarySettingsLineTakesTheSettingsVerbsPath(t *testing.T) { // Every line is said in the controller's journal, with its call, who asked where and how it ran — its command word // only, never the rest of the line (review of ADR 0272). -func TestEveryMeshCLILineIsSaidInTheJournal(t *testing.T) { - t.Setenv(echoEnvironment, "1") - var said []string - was := cliJournal - cliJournal = func(line string) { said = append(said, line) } - t.Cleanup(func() { cliJournal = was }) - ctx := link.WithCallID(context.Background(), "call-1") - runForMeshCLI(ctx, "control", cliAsked("operator", 1000, "settings", "set", "x", `{"password":"s3cret"}`), - cliVerdict{terminal: true, why: "the terminal"}) - runForMeshCLI(ctx, "control", cliAsked("agent", 1001, "status"), cliVerdict{refused: "agent is not answered"}) - all := strings.Join(said, "\n") - if len(said) != 2 || !strings.Contains(all, "call-1") || !strings.Contains(all, "operator on control") || - !strings.Contains(all, "as the controller's terminal") || !strings.Contains(all, "refused") { - t.Fatalf("the journal said %q", said) - } - if strings.Contains(all, "s3cret") { - t.Fatalf("the journal carries the line's values: %q", said) - } -} // **An ordinary line runs only what the generic command verb would** (review of ADR 0272, ADR 0266): its reading // forms, and never a command that is the terminal's alone. Each of these, from another node's operator, is refused // and runs nothing. -func TestAnOrdinaryLineRunsNothingTheCommandVerbWouldRefuse(t *testing.T) { - t.Setenv(echoEnvironment, "1") - for _, line := range [][]string{ - {"node", "account", "control", "x"}, - {"node", "agent-account", "control", "x", "--clear"}, - {"token", "issue", "laptop"}, - {"secret", "export", "x"}, - {"operator", "key", "set", "x"}, - {"assign", "laptop", "zsh"}, - } { - if _, err := ordinaryLine(line); err == nil { - t.Errorf("%q composed as an ordinary line", line) - } - a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, line...), cliVerdict{why: "not the terminal"}) - if a.Refused == "" || len(a.Stdout) != 0 { - t.Errorf("%q ran as an ordinary line: %+v", line, a) - } - } - if argv, err := ordinaryLine([]string{"status"}); err != nil || argv[0] != "status" { - t.Fatalf("a read was refused: %v", err) - } -} // **`calls` never shows a mesh-cli line's answer** (review of ADR 0272): the verb's own answer is read for a line // served over a bus, and neither the answer's text nor the base64 JSON writes its bytes in is there. -func TestTheCallsVerbNeverShowsAMeshCLILinesAnswer(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - stop, err := link.OverNATS{Conn: conn}.ServeCLI(func(context.Context, string, link.CLIAsked) link.CLIAnswer { - return link.CLIAnswer{Stdout: []byte("s3cret-join")} - }, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - body, _ := json.Marshal(link.CLIAsked{Line: []string{"token", "issue", "x"}, Account: "operator"}) - msg, err := conn.Request(link.CLISubject("control"), body, 5*time.Second) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(msg.Data), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { - t.Fatalf("the asker was not given its answer: %s", msg.Data) - } - recent, _ := link.Calls.Recent() - var id string - for _, c := range recent { - if c.Seat == link.CLISeat { - id = c.ID - break - } - } - shown, err := callsAnswer(link.Calls, id) - if err != nil { - t.Fatal(err) - } - said, _ := json.Marshal(shown) - if strings.Contains(string(said), "s3cret-join") || - strings.Contains(string(said), base64.StdEncoding.EncodeToString([]byte("s3cret-join"))) { - t.Fatalf("calls showed a mesh-cli line's answer: %s", said) - } -} // **Only the terminal's line carries the terminal's mark** (review of ADR 0272): a mark the serving controller's // environment holds — leaked, or set by anything — is stripped from every other command line it runs, a verb's and // an ordinary mesh-cli line alike, so neither reads as the terminal. -func TestTheTerminalsMarkIsStrippedFromEveryOtherLine(t *testing.T) { - t.Setenv(echoEnvironment, "1") - t.Setenv(cliTerminalVar, "1") - t.Setenv(servedVar, "1") - for _, env := range [][]string{commandEnvironment("someone", "status", false)} { - for _, kv := range env { - if strings.HasPrefix(kv, cliTerminalVar+"=") { - t.Fatalf("a verb's line carries the terminal's mark: %s", kv) - } - } - } - a := runForMeshCLI(context.Background(), "laptop", cliAsked("operator", 1000, "status"), cliVerdict{why: "not the terminal"}) - if got := string(a.Stdout); !strings.Contains(got, "terminal=false") || !strings.Contains(got, `verb="mesh-cli"`) { - t.Fatalf("an ordinary line with the mark in the serving environment ran as %s", got) - } - a = runForMeshCLI(context.Background(), "control", cliAsked("operator", 1000, "status"), cliVerdict{terminal: true}) - if got := string(a.Stdout); !strings.Contains(got, "terminal=true") { - t.Fatalf("the terminal's line ran as %s", got) - } -} // **Only a login session is the terminal** (review of ADR 0272): the operator's account on the control-node, asking // from a service — the tool runner, a user unit — and not a login session, is an ordinary call. diff --git a/cmd/mesh-controller/mirrors_integration_test.go b/cmd/mesh-controller/mirrors_integration_test.go new file mode 100644 index 00000000..b2e5cfc7 --- /dev/null +++ b/cmd/mesh-controller/mirrors_integration_test.go @@ -0,0 +1,127 @@ +//go:build integration + +package main + +import ( + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestRecordingACopyTakesWhatTheStoreHoldsAndDeletesNothing(t *testing.T) { + inv := inventory.ForTest(t) + held := ref("route-proxy", "on-go_base", 1) + absent := ref("route-proxy", "on-go_base", 2) + known := ref("nats", "on-nats_base", 3) + notACopy := ref("route-proxy", "server", 4) + if err := inv.RecordMirrored(t.Context(), "b1", "", []string{known}); err != nil { + t.Fatal(err) + } + store, asked := heldStore(t, fmt.Sprintf("%064x", 1)) + given := []string{held, absent, known, notACopy} + mirrored, err := inv.Mirrored(t.Context()) + if err != nil { + t.Fatal(err) + } + + // A dry run says, and records nothing. + dry, err := recordMirrors(t.Context(), inv, store, given, mirrored, "", false) + if err != nil { + t.Fatal(err) + } + if !dry.DryRun || !slices.Equal(dry.Recorded, []string{held}) || !slices.Equal(dry.AlreadyRecorded, []string{known}) || + len(dry.Refused) != 2 || dry.Refused[absent] == "" || dry.Refused[notACopy] == "" { + t.Fatalf("a dry run answered %+v", dry) + } + if !strings.Contains(dry.Then, "would become eligible") || !strings.Contains(dry.Then, "next sweep") { + t.Fatalf("a dry run did not say what recording does: %q", dry.Then) + } + if again, _ := inv.Mirrored(t.Context()); again[held] { + t.Fatal("a dry run recorded a copy") + } + + // A real one records what the store holds, with the person's why, and the sweep may now decide. + real, err := recordMirrors(t.Context(), inv, store, given, mirrored, "copied before copies were recorded", true) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(real.Recorded, []string{held}) { + t.Fatalf("recorded %v", real.Recorded) + } + left, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(left, held) { + t.Fatalf("a recorded copy no kept build stood on is not offered to collect: %v", left) + } + for _, r := range *asked { + if !strings.HasPrefix(r, "HEAD ") { + t.Fatalf("recording a copy asked the store %s", r) + } + } +} + +func TestAConfirmedCollectLetsAnIndexGoWithItsOwnPlatformsOnly(t *testing.T) { + inv := inventory.ForTest(t) + _, eligible, reg := twoCopies(t, inv) + store := reg.serve(t) + references, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(references, []string{eligible}) { + t.Fatalf("offered %v, want the failed build's copy", references) + } + a := runCollect(t.Context(), inv, store, references, nil, true, + sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) + if !slices.Equal(a.LetGo, []string{eligible}) || a.Stopped != "" { + t.Fatalf("let go %v, stopped %q", a.LetGo, a.Stopped) + } + if !slices.Equal(reg.deleted, []string{copyDigest(13), copyDigest(2)}) { + t.Fatalf("deleted %v; want its own platform, then the index", reg.deleted) + } +} + +func TestTheSweepAfterABuildLeavesAnIndexForAConfirmedCollect(t *testing.T) { + inv := inventory.ForTest(t) + _, eligible, reg := twoCopies(t, inv) + reg.held[copyDigest(5)] = true + image := catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@" + copyDigest(5) + store := reg.serve(t) + r := sweep(t.Context(), inv, store, []string{eligible, image}, nil, afterBuild) + if r.Indexes != 1 || !slices.Equal(r.LetGo, []string{image}) || !slices.Equal(reg.deleted, []string{copyDigest(5)}) { + t.Fatalf("after a build: %d indexes left, let go %v, deleted %v; want the index left and the image gone", + r.Indexes, r.LetGo, reg.deleted) + } + left, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(left, []string{eligible}) { + t.Fatalf("after the sweep the records offer %v; want the index still eligible", left) + } + a := runCollect(t.Context(), inv, store, left, nil, true, + sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) + if !slices.Equal(a.LetGo, []string{eligible}) || + !slices.Equal(reg.deleted, []string{copyDigest(5), copyDigest(13), copyDigest(2)}) { + t.Fatalf("a confirmed collect let go %v, deleted %v; want the index with its own platform", a.LetGo, reg.deleted) + } +} + +func TestASpareListThatCannotBeReadStopsTheSweepBeforeAnyDelete(t *testing.T) { + inv := inventory.ForTest(t) + _, eligible, reg := twoCopies(t, inv) + reg.fail[copyDigest(1)] = true + store := reg.serve(t) + a := runCollect(t.Context(), inv, store, []string{eligible}, nil, true, + sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) + if len(a.LetGo) != 0 || len(reg.deleted) != 0 || !strings.Contains(a.Stopped, "nothing was let go") { + t.Fatalf("let go %v, deleted %v, stopped %q", a.LetGo, reg.deleted, a.Stopped) + } +} diff --git a/cmd/mesh-controller/mirrors_test.go b/cmd/mesh-controller/mirrors_test.go index de85038c..fee175df 100644 --- a/cmd/mesh-controller/mirrors_test.go +++ b/cmd/mesh-controller/mirrors_test.go @@ -8,7 +8,6 @@ import ( "strings" "sync" "testing" - "time" "github.com/novox/mesh-controller/internal/artifacts" "github.com/novox/mesh-controller/internal/catalogue" @@ -88,60 +87,6 @@ func heldStore(t *testing.T, holds ...string) (artifacts.Store, *[]string) { return artifacts.Store{Address: strings.TrimPrefix(server.URL, "http://")}, &asked } -func TestRecordingACopyTakesWhatTheStoreHoldsAndDeletesNothing(t *testing.T) { - inv := inventory.ForTest(t) - held := ref("route-proxy", "on-go_base", 1) - absent := ref("route-proxy", "on-go_base", 2) - known := ref("nats", "on-nats_base", 3) - notACopy := ref("route-proxy", "server", 4) - if err := inv.RecordMirrored(t.Context(), "b1", "", []string{known}); err != nil { - t.Fatal(err) - } - store, asked := heldStore(t, fmt.Sprintf("%064x", 1)) - given := []string{held, absent, known, notACopy} - mirrored, err := inv.Mirrored(t.Context()) - if err != nil { - t.Fatal(err) - } - - // A dry run says, and records nothing. - dry, err := recordMirrors(t.Context(), inv, store, given, mirrored, "", false) - if err != nil { - t.Fatal(err) - } - if !dry.DryRun || !slices.Equal(dry.Recorded, []string{held}) || !slices.Equal(dry.AlreadyRecorded, []string{known}) || - len(dry.Refused) != 2 || dry.Refused[absent] == "" || dry.Refused[notACopy] == "" { - t.Fatalf("a dry run answered %+v", dry) - } - if !strings.Contains(dry.Then, "would become eligible") || !strings.Contains(dry.Then, "next sweep") { - t.Fatalf("a dry run did not say what recording does: %q", dry.Then) - } - if again, _ := inv.Mirrored(t.Context()); again[held] { - t.Fatal("a dry run recorded a copy") - } - - // A real one records what the store holds, with the person's why, and the sweep may now decide. - real, err := recordMirrors(t.Context(), inv, store, given, mirrored, "copied before copies were recorded", true) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(real.Recorded, []string{held}) { - t.Fatalf("recorded %v", real.Recorded) - } - left, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(left, held) { - t.Fatalf("a recorded copy no kept build stood on is not offered to collect: %v", left) - } - for _, r := range *asked { - if !strings.HasPrefix(r, "HEAD ") { - t.Fatalf("recording a copy asked the store %s", r) - } - } -} - // indexRegistry holds indexes and platforms of one image's repository, answers GETs with their // documents, refuses GETs for the digests in fail, and records every delete. type indexRegistry struct { @@ -234,65 +179,9 @@ func twoCopies(t *testing.T, inv *inventory.Inventory) (kept, eligible string, r // Through the records: a confirmed collect lets the eligible index go with the platform only it names, // and leaves the platform the kept index names. -func TestAConfirmedCollectLetsAnIndexGoWithItsOwnPlatformsOnly(t *testing.T) { - inv := inventory.ForTest(t) - _, eligible, reg := twoCopies(t, inv) - store := reg.serve(t) - references, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(references, []string{eligible}) { - t.Fatalf("offered %v, want the failed build's copy", references) - } - a := runCollect(t.Context(), inv, store, references, nil, true, - sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) - if !slices.Equal(a.LetGo, []string{eligible}) || a.Stopped != "" { - t.Fatalf("let go %v, stopped %q", a.LetGo, a.Stopped) - } - if !slices.Equal(reg.deleted, []string{copyDigest(13), copyDigest(2)}) { - t.Fatalf("deleted %v; want its own platform, then the index", reg.deleted) - } -} // The sweep after a build leaves an eligible index in place and eligible: let go of alone, its // platforms would stay for ever under a record that says collected. A later collect a person confirms // takes it with the platform only it names. An image the same sweep reaches is let go of as before. -func TestTheSweepAfterABuildLeavesAnIndexForAConfirmedCollect(t *testing.T) { - inv := inventory.ForTest(t) - _, eligible, reg := twoCopies(t, inv) - reg.held[copyDigest(5)] = true - image := catalogue.ArtifactStoreScheme + "upstream/docker.io/library/golang@" + copyDigest(5) - store := reg.serve(t) - r := sweep(t.Context(), inv, store, []string{eligible, image}, nil, afterBuild) - if r.Indexes != 1 || !slices.Equal(r.LetGo, []string{image}) || !slices.Equal(reg.deleted, []string{copyDigest(5)}) { - t.Fatalf("after a build: %d indexes left, let go %v, deleted %v; want the index left and the image gone", - r.Indexes, r.LetGo, reg.deleted) - } - left, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(left, []string{eligible}) { - t.Fatalf("after the sweep the records offer %v; want the index still eligible", left) - } - a := runCollect(t.Context(), inv, store, left, nil, true, - sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) - if !slices.Equal(a.LetGo, []string{eligible}) || - !slices.Equal(reg.deleted, []string{copyDigest(5), copyDigest(13), copyDigest(2)}) { - t.Fatalf("a confirmed collect let go %v, deleted %v; want the index with its own platform", a.LetGo, reg.deleted) - } -} // A kept index the store will not answer for stops a confirmed collect before its first delete. -func TestASpareListThatCannotBeReadStopsTheSweepBeforeAnyDelete(t *testing.T) { - inv := inventory.ForTest(t) - _, eligible, reg := twoCopies(t, inv) - reg.fail[copyDigest(1)] = true - store := reg.serve(t) - a := runCollect(t.Context(), inv, store, []string{eligible}, nil, true, - sweepBounds{most: 10, budget: 5 * time.Second, platforms: true}) - if len(a.LetGo) != 0 || len(reg.deleted) != 0 || !strings.Contains(a.Stopped, "nothing was let go") { - t.Fatalf("let go %v, deleted %v, stopped %q", a.LetGo, reg.deleted, a.Stopped) - } -} diff --git a/cmd/mesh-controller/module_health_test.go b/cmd/mesh-controller/module_health_test.go index dba65381..f3706247 100644 --- a/cmd/mesh-controller/module_health_test.go +++ b/cmd/mesh-controller/module_health_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/moved_bus_test.go b/cmd/mesh-controller/moved_bus_test.go index c793ba96..d1fad9de 100644 --- a/cmd/mesh-controller/moved_bus_test.go +++ b/cmd/mesh-controller/moved_bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/network_test.go b/cmd/mesh-controller/network_test.go index 2c773cd8..de917559 100644 --- a/cmd/mesh-controller/network_test.go +++ b/cmd/mesh-controller/network_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/new_module_on_merge_test.go b/cmd/mesh-controller/new_module_on_merge_test.go index f5080df1..9bd9623d 100644 --- a/cmd/mesh-controller/new_module_on_merge_test.go +++ b/cmd/mesh-controller/new_module_on_merge_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/nodes_test.go b/cmd/mesh-controller/nodes_test.go index 86b95588..89ad569b 100644 --- a/cmd/mesh-controller/nodes_test.go +++ b/cmd/mesh-controller/nodes_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/non_trunk_merge_test.go b/cmd/mesh-controller/non_trunk_merge_test.go new file mode 100644 index 00000000..be0ace88 --- /dev/null +++ b/cmd/mesh-controller/non_trunk_merge_test.go @@ -0,0 +1,55 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// A merge into a branch other than the trunk starts no delivery (novox/hq ADR 0401 clause 3, issue 889): a +// module recorded with no ref follows the trunk, main, and no other branch. +func TestAMergeIntoANonTrunkBranchStartsNoDelivery(t *testing.T) { + for ref, name := range map[string]string{"": "noref", "main": "mainref"} { + t.Run(name, func(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "free", Version: "1"}, + inventory.Source{Repository: "novox/free", Seat: "git", Ref: ref, BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + feature := lone(repoMerge("free", "f1", t0)) + feature.Base = "feat/57-config-and-provisioning" + said := captured(t, func() error { return (following{open: open}).hearMerge(t.Context(), feature, t0) }) + if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 0 { + t.Fatalf("a merge into a feature branch: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + if !strings.Contains(said, feature.Base) || !strings.Contains(said, "starts no delivery") { + t.Fatalf("the merge was not said to start no delivery: %q", said) + } + hear(t, open, lone(repoMerge("free", "m1", t0)), t0) + if ws, bs := walks(t, open); len(ws)+len(bs) != 1 { + t.Fatalf("a merge into main: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + }) + } +} + +// A module recorded with an explicit branch keeps following that branch (the lead's settlement of issue 977 Q5). +func TestAModuleRecordedWithABranchStillFollowsIt(t *testing.T) { + open := windowed(t) + asksWithPaths(t) + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "pinned", Version: "1"}, + inventory.Source{Repository: "novox/pinned", Seat: "git", Ref: "feat/x", BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := lone(repoMerge("pinned", "p1", t0)) + m.Base = "feat/x" + hear(t, open, m, t0) + if ws, bs := walks(t, open); len(ws)+len(bs) != 1 { + t.Fatalf("a merge into the branch a module follows: %d walk(s), %d batch(es)", len(ws), len(bs)) + } +} diff --git a/cmd/mesh-controller/one_group_one_plan_test.go b/cmd/mesh-controller/one_group_one_plan_test.go new file mode 100644 index 00000000..84c7af6c --- /dev/null +++ b/cmd/mesh-controller/one_group_one_plan_test.go @@ -0,0 +1,177 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +// Acceptance 1 of design 47, the controller's part: a feature across the controller and the catalogue with a new seat +// verb is delivered as one group, with no `after:` line — its merges are one batch, each module is built once and sent +// once, its commits are carried by that one plan, and the plan names the edge that placed each tier. The Approve and +// the one-pass merge are mesh-delivery's. + +// manifestsHeld stubs the forge for a test: the manifest each (repository, path, commit) holds, and none anywhere else. +func manifestsHeld(t *testing.T, held map[[3]string]catalogue.Manifest) { + t.Helper() + was, wasParent := manifestAt, parentOf + t.Cleanup(func() { manifestAt, parentOf = was, wasParent }) + manifestsReadMu.Lock() + clear(manifestsRead) + manifestsReadMu.Unlock() + parentOf = func(context.Context, string, string, string) (string, error) { return "", nil } + manifestAt = func(_ context.Context, _, repo, file, ref string) (catalogue.Manifest, bool, error) { + m, ok := held[[3]string{repo, file, ref}] + return m, ok, nil + } +} + +func TestAFeatureWithANewSeatVerbIsOneBatchAndOnePlanInOrderWithNoAfterLine(t *testing.T) { + for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { + t.Run(name, func(t *testing.T) { + open := windowed(t) + asked := asksWithPaths(t) + ctx := t.Context() + const head = "feat/one-group-one-plan" + // mesh-delivery holds its seat, so a catalogue plan would wait for its word; this plan carries the controller. + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "mesh-delivery", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.DeliverySeat, Scope: catalogue.ScopeMesh}}}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/mesh-delivery", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "anchor", "mesh-delivery"); err != nil { + t.Fatal(err) + } + controller := fromHead(repoMerge("mesh-controller", "k1", t0), head) + controller.Paths = []string{"module.json", "cmd/mesh-controller/main.go"} + catalog := fromHead(catalogueMerge("cat1aaaaaaaa", "app", t0), head) + manifestsHeld(t, map[[3]string]catalogue.Manifest{ + {"mesh-controller", "module.json", "k1"}: {Module: "mesh-controller", + DefinesSeats: []catalogue.SeatDeclaration{{Name: "seat", Serves: []catalogue.Verb{{Name: "verb"}}}}}, + {"mesh-catalog", "modules/app/module.json", "cat1aaaaaaaa"}: {Module: "app", Invokes: []string{"seat:seat.verb"}}, + }) + states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} + groupStates(t, head, states) + first, second := controller, catalog + if !controllerFirst { + first, second = catalog, controller + } + // The first member merges; the group waits for the second, however long it takes. + hear(t, open, first, t0) + cutAt(t, open, t0.Add(30*time.Minute)) + if ws, bs := walks(t, open); len(ws) != 0 || len(bs) != 1 { + t.Fatalf("a member of a group alone: %d walk(s), %d batch(es)", len(ws), len(bs)) + } + states["novox/"+second.Repo] = "merged" + second.MergedAt = t0.Add(30 * time.Minute).Format(time.RFC3339Nano) + hear(t, open, second, t0.Add(30*time.Minute)) + cutAt(t, open, t0.Add(30*time.Minute+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 { + t.Fatalf("after the window: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) + } + plan := ws[0] + if plan.CommitOf("novox/mesh-controller") != "k1" || plan.CommitOf("novox/mesh-catalog") != "cat1aaaaaaaa" { + t.Fatalf("the plan carries %+v", plan.Commits) + } + if len(plan.Tiers) != 2 || strings.Join(plan.Tiers[0], ",") != "mesh-controller" || strings.Join(plan.Tiers[1], ",") != "app" { + t.Fatalf("the plan's tiers are %v; want the controller, then the caller of its verb", plan.Tiers) + } + if placed := plan.Modules["app"].Placed; !strings.Contains(placed, "verb before caller: seat.verb") || + !strings.Contains(placed, "after mesh-controller") { + t.Fatalf("the plan does not name the edge that placed the caller: %q", placed) + } + if plan.Modules["mesh-controller"].Placed != "" { + t.Fatalf("the first tier is placed by %q", plan.Modules["mesh-controller"].Placed) + } + if plan.Delivery != nil && (plan.Delivery.Awaits != "" || plan.Waiting()) { + t.Fatalf("a plan carrying the controller waits for %q's word", plan.Delivery.Awaits) + } + if plan.Modules["mesh-controller"].Build == "" || len(*asked) != 1 { + t.Fatalf("the first tier is asked once: build %q, %d ask(s) %v", plan.Modules["mesh-controller"].Build, len(*asked), *asked) + } + // Every merge is answered by that one plan. + for _, repository := range []string{"novox/mesh-controller", "novox/mesh-catalog"} { + merges, err := open.inventory.MergesOf(ctx, plan.ID) + if err != nil || len(merges) != 2 { + t.Fatalf("the plan answers %d merge(s) of %s: %v", len(merges), repository, err) + } + } + }) + } +} + +// Issue 1049: a move delivered half before its second merge. The controller adds a module the catalogue removes, in one +// group: the merges are one batch and one plan in either order, the module is built once and from the controller's +// repository, and the catalogue's removal does not forget it. +func TestAMoveOfAModuleAcrossTwoRepositoriesIsOnePlanAndBuildsItOnceFromTheNewOne(t *testing.T) { + for name, controllerFirst := range map[string]bool{"controller_first": true, "catalogue_first": false} { + t.Run(name, func(t *testing.T) { + open := windowed(t) + asked := asksWithPaths(t) + ctx := t.Context() + const head = "feat/whole-modules" + // The module runs on a machine from the catalogue's repository, so its removal there cannot forget it. + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "route-proxy", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/route-proxy", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "anchor", "route-proxy"); err != nil { + t.Fatal(err) + } + manifestsHeld(t, nil) + controller := fromHead(repoMerge("mesh-controller", "k1", t0), head) + controller.Paths = []string{"modules/route-proxy/module.json", "modules/route-proxy/main.go"} + controller.ModuleDirs, controller.ModuleDirsSaid = []string{"modules/route-proxy"}, true + catalog := fromHead(catalogueMerge("cat1aaaaaaaa", "route-proxy", t0), head) + catalog.Removed = []string{"modules/route-proxy/module.json"} + catalog.ModuleDirs, catalog.ModuleDirsSaid = nil, true + states := map[string]string{"novox/mesh-controller": "ready", "novox/mesh-catalog": "ready"} + groupStates(t, head, states) + first, second := controller, catalog + if !controllerFirst { + first, second = catalog, controller + } + hear(t, open, first, t0) + states["novox/"+second.Repo] = "merged" + second.MergedAt = t0.Add(20 * time.Second).Format(time.RFC3339Nano) + hear(t, open, second, t0.Add(20*time.Second)) + cutAt(t, open, t0.Add(20*time.Second+90*time.Second)) + ws, bs := walks(t, open) + if len(ws) != 1 || len(bs) != 0 { + t.Fatalf("a move: %d walk(s), %d batch(es); want exactly one plan", len(ws), len(bs)) + } + var fromController int + for _, a := range *asked { + if strings.Contains(a[0], "mesh-controller") && a[1] == "modules/route-proxy" { + fromController++ + } + if strings.Contains(a[0], "mesh-catalog") && a[1] == "modules/route-proxy" { + t.Errorf("route-proxy was built from the repository it left: %v", a) + } + } + if fromController != 1 { + t.Fatalf("route-proxy was built %d time(s) from the controller's repository: %v", fromController, *asked) + } + entries, err := open.inventory.Catalogued(ctx) + if err != nil { + t.Fatal(err) + } + kept := false + for _, e := range entries { + kept = kept || e.Manifest.Module == "route-proxy" + } + if !kept { + t.Fatal("the catalogue's removal forgot route-proxy, which a machine runs") + } + }) + } +} diff --git a/cmd/mesh-controller/one_resolution_integration_test.go b/cmd/mesh-controller/one_resolution_integration_test.go new file mode 100644 index 00000000..989f3ca2 --- /dev/null +++ b/cmd/mesh-controller/one_resolution_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" +) + +func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + twoRivalsAndANote(t, open) + // Straight into the store, as a node that came to this before the act. + for _, m := range []string{"rival-one", "rival-two", "notes"} { + if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil { + t.Fatal(err) + } + } + + said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) }) + + if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) { + t.Fatalf("laptop resolves with two claimants of one seat: %v", err) + } + if n := strings.Count(said, "laptop does not resolve now"); n != 1 { + t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said) + } + if strings.Contains(said, "is left out of the rest of the mesh") { + t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said) + } +} + +func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + aProviderAndItsConsumer(t, open) + if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil { + t.Fatal(err) + } + said, err := unassign(ctx, open, "laptop", "app") + if err != nil { + t.Fatal(err) + } + if _, _, err := planFor(ctx, open, "laptop"); err != nil { + t.Fatal(err) + } + if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") { + t.Errorf("a node that resolves is said not to:\n%s", said) + } +} + +func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + twoRivalsAndANote(t, open) + if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil { + t.Fatal(err) + } + // Kept and not resolvable: a second claimant of the seat, which assignment keeps. + said, err := assign(ctx, open, "laptop", "rival-two") + if err == nil { + t.Fatalf("two claimants of one node seat resolved:\n%s", said) + } + _, _, planErr := planFor(ctx, open, "laptop") + if planErr == nil || planErr.Error() != err.Error() { + t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr) + } + if strings.Contains(said, "now has") { + t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said) + } +} + +// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq +// issue 188); an act sends it elsewhere. +func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) { + var buf bytes.Buffer + sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why")) + if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" { + t.Errorf("said %q", buf.String()) + } + got := answered(t, func() error { + sayLeftOut(context.Background(), "laptop", errors.New("why")) + return nil + }) + if !strings.Contains(got, "laptop is left out") { + t.Errorf("without an act's writer the line did not reach standard error: %q", got) + } +} diff --git a/cmd/mesh-controller/one_resolution_test.go b/cmd/mesh-controller/one_resolution_test.go index c49b91c6..82a873a9 100644 --- a/cmd/mesh-controller/one_resolution_test.go +++ b/cmd/mesh-controller/one_resolution_test.go @@ -1,10 +1,6 @@ package main import ( - "bytes" - "context" - "errors" - "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" @@ -29,84 +25,3 @@ func twoRivalsAndANote(t *testing.T, open *stores) { register(t, open, two) register(t, open, catalogue.Manifest{Module: "notes", Version: "1"}) } - -func TestAnUnassignThatLeavesItsNodeUnresolvedSaysSoOnceInItsAnswer(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - twoRivalsAndANote(t, open) - // Straight into the store, as a node that came to this before the act. - for _, m := range []string{"rival-one", "rival-two", "notes"} { - if _, err := open.inventory.Assign(ctx, "laptop", m); err != nil { - t.Fatal(err) - } - } - - said := answered(t, func() error { return assignCommand(ctx, "unassign", []string{"laptop", "notes"}) }) - - if _, _, err := planFor(ctx, open, "laptop"); err == nil || !unresolvable(err) { - t.Fatalf("laptop resolves with two claimants of one seat: %v", err) - } - if n := strings.Count(said, "laptop does not resolve now"); n != 1 { - t.Errorf("the answer says laptop does not resolve %d times, want once:\n%s", n, said) - } - if strings.Contains(said, "is left out of the rest of the mesh") { - t.Errorf("the answer carries a line from a read that is not its resolution:\n%s", said) - } -} - -func TestAnUnassignThatLeavesItsNodeResolvedSaysNothingOfIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - aProviderAndItsConsumer(t, open) - if _, err := assign(ctx, open, "laptop", "db", "app"); err != nil { - t.Fatal(err) - } - said, err := unassign(ctx, open, "laptop", "app") - if err != nil { - t.Fatal(err) - } - if _, _, err := planFor(ctx, open, "laptop"); err != nil { - t.Fatal(err) - } - if strings.Contains(said, "does not resolve") || strings.Contains(said, "could not be read") { - t.Errorf("a node that resolves is said not to:\n%s", said) - } -} - -func TestAnAssignThatLeavesItsNodeUnresolvedSaysOnlyThePlansRefusal(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - twoRivalsAndANote(t, open) - if _, err := assign(ctx, open, "laptop", "rival-one"); err != nil { - t.Fatal(err) - } - // Kept and not resolvable: a second claimant of the seat, which assignment keeps. - said, err := assign(ctx, open, "laptop", "rival-two") - if err == nil { - t.Fatalf("two claimants of one node seat resolved:\n%s", said) - } - _, _, planErr := planFor(ctx, open, "laptop") - if planErr == nil || planErr.Error() != err.Error() { - t.Errorf("the act's refusal is not the plan's right after:\nact: %v\nplan: %v", err, planErr) - } - if strings.Contains(said, "now has") { - t.Errorf("the answer says a seat is now held on a node that does not resolve:\n%s", said) - } -} - -// The other readers of the whole mesh still say a machine they leave out, on standard error (novox/hq -// issue 188); an act sends it elsewhere. -func TestAMachineLeftOutIsSaidWhereTheContextSays(t *testing.T) { - var buf bytes.Buffer - sayLeftOut(leftOutSaidBy(context.Background(), &buf), "laptop", errors.New("why")) - if buf.String() != "laptop is left out of the rest of the mesh: it does not resolve: why\n" { - t.Errorf("said %q", buf.String()) - } - got := answered(t, func() error { - sayLeftOut(context.Background(), "laptop", errors.New("why")) - return nil - }) - if !strings.Contains(got, "laptop is left out") { - t.Errorf("without an act's writer the line did not reach standard error: %q", got) - } -} diff --git a/cmd/mesh-controller/operator_wait_integration_test.go b/cmd/mesh-controller/operator_wait_integration_test.go new file mode 100644 index 00000000..98a468ef --- /dev/null +++ b/cmd/mesh-controller/operator_wait_integration_test.go @@ -0,0 +1,133 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil { + t.Fatal(err) + } + if got, _ := needsOperatorOpen(t, k); got != nil { + t.Fatal("raised on one statement") + } + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil { + t.Fatal(err) + } + got, open := needsOperatorOpen(t, k) + if got == nil { + t.Fatalf("not raised on two statements: %+v", open) + } + for _, c := range open { + if c.Kind == kindModuleUnhealthy { + t.Fatalf("raised as a fault too: %+v", c) + } + } + if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning { + t.Fatalf("the condition: %+v", got) + } + if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") || + !strings.Contains(got.Explanation, "hidden prompt opens at the desk") { + t.Fatalf("its needs do not name the act: %q", got.Needs) + } + if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") { + t.Fatalf("its explanation: %q", got.Explanation) + } + if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") { + t.Fatalf("its summary does not name the secret and the line: %q", got.Summary) + } + // Long open is still a warning: only the operator can end it. + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil { + t.Fatal(err) + } + if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent { + t.Fatalf("after two days: %+v", got) + } + // Given: the next statement does not say it, and it clears. + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { + t.Fatal(err) + } + if got, _ := needsOperatorOpen(t, k); got != nil { + t.Fatal("not cleared once given") + } +} + +func TestASettingsWaitAsksForTheApproval(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}} + for i := 1; i <= 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, _ := needsOperatorOpen(t, k) + if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") { + t.Fatalf("the condition: %+v", got) + } +} + +func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, + factsGiven(map[string]time.Time{"smb-password-games": time.Now()})) + rs := map[string][]inventory.ResourceHealth{"mounts": checked} + for i := 1; i <= 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got != nil { + t.Fatalf("a wait for a secret given raised needs-operator: %+v", got) + } + unhealthy := false + for _, c := range open { + unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation") + } + if !unhealthy { + t.Fatalf("not raised as unhealthy: %+v", open) + } +} + +func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) { + k, _ := withConditionsInMemory(t) + ctx := t.Context() + waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} + for i := 1; i <= 2; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + broken := waitingResource() + broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil + for i := 3; i <= 4; i++ { + if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}}, + map[string]int{"mounts": i}, time.Now()); err != nil { + t.Fatal(err) + } + } + got, open := needsOperatorOpen(t, k) + if got != nil { + t.Fatal("needs-operator still open after it became a fault") + } + found := false + for _, c := range open { + found = found || c.Key == moduleUnhealthyKey("mounts", "workstation") + } + if !found { + t.Fatalf("the fault is not raised: %+v", open) + } +} diff --git a/cmd/mesh-controller/operator_wait_test.go b/cmd/mesh-controller/operator_wait_test.go index ff98a88b..1961f821 100644 --- a/cmd/mesh-controller/operator_wait_test.go +++ b/cmd/mesh-controller/operator_wait_test.go @@ -141,124 +141,7 @@ func needsOperatorOpen(t *testing.T, k *conditions.Keeper) (*conditions.Conditio // Rule 5: two statements of an excused wait raise needs-operator, the operator's, a warning however long, naming the // act; a statement without it clears it. -func TestTheNeedsOperatorConditionNamesTheAct(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 1}, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("raised on one statement") - } - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 2}, time.Now()); err != nil { - t.Fatal(err) - } - got, open := needsOperatorOpen(t, k) - if got == nil { - t.Fatalf("not raised on two statements: %+v", open) - } - for _, c := range open { - if c.Kind == kindModuleUnhealthy { - t.Fatalf("raised as a fault too: %+v", c) - } - } - if got.Kind != kindNeedsOperator || got.Resolver != conditions.ResolverOperator || got.Severity != conditions.Warning { - t.Fatalf("the condition: %+v", got) - } - if !strings.Contains(got.Needs, "type the password of the source games at workstation's desk prompt") || - !strings.Contains(got.Explanation, "hidden prompt opens at the desk") { - t.Fatalf("its needs do not name the act: %q", got.Needs) - } - if strings.Contains(strings.ToLower(got.Explanation), "nothing for you") || !strings.Contains(got.Explanation, "nothing was undone") { - t.Fatalf("its explanation: %q", got.Explanation) - } - if !strings.Contains(got.Summary, "smb-password-games") || !strings.Contains(got.Summary, "nox secret ask workstation mounts smb-password-games") { - t.Fatalf("its summary does not name the secret and the line: %q", got.Summary) - } - // Long open is still a warning: only the operator can end it. - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": 3}, time.Now().Add(48*time.Hour)); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got == nil || got.Severity == conditions.Urgent { - t.Fatalf("after two days: %+v", got) - } - // Given: the next statement does not say it, and it clears. - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{}, nil, time.Now()); err != nil { - t.Fatal(err) - } - if got, _ := needsOperatorOpen(t, k); got != nil { - t.Fatal("not cleared once given") - } -} - -func TestASettingsWaitAsksForTheApproval(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - rs := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(usernameWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, _ := needsOperatorOpen(t, k) - if got == nil || !strings.Contains(got.Needs, "approve smb-users of mounts on workstation when it is proposed to you") { - t.Fatalf("the condition: %+v", got) - } -} // A wait that fails its check is the module's own fault: unhealthy, with why, and no needs-operator. -func TestAWaitThatFailsItsCheckRaisesUnhealthy(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - checked := checkWaiting("workstation", []inventory.ResourceHealth{waitingResource(passwordWait)}, - factsGiven(map[string]time.Time{"smb-password-games": time.Now()})) - rs := map[string][]inventory.ResourceHealth{"mounts": checked} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", rs, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatalf("a wait for a secret given raised needs-operator: %+v", got) - } - unhealthy := false - for _, c := range open { - unhealthy = unhealthy || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !unhealthy { - t.Fatalf("not raised as unhealthy: %+v", open) - } -} // A module that waited and is then broken says so when the wait clears, and the other way round. -func TestANeedsOperatorThatBecameUnhealthySaysSo(t *testing.T) { - k, _ := withConditionsInMemory(t) - ctx := t.Context() - waiting := map[string][]inventory.ResourceHealth{"mounts": {waitingResource(passwordWait)}} - for i := 1; i <= 2; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", waiting, map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - broken := waitingResource() - broken.State, broken.Reason, broken.Waits = link.StateUnhealthy, "the source games refused its login", nil - for i := 3; i <= 4; i++ { - if err := judgeModuleHealth(ctx, nil, k, "workstation", map[string][]inventory.ResourceHealth{"mounts": {broken}}, - map[string]int{"mounts": i}, time.Now()); err != nil { - t.Fatal(err) - } - } - got, open := needsOperatorOpen(t, k) - if got != nil { - t.Fatal("needs-operator still open after it became a fault") - } - found := false - for _, c := range open { - found = found || c.Key == moduleUnhealthyKey("mounts", "workstation") - } - if !found { - t.Fatalf("the fault is not raised: %+v", open) - } -} diff --git a/cmd/mesh-controller/outside_accept_test.go b/cmd/mesh-controller/outside_accept_test.go index b3baff33..bf314b1c 100644 --- a/cmd/mesh-controller/outside_accept_test.go +++ b/cmd/mesh-controller/outside_accept_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/outside_not_given_test.go b/cmd/mesh-controller/outside_not_given_test.go index 11a096c0..74421aa9 100644 --- a/cmd/mesh-controller/outside_not_given_test.go +++ b/cmd/mesh-controller/outside_not_given_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/over_budget_integration_test.go b/cmd/mesh-controller/over_budget_integration_test.go new file mode 100644 index 00000000..38fc57ab --- /dev/null +++ b/cmd/mesh-controller/over_budget_integration_test.go @@ -0,0 +1,50 @@ +//go:build integration + +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) { + entries := []inventory.Entry{ + {Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}}, + {Manifest: catalogue.Manifest{Module: "gitea"}}, + } + walk := func(modules ...string) inventory.Plan { + p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}} + for _, m := range modules { + p.Modules[m] = &inventory.PlanModule{} + } + return p + } + for want, w := range map[string]inventory.Plan{ + inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"), + } { + if got := classOf(w, entries); got != want { + t.Errorf("%v: %s, want %s", w.Modules, got, want) + } + } + if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore { + t.Errorf("the resolver's holder is core: %s", got) + } + now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) + batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ + ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}} + times := walkTimesAtCut(batch, walk("gitea"), entries, now) + if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) || + times.Class != inventory.ClassLeaf { + t.Fatalf("times at the cut: %+v", times) + } + batch.Delivery.Batch.AtMost = now.Add(-time.Minute) + if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) { + t.Fatalf("a window closed at its maximum: %v", times.WindowClosed) + } + if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil { + t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed) + } +} diff --git a/cmd/mesh-controller/over_budget_test.go b/cmd/mesh-controller/over_budget_test.go index 3131e9b8..52773bbc 100644 --- a/cmd/mesh-controller/over_budget_test.go +++ b/cmd/mesh-controller/over_budget_test.go @@ -61,44 +61,6 @@ func TestTheControllerMayAskForTheDeliveryTimes(t *testing.T) { // A walk's class is core when it walks a module of the controller's own path or one holding the mesh's resolver, // leaf otherwise; its window closed at its batch's window, or its maximum, never after its cut. -func TestAWalksClassAndWindowAreReadAtItsCut(t *testing.T) { - entries := []inventory.Entry{ - {Manifest: catalogue.Manifest{Module: "dnsmasq", Claims: []catalogue.Claim{{Name: resolverSeat}}}}, - {Manifest: catalogue.Manifest{Module: "gitea"}}, - } - walk := func(modules ...string) inventory.Plan { - p := inventory.Plan{Modules: map[string]*inventory.PlanModule{}} - for _, m := range modules { - p.Modules[m] = &inventory.PlanModule{} - } - return p - } - for want, w := range map[string]inventory.Plan{ - inventory.ClassLeaf: walk("gitea"), inventory.ClassCore: walk("gitea", "mesh-controller"), - } { - if got := classOf(w, entries); got != want { - t.Errorf("%v: %s, want %s", w.Modules, got, want) - } - } - if got := classOf(walk("dnsmasq"), entries); got != inventory.ClassCore { - t.Errorf("the resolver's holder is core: %s", got) - } - now := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - batch := inventory.Plan{Delivery: &inventory.PlanDelivery{Batch: &inventory.PlanBatch{ - ClosesAt: now.Add(-20 * time.Second), AtMost: now.Add(5 * time.Minute)}}} - times := walkTimesAtCut(batch, walk("gitea"), entries, now) - if times.Cut == nil || !times.Cut.Equal(now) || times.WindowClosed == nil || !times.WindowClosed.Equal(now.Add(-20*time.Second)) || - times.Class != inventory.ClassLeaf { - t.Fatalf("times at the cut: %+v", times) - } - batch.Delivery.Batch.AtMost = now.Add(-time.Minute) - if times := walkTimesAtCut(batch, walk("gitea"), entries, now); !times.WindowClosed.Equal(now.Add(-time.Minute)) { - t.Fatalf("a window closed at its maximum: %v", times.WindowClosed) - } - if times := walkTimesAtCut(inventory.Plan{Delivery: &inventory.PlanDelivery{Alone: true}}, walk("gitea"), entries, now); times.WindowClosed != nil { - t.Fatalf("a merge walked alone had no window: %v", times.WindowClosed) - } -} // What each machine of the rest was sent is kept only for the machines the send reached. func TestTheRestIsKeptForTheMachinesTheSendReached(t *testing.T) { diff --git a/cmd/mesh-controller/pending_integration_test.go b/cmd/mesh-controller/pending_integration_test.go new file mode 100644 index 00000000..bbb10749 --- /dev/null +++ b/cmd/mesh-controller/pending_integration_test.go @@ -0,0 +1,733 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func failedBuild(t *testing.T, open *stores, id string) { + t.Helper() + if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", + "", "boom")); err != nil { + t.Fatal(err) + } +} + +func aCatalogueMesh(t *testing.T) *stores { + t.Helper() + open := aMesh(t) + if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + return open +} + +func mergeAdding(t *testing.T, open *stores, dir, commit string) { + t.Helper() + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, + Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { + t.Fatal(err) + } +} + +func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + + requests, err := open.inventory.RequestsNamed(ctx, "sensors") + if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { + t.Fatalf("the merge's build request: %+v %v", requests, err) + } + + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil { + t.Fatalf("an assignment while its build runs was refused: %v", err) + } + t.Logf("assign laptop sensors, while its build runs:\n%s", said) + for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", + "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a module not registered was assigned") + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { + t.Fatalf("pending: %+v", pending) + } + if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { + t.Fatalf("a second assignment: %q %v", again, err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if asked.well() { + t.Fatal("status called the mesh well while an assignment waits") + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc meshStatus + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { + t.Fatalf("status says pending %+v", doc.Pending) + } + + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("the build registered sensors and the pending assignment was not made") + } + pending = pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingApplied || + !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { + t.Fatalf("pending after the build: %+v", pending) + } +} + +func TestAStatusReadSettlesNothing(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + for range 2 { + if _, err := theThreeQuestions(ctx, open); err != nil { + t.Fatal(err) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { + t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) + } + said := settlePending(ctx, open, time.Now()) + if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { + t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) + } +} + +func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { + t.Fatal(err) + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].State != inventory.PendingExpired { + t.Fatalf("pending after a failed build: %+v", pending) + } + for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", + "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { + if !strings.Contains(pending[0].Note, want) { + t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) + } + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a failed build's module was assigned") + } + + settlePending(ctx, open, time.Now()) + key := pendingObservation(pending[0]).Key() + c, found, err := conditionsFrom.Get(ctx, key) + if err != nil || !found || c.Kind != kindPendingEnded { + t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) + } + if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { + t.Fatalf("status does not hold the open condition: %v", err) + } + + _, err = assign(ctx, open, "laptop", "sensors") + if !errors.Is(err, inventory.ErrNoSuchModule) { + t.Fatalf("a module whose build failed: %v", err) + } + for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", + "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q:\n%v", want, err) + } + } + + said, err := unassign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "taken back") { + t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) + } + settlePending(ctx, open, time.Now()) + if _, found, _ := conditionsFrom.Get(ctx, key); found { + t.Fatal("the condition stayed open after the assignment was taken back") + } + // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. + asked, err := theThreeQuestions(ctx, open) + if err != nil || pendingOpen(asked.pending) { + t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) + } + for _, c := range asked.conditions { + if c.Kind == kindPendingEnded { + t.Fatalf("status still holds the condition: %+v", c) + } + } +} + +func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "", "boom")); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + key := pendingObservation(pendingOf(t, open)[0]).Key() + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if _, found, _ := conditionsFrom.Get(ctx, key); found { + t.Fatal("the condition stayed open after the module was assigned") + } +} + +func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { + t.Fatal(err) + } + asked := asksWithPaths(t) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil { + t.Fatal(err) + } + if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { + t.Fatalf("asked %v", *asked) + } + for _, want := range []string{"known and not registered", "no space left on device", + "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { + if !strings.Contains(said, want) { + t.Fatalf("the answer does not say %q:\n%s", want, said) + } + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { + t.Fatalf("pending: %+v", pending) + } + requests, _ := inv.RequestsNamed(ctx, "sensors") + if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { + t.Fatalf("the request is not kept as assign's: %+v", requests) + } +} + +func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { + t.Fatal(err) + } + // Waiting on the failed build, its expiry not yet settled. + if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", + Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { + t.Fatal(err) + } + _, err := assign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { + t.Fatalf("known and not built with a pending assignment waiting: %v", err) + } + asked := asksWithPaths(t) + said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { + t.Fatalf("assign with build: %q %v", said, err) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + pending := pendingOf(t, open) + if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { + t.Fatalf("pending: %+v", pending) + } + // The new build in flight now: build "true" again asks nothing. + if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || + !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { + t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) + } +} + +func TestAFailedAskIsNotABuildInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + return "", errors.New("cannot submit a build: nats: timeout") + } + t.Cleanup(func() { askABuild = was }) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + _, err := assign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { + t.Fatalf("a merge that could not ask: %v", err) + } + if len(pendingOf(t, open)) != 0 { + t.Fatal("a pending assignment waits on a build never asked") + } + + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { + t.Fatal(err) + } + if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { + t.Fatal(err) + } + _, err = assign(ctx, open, "laptop", "gauges") + if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { + t.Fatalf("a build not handed over: %v", err) + } + // Heard first, the outcome stands: marking it afterwards changes nothing. + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { + t.Fatal(err) + } + _, err = assign(ctx, open, "laptop", "gauges") + if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { + t.Fatalf("an outcome heard after a failed wait: %v", err) + } +} + +func TestAPlansBuildRequestIsThePlans(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + requests, err := open.inventory.RequestsNamed(ctx, "app") + if err != nil || len(requests) != 1 || requests[0].For != "plan" { + t.Fatalf("the plan's request: %+v %v", requests, err) + } +} + +func TestABuildBeingRegisteredIsInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", + Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { + t.Fatal(err) + } + said, err := assign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { + t.Fatalf("a build being registered: %q %v", said, err) + } + settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) + p := pendingOf(t, open) + if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { + t.Fatalf("past the grace: %+v", p) + } +} + +func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) + asked := asksWithPaths(t) + _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") + if !errors.Is(err, inventory.ErrNoSuchModule) { + t.Fatalf("an unknown module: %v", err) + } + for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", + "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", + "asks for nothing here"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("the refusal does not say %q:\n%v", want, err) + } + } + if strings.Contains(err.Error(), "merge") { + t.Fatalf("the refusal claims a merge it never looked at: %v", err) + } + if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { + t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) + } +} + +func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + _, err := assign(ctx, open, "laptop", "known", "sensors") + if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { + t.Fatalf("an act naming a module in flight: %v", err) + } + if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { + t.Fatal("an act refused whole was made in part") + } +} + +func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + said, err := unassign(ctx, open, "laptop", "sensors") + if err != nil || !strings.Contains(said, "withdrawn") { + t.Fatalf("unassign of a pending assignment: %q %v", said, err) + } + if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", + "3da80a4b00aa", "sensors", "")); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a withdrawn assignment was made") + } +} + +func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + p := pendingOf(t, open)[0] + if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { + t.Fatalf("claim: %v %v", ok, err) + } + _, err := unassign(ctx, open, "laptop", "sensors") + if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { + t.Fatalf("unassign of a claimed pending assignment: %v", err) + } + if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { + t.Fatalf("the claim was overridden: %+v", got) + } + // Pending(zero) is the waiting ones alone (review point 7). + if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { + t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) + } + + // Left by a controller that stopped: back to waiting, since sensors is not assigned. + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { + t.Fatalf("a stale claim was not released: %+v", got) + } + + // Withdrawn, then the build registers it: applyPending finds nothing to claim. + if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) + if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { + t.Fatalf("a withdrawn pending assignment was made: %s", line) + } + if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatal("a withdrawn pending assignment was assigned") + } + + // A stale claim whose module was assigned is applied. + q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { + t.Fatal("claim") + } + if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + rows, _ := inv.PendingFor(ctx, "anchor", "sensors") + if len(rows) != 1 || rows[0].State != inventory.PendingApplied { + t.Fatalf("a stale claim of an assigned module: %+v", rows) + } +} + +func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + for _, r := range []inventory.BuildRequest{ + {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", + For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, + {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", + For: "merge", At: time.Now()}, + } { + if err := inv.RecordBuildRequest(ctx, r); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"lost", "heard"} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) + + settlePending(ctx, open, time.Now().Add(buildRequestBound)) + if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { + t.Fatal("a module registered meanwhile was not assigned at the next tick") + } + byModule := map[string]inventory.PendingAssignment{} + for _, p := range pendingOf(t, open) { + byModule[p.Module] = p + } + if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { + t.Fatalf("a build that said nothing: %+v", p) + } + if p := byModule["heard"]; p.State != inventory.PendingApplied { + t.Fatalf("a module registered meanwhile: %+v", p) + } + + // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is + // still open (second review, bug B). + if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { + t.Fatal(err) + } + said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + rows, err := inv.Pending(ctx, time.Unix(0, 0)) + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if !r.Open() && (r.Raised == nil || r.Cleared != nil) { + t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) + } + } + if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { + t.Fatalf("the applied row was not deleted: %v", said) + } +} + +func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + asked := 0 + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { + asked++ + return fmt.Sprintf("b-%d", asked), nil + } + t.Cleanup(func() { askABuild = was }) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + failedBuild(t, open, "b-1") + settlePending(ctx, open, time.Now()) + first := pendingOf(t, open)[0] + if !conditionOpen(t, first) { + t.Fatal("the first assignment not made raised nothing") + } + + if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil { + t.Fatal(err) + } + failedBuild(t, open, "b-2") + settlePending(ctx, open, time.Now()) + + var second inventory.PendingAssignment + for _, p := range pendingOf(t, open) { + if p.Build == "b-2" { + second = p + } + } + if second.State != inventory.PendingExpired { + t.Fatalf("the second: %+v", second) + } + if !conditionOpen(t, second) { + t.Fatal("the second assignment's condition was cleared in the tick that raised it") + } + if !conditionOpen(t, rowOf(t, open, first.ID)) { + t.Fatal("the first was judged answered by a second that itself was not made") + } + // unassign takes both back, and the next tick clears both. + if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") || + !strings.Contains(said, "b-2") { + t.Fatalf("unassign: %q %v", said, err) + } + settlePending(ctx, open, time.Now()) + if conditionOpen(t, first) || conditionOpen(t, second) { + t.Fatal("a condition stayed open after both were taken back") + } +} + +func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + asksWithPaths(t) + mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") + if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + failedBuild(t, open, "b-modules/sensors") + settlePending(ctx, open, time.Now()) + row := pendingOf(t, open)[0] + + settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { + t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") + } + if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) + if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { + t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) + } + + // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. + if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { + t.Fatal("known and not built was not refused") + } + q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + q.Node = "anchor" + if !conditionOpen(t, q) { + t.Fatal("not raised") + } + if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { + t.Fatal(err) + } + settlePending(ctx, open, time.Now()) + if conditionOpen(t, q) { + t.Fatal("a removed machine's condition stayed open") + } +} + +func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + claimed := func(node string) inventory.PendingAssignment { + p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) + if err != nil { + t.Fatal(err) + } + if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { + t.Fatalf("claim: %v %v", ok, err) + } + return p + } + notAssigned, assigned := claimed("laptop"), claimed("anchor") + register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { + t.Fatal(err) + } + + settlePending(ctx, open, time.Now()) + if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { + t.Fatal("a fresh claim was settled") + } + settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) + if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { + t.Fatalf("a stale claim of an assigned module: %+v", got) + } + // Released to waiting, and in the same tick made, since sensors is registered now. + if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || + !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { + t.Fatalf("a stale claim of a module not assigned: %+v", got) + } +} + +func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) { + open := aCatalogueMesh(t) + ctx := t.Context() + inv := open.inventory + for _, id := range []string{"build-timeout", "build-nothandedover"} { + dir := "modules/" + strings.TrimPrefix(id, "build-") + if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog", + Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil { + t.Fatal(err) + } + } + markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s")) + markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver)) + + said, err := assign(ctx, open, "laptop", "timeout") + if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") { + t.Fatalf("a build no longer waited for: %q %v", said, err) + } + if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") { + t.Fatalf("a build never handed over: %v", err) + } + settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute)) + rows, _ := inv.PendingFor(ctx, "laptop", "timeout") + if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") { + t.Fatalf("past the bound: %+v", rows) + } +} diff --git a/cmd/mesh-controller/pending_test.go b/cmd/mesh-controller/pending_test.go index 3efc8053..d949c02c 100644 --- a/cmd/mesh-controller/pending_test.go +++ b/cmd/mesh-controller/pending_test.go @@ -1,12 +1,8 @@ package main import ( - "context" "encoding/json" - "errors" - "fmt" "slices" - "strings" "testing" "time" @@ -17,13 +13,6 @@ import ( ) // failedBuild settles the merge's build of modules/sensors as failed. -func failedBuild(t *testing.T, open *stores, id string) { - t.Helper() - if err := (builds{open.inventory, open}).Built(t.Context(), outcome(id, "modules/sensors", "3da80a4b00aa", - "", "boom")); err != nil { - t.Fatal(err) - } -} func conditionOpen(t *testing.T, p inventory.PendingAssignment) bool { t.Helper() @@ -54,26 +43,8 @@ func rowOf(t *testing.T, open *stores, id int64) inventory.PendingAssignment { // closest names) — and a pending assignment is made when its build registers the module, or ends with why. // aCatalogueMesh is aMesh with one module held from the catalogue, so a merge of it is acted on. -func aCatalogueMesh(t *testing.T) *stores { - t.Helper() - open := aMesh(t) - if err := open.inventory.RegisterModule(t.Context(), catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - return open -} // mergeAdding is the catalogue's merge adding one module's directory, acted on as the bus hands it over. -func mergeAdding(t *testing.T, open *stores, dir, commit string) { - t.Helper() - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: commit, - Paths: []string{dir + "/module.json"}, ModuleDirs: []string{dir}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(t.Context(), m); err != nil { - t.Fatal(err) - } -} // outcome is a build's outcome as the build seat announces it: registered as module, or failed. func outcome(id, dir, commit, module, failed string) link.BuildResult { @@ -107,554 +78,51 @@ func pendingOf(t *testing.T, open *stores) []inventory.PendingAssignment { // A build in flight: the merge asked for it, the request is kept as the merge's, the assignment is kept // pending and said so with the build, status reads it without settling anything, and the build's outcome // makes it — saying that a push sends it, and nothing more. -func TestAnAssignmentWaitsForTheBuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - - requests, err := open.inventory.RequestsNamed(ctx, "sensors") - if err != nil || len(requests) != 1 || requests[0].For != "merge" || requests[0].Commit != "3da80a4b00aa" { - t.Fatalf("the merge's build request: %+v %v", requests, err) - } - - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil { - t.Fatalf("an assignment while its build runs was refused: %v", err) - } - t.Logf("assign laptop sensors, while its build runs:\n%s", said) - for _, want := range []string{"being built from novox/mesh-catalog@3da80a4b (modules/sensors)", - "build b-modules/sensors", "kept as pending", "the controller assigns sensors to laptop", "`unassign laptop sensors`"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a module not registered was assigned") - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingWaiting || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - if again, err := assign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(again, "already waits") { - t.Fatalf("a second assignment: %q %v", again, err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if asked.well() { - t.Fatal("status called the mesh well while an assignment waits") - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc meshStatus - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Pending) != 1 || doc.Pending[0].State != "waiting" || doc.Pending[0].Module != "sensors" { - t.Fatalf("status says pending %+v", doc.Pending) - } - - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("the build registered sensors and the pending assignment was not made") - } - pending = pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingApplied || - !strings.HasSuffix(pending[0].Note, "`push laptop` sends it") { - t.Fatalf("pending after the build: %+v", pending) - } -} // **A read settles nothing** (review of #150, point 6): status — and so the board, the summary and the // probe, which compose from the same reading — leaves a pending assignment whose module is registered as it // is; the controller's tick makes it. -func TestAStatusReadSettlesNothing(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - for range 2 { - if _, err := theThreeQuestions(ctx, open); err != nil { - t.Fatal(err) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingWaiting { - t.Fatalf("a status read settled a pending assignment: %+v", pendingOf(t, open)) - } - said := settlePending(ctx, open, time.Now()) - if !slices.Contains(assignedTo(t, open, "laptop"), "sensors") || pendingOf(t, open)[0].State != inventory.PendingApplied { - t.Fatalf("the tick did not make it: %v %+v", said, pendingOf(t, open)) - } -} // The build of a pending assignment fails: it expires with the build's words and nothing is assigned; the // tick raises it as a condition once; it then reads as known and not built; and `unassign` takes it back, // after which the tick clears the condition. Status is not called well while the condition is open, and is // again once it clears — no fixed day of "not well" (review point 6). -func TestAPendingAssignmentExpiresWhenItsBuildFails(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "go build: undefined: sensorsRead")); err != nil { - t.Fatal(err) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].State != inventory.PendingExpired { - t.Fatalf("pending after a failed build: %+v", pending) - } - for _, want := range []string{"expired", "build b-modules/sensors of modules/sensors failed", - "undefined: sensorsRead", "nothing was assigned", "build \"true\""} { - if !strings.Contains(pending[0].Note, want) { - t.Fatalf("the expiry does not say %q: %s", want, pending[0].Note) - } - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a failed build's module was assigned") - } - - settlePending(ctx, open, time.Now()) - key := pendingObservation(pending[0]).Key() - c, found, err := conditionsFrom.Get(ctx, key) - if err != nil || !found || c.Kind != kindPendingEnded { - t.Fatalf("no condition %s for the assignment not made: %+v %v %v", key, c, found, err) - } - if asked, err := theThreeQuestions(ctx, open); err != nil || asked.well() || len(asked.conditions) == 0 { - t.Fatalf("status does not hold the open condition: %v", err) - } - - _, err = assign(ctx, open, "laptop", "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("a module whose build failed: %v", err) - } - for _, want := range []string{"sensors is known and not registered", "its last build, b-modules/sensors", - "failed: go build: undefined: sensorsRead", "build \"true\"", "repository novox/mesh-catalog, path modules/sensors"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "taken back") { - t.Fatalf("unassign of an expired pending assignment: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the assignment was taken back") - } - // Nothing of it keeps status from being well any more: no open pending assignment, no open condition of it. - asked, err := theThreeQuestions(ctx, open) - if err != nil || pendingOpen(asked.pending) { - t.Fatalf("status still counts the pending assignment: %+v %v", asked.pending, err) - } - for _, c := range asked.conditions { - if c.Kind == kindPendingEnded { - t.Fatalf("status still holds the condition: %+v", c) - } - } -} // The condition also clears when the module is later assigned to the machine. -func TestTheConditionClearsWhenTheModuleIsAssignedLater(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "", "boom")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - key := pendingObservation(pendingOf(t, open)[0]).Key() - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if _, found, _ := conditionsFrom.Get(ctx, key); found { - t.Fatal("the condition stayed open after the module was assigned") - } -} // Known and not built, asked with build: the build is asked from where the last one was, the request kept as // assign's, and the assignment kept pending on the new build. -func TestAssignWithBuildAsksForAKnownModule(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "build", At: time.Now().Add(-time.Hour)}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", On: "anchor", Failed: "no space left on device"}); err != nil { - t.Fatal(err) - } - asked := asksWithPaths(t) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil { - t.Fatal(err) - } - if len(*asked) != 1 || (*asked)[0] != [3]string{"novox/mesh-catalog", "modules/sensors", "main"} { - t.Fatalf("asked %v", *asked) - } - for _, want := range []string{"known and not registered", "no space left on device", - "build b-modules/sensors of novox/mesh-catalog@main (modules/sensors) is asked for now", "kept as pending"} { - if !strings.Contains(said, want) { - t.Fatalf("the answer does not say %q:\n%s", want, said) - } - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].Build != "b-modules/sensors" { - t.Fatalf("pending: %+v", pending) - } - requests, _ := inv.RequestsNamed(ctx, "sensors") - if len(requests) != 2 || requests[0].ID != "b-modules/sensors" || requests[0].For != "assign" { - t.Fatalf("the request is not kept as assign's: %+v", requests) - } -} // **build "true" with a pending assignment already waiting** (review point 5): the waiting one is made to // wait for the new build, and no build is asked that nothing waits on. -func TestAssignWithBuildRepointsTheWaitingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-old", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge", At: time.Now().Add(-time.Hour)}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-old", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", On: "anchor", Failed: "boom"}); err != nil { - t.Fatal(err) - } - // Waiting on the failed build, its expiry not yet settled. - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "laptop", Module: "sensors", - Build: "build-old", Repository: "novox/mesh-catalog", Path: "modules/sensors"}); err != nil { - t.Fatal(err) - } - _, err := assign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "already has a pending assignment of sensors, waiting for build build-old") { - t.Fatalf("known and not built with a pending assignment waiting: %v", err) - } - asked := asksWithPaths(t) - said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if err != nil || !strings.Contains(said, "which waited for build build-old, now waits for build b-modules/sensors") { - t.Fatalf("assign with build: %q %v", said, err) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - pending := pendingOf(t, open) - if len(pending) != 1 || pending[0].Build != "b-modules/sensors" || pending[0].State != inventory.PendingWaiting { - t.Fatalf("pending: %+v", pending) - } - // The new build in flight now: build "true" again asks nothing. - if said, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil || - !strings.Contains(said, "no second build was asked") || len(*asked) != 1 { - t.Fatalf("a second build true while the build runs: %q %v, asked %v", said, err, *asked) - } -} // **A failed ask never reads as in flight** (review point 2): a merge whose ask could not be made keeps the // request as not asked, and assign says the merge could not ask; a request whose asker could not hand it over // or stopped waiting reads the same, unless its outcome was heard. -func TestAFailedAskIsNotABuildInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { - return "", errors.New("cannot submit a build: nats: timeout") - } - t.Cleanup(func() { askABuild = was }) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "the merge that added it (3da80a4b) could not ask for its build: cannot submit") { - t.Fatalf("a merge that could not ask: %v", err) - } - if len(pendingOf(t, open)) != 0 { - t.Fatal("a pending assignment waits on a build never asked") - } - - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-waited", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/gauges", Ref: "main", For: "build"}); err != nil { - t.Fatal(err) - } - if err := inv.MarkNotAsked(ctx, "build-waited", "cannot submit a build: no responders"); err != nil { - t.Fatal(err) - } - _, err = assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "build build-waited, asked by build, was not handed over") { - t.Fatalf("a build not handed over: %v", err) - } - // Heard first, the outcome stands: marking it afterwards changes nothing. - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-waited", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/gauges", On: "anchor", Failed: "the real failure"}); err != nil { - t.Fatal(err) - } - _, err = assign(ctx, open, "laptop", "gauges") - if err == nil || !strings.Contains(err.Error(), "failed: the real failure") { - t.Fatalf("an outcome heard after a failed wait: %v", err) - } -} // A plan's tier keeps its requests as the plan's (review point 3). -func TestAPlansBuildRequestIsThePlans(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - if err := open.inventory.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - requests, err := open.inventory.RequestsNamed(ctx, "app") - if err != nil || len(requests) != 1 || requests[0].For != "plan" { - t.Fatalf("the plan's request: %+v %v", requests, err) - } -} // **A build heard as built and not registered yet is in flight** (review point 4): the outcome is recorded // before the module is registered; for that moment assign keeps the assignment pending. Past the grace it is // recorded and not registered, and the tick ends the pending assignment with that. -func TestABuildBeingRegisteredIsInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: "build-done", Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/sensors", Ref: "main", For: "merge"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-done", Repository: "novox/mesh-catalog", Ref: "main", - Path: "modules/sensors", Module: "sensors", On: "anchor"}); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "built and being registered") || !strings.Contains(said, "kept as pending") { - t.Fatalf("a build being registered: %q %v", said, err) - } - settlePending(ctx, open, time.Now().Add(registerGrace+time.Minute)) - p := pendingOf(t, open) - if len(p) != 1 || p[0].State != inventory.PendingExpired || !strings.Contains(p[0].Note, "recorded and not registered") { - t.Fatalf("past the grace: %+v", p) - } -} // Unknown: refused as no module of that name, claiming only what was looked at, with the closest names; build // asks for nothing. -func TestAnUnknownModuleIsRefusedWithTheClosestNames(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "sensord", Version: "1"}) - asked := asksWithPaths(t) - _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors") - if !errors.Is(err, inventory.ErrNoSuchModule) { - t.Fatalf("an unknown module: %v", err) - } - for _, want := range []string{"no module of that name: sensors", "the catalogue holds no module of that name", - "no build request the controller kept (the last 30 days)", "the closest names it holds: sensord", - "asks for nothing here"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("the refusal does not say %q:\n%v", want, err) - } - } - if strings.Contains(err.Error(), "merge") { - t.Fatalf("the refusal claims a merge it never looked at: %v", err) - } - if len(*asked) != 0 || len(pendingOf(t, open)) != 0 { - t.Fatalf("an unknown module asked %v, pending %+v", *asked, pendingOf(t, open)) - } -} // Several modules in one act, one of them not registered: nothing is assigned and nothing kept pending, and // each missing one is said. -func TestAnActWithAModuleNotRegisteredIsRefusedWhole(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - register(t, open, catalogue.Manifest{Module: "known", Version: "1"}) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - _, err := assign(ctx, open, "laptop", "known", "sensors") - if err == nil || !strings.Contains(err.Error(), "nothing was assigned") || !strings.Contains(err.Error(), "being built") { - t.Fatalf("an act naming a module in flight: %v", err) - } - if slices.Contains(assignedTo(t, open, "laptop"), "known") || len(pendingOf(t, open)) != 0 { - t.Fatal("an act refused whole was made in part") - } -} // unassign withdraws a waiting pending assignment; the build goes on and registers the module assigned // nowhere: a withdrawn row is never claimed. -func TestUnassignWithdrawsAPendingAssignment(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - said, err := unassign(ctx, open, "laptop", "sensors") - if err != nil || !strings.Contains(said, "withdrawn") { - t.Fatalf("unassign of a pending assignment: %q %v", said, err) - } - if err := (builds{open.inventory, open}).Built(ctx, outcome("b-modules/sensors", "modules/sensors", - "3da80a4b00aa", "sensors", "")); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn assignment was made") - } -} // **A claim is never overridden** (review point 1): a pending assignment being made is claimed // (waiting → applying); an unassign meanwhile is refused and says so, and leaves the claim; making a row // already withdrawn makes nothing. A claim left by a controller that stopped is settled by the tick from what // the mesh holds: back to waiting when the module is not assigned, applied when it is. -func TestAWithdrawalNeverOverridesAClaim(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - p := pendingOf(t, open)[0] - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - _, err := unassign(ctx, open, "laptop", "sensors") - if err == nil || !strings.Contains(err.Error(), "is being assigned sensors now") { - t.Fatalf("unassign of a claimed pending assignment: %v", err) - } - if got := pendingOf(t, open)[0]; got.State != inventory.PendingApplying { - t.Fatalf("the claim was overridden: %+v", got) - } - // Pending(zero) is the waiting ones alone (review point 7). - if waiting, err := inv.Pending(ctx, time.Time{}); err != nil || len(waiting) != 0 { - t.Fatalf("Pending(zero) read a claimed row: %+v %v", waiting, err) - } - - // Left by a controller that stopped: back to waiting, since sensors is not assigned. - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := pendingOf(t, open)[0]; got.State != inventory.PendingWaiting { - t.Fatalf("a stale claim was not released: %+v", got) - } - - // Withdrawn, then the build registers it: applyPending finds nothing to claim. - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - register(t, open, catalogue.Manifest{Module: "sensors", Version: "1"}) - if line := applyPending(ctx, open, p, "b-modules/sensors"); line != "" { - t.Fatalf("a withdrawn pending assignment was made: %s", line) - } - if slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatal("a withdrawn pending assignment was assigned") - } - - // A stale claim whose module was assigned is applied. - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, _ := inv.ClaimPending(ctx, q.ID); !ok { - t.Fatal("claim") - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - rows, _ := inv.PendingFor(ctx, "anchor", "sensors") - if len(rows) != 1 || rows[0].State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", rows) - } -} // A build that says nothing within the bound: the pending assignment expires at the next tick, saying so, // rather than waiting for ever; and a module registered by a process that kept no pending assignment is // assigned at the next tick. -func TestAPendingAssignmentNeverWaitsSilently(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - for _, r := range []inventory.BuildRequest{ - {ID: "build-lost", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/lost", Ref: "main", - For: "merge", At: time.Now().Add(-buildRequestBound / 2)}, - {ID: "build-heard", Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/heard", Ref: "main", - For: "merge", At: time.Now()}, - } { - if err := inv.RecordBuildRequest(ctx, r); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"lost", "heard"} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - register(t, open, catalogue.Manifest{Module: "heard", Version: "1"}) - - settlePending(ctx, open, time.Now().Add(buildRequestBound)) - if !slices.Contains(assignedTo(t, open, "laptop"), "heard") { - t.Fatal("a module registered meanwhile was not assigned at the next tick") - } - byModule := map[string]inventory.PendingAssignment{} - for _, p := range pendingOf(t, open) { - byModule[p.Module] = p - } - if p := byModule["lost"]; p.State != inventory.PendingExpired || !strings.Contains(p.Note, "no outcome of build build-lost") { - t.Fatalf("a build that said nothing: %+v", p) - } - if p := byModule["heard"]; p.State != inventory.PendingApplied { - t.Fatalf("a module registered meanwhile: %+v", p) - } - - // Ended rows are deleted after KeptFor (review point 7); open ones never, nor one whose condition is - // still open (second review, bug B). - if _, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "lost", Build: "build-lost"}); err != nil { - t.Fatal(err) - } - said := settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - rows, err := inv.Pending(ctx, time.Unix(0, 0)) - if err != nil { - t.Fatal(err) - } - for _, r := range rows { - if !r.Open() && (r.Raised == nil || r.Cleared != nil) { - t.Fatalf("an ended pending assignment with no open condition outlived %s: %+v (%v)", inventory.KeptFor, r, said) - } - } - if !strings.Contains(strings.Join(said, "\n"), "deleted 1 pending assignment(s)") { - t.Fatalf("the applied row was not deleted: %v", said) - } -} // The seat's assign passes build on; unassign takes none. func TestTheSeatsAssignPassesBuild(t *testing.T) { @@ -685,180 +153,14 @@ func TestAnAssignmentNotMadeIsSaidPlainly(t *testing.T) { // pending assignment; its build fails too before the next tick. That tick raises the second, and must not // clear it by judging the first answered by the second: each row has its own condition, and a newer row that // itself ended unmade answers nothing. -func TestASecondAssignmentNotMadeKeepsItsCondition(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - asked := 0 - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { - asked++ - return fmt.Sprintf("b-%d", asked), nil - } - t.Cleanup(func() { askABuild = was }) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-1") - settlePending(ctx, open, time.Now()) - first := pendingOf(t, open)[0] - if !conditionOpen(t, first) { - t.Fatal("the first assignment not made raised nothing") - } - - if _, err := assignWith(ctx, open, "laptop", assignOptions{Build: true}, "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-2") - settlePending(ctx, open, time.Now()) - - var second inventory.PendingAssignment - for _, p := range pendingOf(t, open) { - if p.Build == "b-2" { - second = p - } - } - if second.State != inventory.PendingExpired { - t.Fatalf("the second: %+v", second) - } - if !conditionOpen(t, second) { - t.Fatal("the second assignment's condition was cleared in the tick that raised it") - } - if !conditionOpen(t, rowOf(t, open, first.ID)) { - t.Fatal("the first was judged answered by a second that itself was not made") - } - // unassign takes both back, and the next tick clears both. - if said, err := unassign(ctx, open, "laptop", "sensors"); err != nil || !strings.Contains(said, "b-1") || - !strings.Contains(said, "b-2") { - t.Fatalf("unassign: %q %v", said, err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, first) || conditionOpen(t, second) { - t.Fatal("a condition stayed open after both were taken back") - } -} // **A raised row outlives the pruning until its condition clears** (second review, bug B); and a machine's // removal, which takes its rows with it, clears their conditions at the next tick. -func TestARaisedRowIsKeptUntilItsConditionClears(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - asksWithPaths(t) - mergeAdding(t, open, "modules/sensors", "3da80a4b00aa") - if _, err := assign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - failedBuild(t, open, "b-modules/sensors") - settlePending(ctx, open, time.Now()) - row := pendingOf(t, open)[0] - - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if rowOf(t, open, row.ID).Raised == nil || !conditionOpen(t, row) { - t.Fatal("a raised row was pruned, or its condition closed, while the condition was open") - } - if _, err := unassign(ctx, open, "laptop", "sensors"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - settlePending(ctx, open, time.Now().Add(inventory.KeptFor+time.Hour)) - if known, err := inv.PendingKnown(ctx, []int64{row.ID}); err != nil || known[row.ID] { - t.Fatalf("a cleared row outlived %s: %v", inventory.KeptFor, err) - } - - // On anchor, then anchor removed: the row goes, and its condition with it at the next tick. - if _, err := assign(ctx, open, "anchor", "sensors"); err == nil { - t.Fatal("known and not built was not refused") - } - q, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: "anchor", Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if _, err := inv.SettlePending(ctx, q.ID, inventory.PendingWaiting, inventory.PendingExpired, "boom"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - q.Node = "anchor" - if !conditionOpen(t, q) { - t.Fatal("not raised") - } - if _, err := inv.RemoveNodeForTest(ctx, "anchor"); err != nil { - t.Fatal(err) - } - settlePending(ctx, open, time.Now()) - if conditionOpen(t, q) { - t.Fatal("a removed machine's condition stayed open") - } -} // **A claim left by a controller that stopped** (second review): the tick settles it from what the mesh // holds — back to waiting when the module is not assigned there, applied when it is — and leaves a fresh // claim alone. -func TestAStaleClaimIsSettledFromWhatTheMeshHolds(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - claimed := func(node string) inventory.PendingAssignment { - p, err := inv.RecordPending(ctx, inventory.PendingAssignment{Node: node, Module: "sensors", Build: "b-x"}) - if err != nil { - t.Fatal(err) - } - if ok, err := inv.ClaimPending(ctx, p.ID); err != nil || !ok { - t.Fatalf("claim: %v %v", ok, err) - } - return p - } - notAssigned, assigned := claimed("laptop"), claimed("anchor") - register(t, open, catalogue.Manifest{Module: "unrelated", Version: "1"}) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "sensors", Version: "1"}, inventory.Source{}); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "anchor", "sensors"); err != nil { - t.Fatal(err) - } - - settlePending(ctx, open, time.Now()) - if rowOf(t, open, notAssigned.ID).State != inventory.PendingApplying || rowOf(t, open, assigned.ID).State != inventory.PendingApplying { - t.Fatal("a fresh claim was settled") - } - settlePending(ctx, open, time.Now().Add(claimStaleAfter+time.Minute)) - if got := rowOf(t, open, assigned.ID); got.State != inventory.PendingApplied { - t.Fatalf("a stale claim of an assigned module: %+v", got) - } - // Released to waiting, and in the same tick made, since sensors is registered now. - if got := rowOf(t, open, notAssigned.ID); got.State != inventory.PendingApplied || - !slices.Contains(assignedTo(t, open, "laptop"), "sensors") { - t.Fatalf("a stale claim of a module not assigned: %+v", got) - } -} // **A waited build whose asker stopped waiting is asked, outcome unknown** (second review): it may still run, // so it reads as in flight until its outcome or its bound, never as not asked; a build never handed over // is not asked. -func TestABuildNoLongerWaitedForIsStillInFlight(t *testing.T) { - open := aCatalogueMesh(t) - ctx := t.Context() - inv := open.inventory - for _, id := range []string{"build-timeout", "build-nothandedover"} { - dir := "modules/" + strings.TrimPrefix(id, "build-") - if err := inv.RecordBuildRequest(ctx, inventory.BuildRequest{ID: id, Repository: "novox/mesh-catalog", - Seat: "git", Path: dir, Ref: "main", For: "build"}); err != nil { - t.Fatal(err) - } - } - markWaitFailed(ctx, "build-timeout", errors.New("no build machine answered within 10m0s")) - markWaitFailed(ctx, "build-nothandedover", fmt.Errorf("%w: cannot submit a build: nats: timeout", link.ErrNotHandedOver)) - - said, err := assign(ctx, open, "laptop", "timeout") - if err != nil || !strings.Contains(said, "being built") || !strings.Contains(said, "kept as pending") { - t.Fatalf("a build no longer waited for: %q %v", said, err) - } - if _, err := assign(ctx, open, "laptop", "nothandedover"); err == nil || !strings.Contains(err.Error(), "was not handed over") { - t.Fatalf("a build never handed over: %v", err) - } - settlePending(ctx, open, time.Now().Add(buildRequestBound+time.Minute)) - rows, _ := inv.PendingFor(ctx, "laptop", "timeout") - if len(rows) != 1 || rows[0].State != inventory.PendingExpired || !strings.Contains(rows[0].Note, "its asker stopped waiting") { - t.Fatalf("past the bound: %+v", rows) - } -} diff --git a/cmd/mesh-controller/person_wait_integration_test.go b/cmd/mesh-controller/person_wait_integration_test.go new file mode 100644 index 00000000..72bb5dd6 --- /dev/null +++ b/cmd/mesh-controller/person_wait_integration_test.go @@ -0,0 +1,165 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + k, _ := withConditionsInMemory(t) + at := h0 + say := func(rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, + Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + asLink := func(r inventory.ResourceHealth) link.ResourceHealth { + return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, + Reason: r.Reason, Account: r.Account} + } + account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) + openNow := func() []conditions.Condition { + t.Helper() + list, err := k.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + say(account, unit) + say(account, unit) + list := openNow() + if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { + t.Fatalf("raised %+v; want lights' relogin-needed alone", list) + } + c := list[0] + if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || + !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { + t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) + } + // After the new login the account is healthy, and the unit, still failing, is the module's own fault. + account.State, account.Reason = link.StateHealthy, "" + say(account, unit) + list = openNow() + if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { + t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) + } + unit.State, unit.Reason = link.StateHealthy, "" + say(account, unit) + if list = openNow(); len(list) != 0 { + t.Fatalf("after the unit runs: %+v; want nothing open", list) + } +} + +func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"laptop": true}, nil + } + backlogFacts := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := backlogFacts(ctx, open, component) + f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), + Resources: []inventory.ResourceHealth{ + {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, + {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, + Reason: "down"}}}} + return f, err + } + gateEvery, gateBound = 0, 300*time.Millisecond + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + advancePlans(ctx, b.open) + if p := b.release(t); p.State != inventory.PlanRolling { + break + } + time.Sleep(20 * time.Millisecond) + } + p := b.release(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) + } + if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || + !strings.Contains(v.Why, "on its own") { + t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { + t.Fatal("late's build is not marked failed at its gate") + } + current, _ := inv.CurrentBuilds(ctx) + if current["app"].Commit != "c2" || current["late"].Commit != "c1" { + t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, + current["late"].Commit) + } + if !strings.Contains(p.Note, "passed on their own and kept: app") { + t.Errorf("the walk does not say what kept its pass: %s", p.Note) + } +} + +func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + store := conditions.NewInMemory() + told := &conditions.Told{} + k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) + t.Cleanup(func() { k.Close(context.Background()) }) + at := h0 + say := func(rs ...link.ResourceHealth) { + t.Helper() + at = at.Add(time.Second) + for i := range rs { + rs[i].Since = at + } + if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, + Resources: rs}, at); err != nil { + t.Fatal(err) + } + } + unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", + Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} + account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", + Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} + say(unit) + say(unit) // lights not working + say(account, unit) + say(account, unit) // now it waits for a login + var resolved []string + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { + time.Sleep(20 * time.Millisecond) + for _, e := range told.Said() { + if e.Change == conditions.ChangeCleared { + resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) + // The clearing line is held to the plain rule too (ADR 0253). + w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, + Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} + if why, ok := conditions.PlainWords(w, "laptop"); !ok { + t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) + } + } + } + } + if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { + t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) + } +} diff --git a/cmd/mesh-controller/person_wait_test.go b/cmd/mesh-controller/person_wait_test.go index d5337899..cd61536e 100644 --- a/cmd/mesh-controller/person_wait_test.go +++ b/cmd/mesh-controller/person_wait_test.go @@ -1,7 +1,6 @@ package main import ( - "context" "strings" "testing" "time" @@ -103,110 +102,9 @@ func TestAWaitForAPersonIsAPassCarriedAlong(t *testing.T) { // The module's condition says the wait in one sentence for a person, and clears on the first statement that // no longer says it — said as the module's own fault when its unit still fails after the new login. -func TestTheReloginConditionSaysTheWaitAndClearsAfterTheLogin(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - k, _ := withConditionsInMemory(t) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - asLink := func(r inventory.ResourceHealth) link.ResourceHealth { - return link.ResourceHealth{Module: r.Module, Resource: r.Resource, Kind: r.Kind, Target: r.Target, State: r.State, - Reason: r.Reason, Account: r.Account} - } - account, unit := asLink(relogin("lights", "operator")), asLink(userUnit("lights", "operator")) - openNow := func() []conditions.Condition { - t.Helper() - list, err := k.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - say(account, unit) - say(account, unit) - list := openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.relogin-needed" { - t.Fatalf("raised %+v; want lights' relogin-needed alone", list) - } - c := list[0] - if c.Severity != conditions.Warning || c.Resolver != conditions.ResolverOperator || - !strings.HasPrefix(c.Summary, "relogin needed on laptop:") { - t.Fatalf("the condition is %s %s %q", c.Severity, c.Resolver, c.Summary) - } - // After the new login the account is healthy, and the unit, still failing, is the module's own fault. - account.State, account.Reason = link.StateHealthy, "" - say(account, unit) - list = openNow() - if len(list) != 1 || list[0].Key != "module.lights.laptop.unhealthy" { - t.Fatalf("after the login: %+v; want lights' own unhealthy condition alone", list) - } - unit.State, unit.Reason = link.StateHealthy, "" - say(account, unit) - if list = openNow(); len(list) != 0 { - t.Fatalf("after the unit runs: %+v; want nothing open", list) - } -} // A module that is healthy on its own keeps its pass when another module of the same send fails at the // bound: it is neither put back nor left without a verdict for every other walk to wait on. -func TestAModuleHealthyOnItsOwnKeepsItsPassWhenItsSendFails(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"laptop": true}, nil - } - backlogFacts := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := backlogFacts(ctx, open, component) - f.health = map[string]inventory.NodeHealth{"laptop": {Node: "laptop", HeardAt: time.Now(), - Resources: []inventory.ResourceHealth{ - {Module: "app", Resource: "app.web", Kind: "container", Target: "app", State: link.StateHealthy}, - {Module: "late", Resource: "late.web", Kind: "container", Target: "late", State: link.StateUnhealthy, - Reason: "down"}}}} - return f, err - } - gateEvery, gateBound = 0, 300*time.Millisecond - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - advancePlans(ctx, b.open) - if p := b.release(t); p.State != inventory.PlanRolling { - break - } - time.Sleep(20 * time.Millisecond) - } - p := b.release(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the walk is %s: %s; want it failed on late", p.State, p.Note) - } - if v, found, err := inv.GateOf(ctx, "build-app-c2"); err != nil || !found || v.Verdict != inventory.GatePassed || - !strings.Contains(v.Why, "on its own") { - t.Fatalf("app's verdict: %+v (found %v, %v); want its own pass kept", v, found, err) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); !failed { - t.Fatal("late's build is not marked failed at its gate") - } - current, _ := inv.CurrentBuilds(ctx) - if current["app"].Commit != "c2" || current["late"].Commit != "c1" { - t.Fatalf("registered: app %s, late %s; want app kept at c2 and late put back to c1", current["app"].Commit, - current["late"].Commit) - } - if !strings.Contains(p.Note, "passed on their own and kept: app") { - t.Errorf("the walk does not say what kept its pass: %s", p.Note) - } -} // What the operator reads of the wait (ADR 0253, ADR 0254): it needs them, so it is never quiet, and it offers // no button, since nothing but their own new login can do it. @@ -242,53 +140,6 @@ func TestTheReloginConditionNeedsTheOperatorAndOffersNoButton(t *testing.T) { // When a module not working turns into one waiting for a new login, and back, the clearing line says what it // became, never that it works again (issue 318 review). -func TestAConditionThatBecameTheOtherKindSaysSoWhenItClears(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - store := conditions.NewInMemory() - told := &conditions.Told{} - k := conditions.NewKeeper(ctx, conditions.Options{Store: store, History: store, Teller: told}) - t.Cleanup(func() { k.Close(context.Background()) }) - at := h0 - say := func(rs ...link.ResourceHealth) { - t.Helper() - at = at.Add(time.Second) - for i := range rs { - rs[i].Since = at - } - if err := stateHealth(ctx, open.inventory, k, "laptop", link.Health{Contract: link.ReadinessContract, At: at, - Resources: rs}, at); err != nil { - t.Fatal(err) - } - } - unit := link.ResourceHealth{Module: "lights", Resource: "lights.daemon", Kind: link.KindUnit, Target: "lights.service", - Account: "operator", State: link.StateUnhealthy, Reason: "failed in the account's own service manager (exit-code)"} - account := link.ResourceHealth{Module: "lights", Resource: "lights.operator", Kind: link.KindAccount, Target: "operator", - Account: "operator", State: link.StateUnhealthy, Reason: link.ReasonRelogin + ": operator is in the group lights"} - say(unit) - say(unit) // lights not working - say(account, unit) - say(account, unit) // now it waits for a login - var resolved []string - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline) && len(resolved) == 0; { - time.Sleep(20 * time.Millisecond) - for _, e := range told.Said() { - if e.Change == conditions.ChangeCleared { - resolved = append(resolved, e.Condition.Key+": "+e.Condition.Resolved) - // The clearing line is held to the plain rule too (ADR 0253). - w := conditions.Words{Headline: e.Condition.Headline, Explanation: e.Condition.Explanation, - Resolved: e.Condition.Resolved, Needs: e.Condition.Needs} - if why, ok := conditions.PlainWords(w, "laptop"); !ok { - t.Errorf("%s: its clearing words are not plain: %s", e.Condition.Key, why) - } - } - } - } - if len(resolved) != 1 || !strings.Contains(resolved[0], "module.lights.laptop.unhealthy: lights on laptop now waits only for a new login") { - t.Fatalf("cleared %v; want the not-working condition cleared as now waiting for a login", resolved) - } -} // Which moves add an account group, read from the builds' manifests (issue 318 review). func TestOnlyAMoveThatAddsAnAccountGroupCanBringAWait(t *testing.T) { diff --git a/cmd/mesh-controller/placement_moved_test.go b/cmd/mesh-controller/placement_moved_test.go index 5831cc56..f4a8938b 100644 --- a/cmd/mesh-controller/placement_moved_test.go +++ b/cmd/mesh-controller/placement_moved_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/placement_words_moved_integration_test.go b/cmd/mesh-controller/placement_words_moved_integration_test.go new file mode 100644 index 00000000..f707f74d --- /dev/null +++ b/cmd/mesh-controller/placement_words_moved_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := theWordedMesh(t) + image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) + register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"), + Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}}) + for _, text := range w.Modules { + m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image))) + if err != nil { + t.Fatal(err) + } + register(t, open, m) + } + on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}} + for _, l := range w.Layers { + settings, err := json.Marshal(l.Settings) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(settings)) + if l.Placement != nil { + placement, err := json.Marshal(l.Placement) + if err != nil { + t.Fatal(err) + } + inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`, + l.Node, l.Module, string(placement)) + } + on[l.Node] = append(on[l.Node], l.Module) + } + for _, node := range []string{"anchor", "laptop"} { + if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, node, on[node]...); err != nil { + t.Fatalf("assign %s %v: %v", node, on[node], err) + } + } + return open, w +} + +func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) { + open, _ := aMeshWithTheWordsInItsLayers(t) + sameAsTheGolden(t, open, "before the move") +} + +func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) { + open, w := aMeshWithTheWordsInItsLayers(t) + ctx := t.Context() + inv := open.inventory + asked, err := runMovePlacement(ctx, inv, false) + if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) { + t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers)) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) { + t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held)) + } + sameAsTheGolden(t, open, "after the move was asked without confirming") + + moved, err := runMovePlacement(ctx, inv, true) + if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) { + t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved) + } + if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 { + t.Fatalf("a settings layer still holds a placement key: %v", held) + } + sameAsTheGolden(t, open, "after the move") + + if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 { + t.Fatalf("asked again, the move moved %+v (%v)", again, err) + } + if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") { + t.Fatalf("an empty move says %q", said) + } +} diff --git a/cmd/mesh-controller/placement_words_moved_test.go b/cmd/mesh-controller/placement_words_moved_test.go index 63aa3185..f6562433 100644 --- a/cmd/mesh-controller/placement_words_moved_test.go +++ b/cmd/mesh-controller/placement_words_moved_test.go @@ -4,12 +4,10 @@ import ( "encoding/json" "os" "path/filepath" - "reflect" "sort" "strings" "testing" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/link" ) @@ -45,50 +43,6 @@ func theWordedMesh(t *testing.T) wordedMesh { // aMeshWithTheWordsInItsLayers is aMesh with every module of the worded mesh registered, a route provider on both // machines, and its layers held as the store holds them before the four words move. -func aMeshWithTheWordsInItsLayers(t *testing.T) (*stores, wordedMesh) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - w := theWordedMesh(t) - image := "registry.invalid:5000/library@sha256:" + strings.Repeat("a", 64) - register(t, open, catalogue.Manifest{Module: "proxy", Version: "1", Provides: catalogue.FromAnywhere("route"), - Receives: map[string]string{"route": "/var/lib/proxy/routes.json"}}) - for _, text := range w.Modules { - m, err := catalogue.ParseManifest([]byte(strings.ReplaceAll(string(text), "IMAGE", image))) - if err != nil { - t.Fatal(err) - } - register(t, open, m) - } - on := map[string][]string{"anchor": {"proxy"}, "laptop": {"proxy"}} - for _, l := range w.Layers { - settings, err := json.Marshal(l.Settings) - if err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `insert into settings (node, module, values) select id, $2, $3 from node where name = $1`, - l.Node, l.Module, string(settings)) - if l.Placement != nil { - placement, err := json.Marshal(l.Placement) - if err != nil { - t.Fatal(err) - } - inv.ExecForTest(t, `insert into placement (node, module, values) select id, $2, $3 from node where name = $1`, - l.Node, l.Module, string(placement)) - } - on[l.Node] = append(on[l.Node], l.Module) - } - for _, node := range []string{"anchor", "laptop"} { - if err := inv.SetPublicDomain(ctx, node, node+".example.test"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, node, on[node]...); err != nil { - t.Fatalf("assign %s %v: %v", node, on[node], err) - } - } - return open, w -} // toldOfTheWords is what a machine is told of the modules whose layers say the four words, and of the route // provider that serves their names: those modules' resources and the filter's rules, as JSON a person can diff. @@ -166,44 +120,11 @@ func sameAsTheGolden(t *testing.T, open *stores, when string) { // issue 736). Every live layer that says `endpoints`, `expose`, `reach` or `ports` is seeded as the store holds it // after delivery, on a module shaped as its definition is, and every machine's declaration is composed. What each // is told is byte for byte what the controller before this change composed from the same seed. -func TestADeclarationIsTheSameWhileTheWordsAreStillInTheLayers(t *testing.T) { - open, _ := aMeshWithTheWordsInItsLayers(t) - sameAsTheGolden(t, open, "before the move") -} // **And after the move, still nothing new** (issue 736). The move of placement, asked without confirming, says // every layer that holds one of the four words and moves nothing; confirmed, it moves them all, no settings layer // holds a placement key after, and each machine is told byte for byte what it was told before the move — and what // the controller before this change told it. Asked again, it moves nothing. -func TestADeclarationIsTheSameAfterTheMoveOfPlacement(t *testing.T) { - open, w := aMeshWithTheWordsInItsLayers(t) - ctx := t.Context() - inv := open.inventory - asked, err := runMovePlacement(ctx, inv, false) - if err != nil || !asked.DryRun || len(asked.Moved) != len(w.Layers) { - t.Fatalf("the move, not confirmed, says %+v (%v); %d layers hold the words", asked, err, len(w.Layers)) - } - if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != len(w.Layers) { - t.Fatalf("the move, not confirmed, moved something: %d layers still hold the words", len(held)) - } - sameAsTheGolden(t, open, "after the move was asked without confirming") - - moved, err := runMovePlacement(ctx, inv, true) - if err != nil || moved.DryRun || !reflect.DeepEqual(moved.Moved, asked.Moved) { - t.Fatalf("the move says %+v (%v), it said it would move %+v", moved, err, asked.Moved) - } - if held := inv.QueryForTest(t, `select module from settings where values ?| $1`, catalogue.PlacementKeys()); len(held) != 0 { - t.Fatalf("a settings layer still holds a placement key: %v", held) - } - sameAsTheGolden(t, open, "after the move") - - if again, err := runMovePlacement(ctx, inv, true); err != nil || len(again.Moved) != 0 { - t.Fatalf("asked again, the move moved %+v (%v)", again, err) - } - if said := describeMovePlacement(movePlacementAnswer{}); !strings.Contains(said, "nothing to move") { - t.Fatalf("an empty move says %q", said) - } -} // The verb's command line: a dry run unless confirmed, and a confirmed move says why, recorded as a person's step // of the release, never a repair (issue 736). diff --git a/cmd/mesh-controller/plan_clock_integration_test.go b/cmd/mesh-controller/plan_clock_integration_test.go new file mode 100644 index 00000000..8581e2e3 --- /dev/null +++ b/cmd/mesh-controller/plan_clock_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + recent, err := inv.RecentPlans(ctx, 1) + if err != nil || len(recent) != 1 || len(*asked) != 1 { + t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) + } + before := recent[0] + if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { + t.Fatalf("the plan is not waiting on its build: %+v", before) + } + + for range 3 { + advanceHeld(ctx, open) + } + after, err := inv.PlanByID(ctx, before.ID) + if err != nil { + t.Fatal(err) + } + if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { + t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", + before.Revision, after.Revision, before.Updated, after.Updated) + } + if len(*asked) != 1 { + t.Fatalf("advancing asked again: %v", *asked) + } +} diff --git a/cmd/mesh-controller/plan_clock_test.go b/cmd/mesh-controller/plan_clock_test.go index 26dc6143..dde106be 100644 --- a/cmd/mesh-controller/plan_clock_test.go +++ b/cmd/mesh-controller/plan_clock_test.go @@ -7,7 +7,6 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 296: a plan saved a few seconds ago that has stood in its tier for forty minutes says @@ -79,43 +78,3 @@ func TestAPlansLineCountsFromItsTierNotFromItsLastSave(t *testing.T) { // novox/hq issue 296: a plan whose tier stands still is not written again by every advance — its revision, // its last save and its `plan-moved` stay where its last change left them. -func TestAPlanStandingStillIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "c1aaaaaaaa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 1 { - t.Fatalf("no plan building: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } - if len(*asked) != 1 { - t.Fatalf("advancing asked again: %v", *asked) - } -} diff --git a/cmd/mesh-controller/plan_looks_integration_test.go b/cmd/mesh-controller/plan_looks_integration_test.go new file mode 100644 index 00000000..df7d3c9f --- /dev/null +++ b/cmd/mesh-controller/plan_looks_integration_test.go @@ -0,0 +1,188 @@ +//go:build integration + +package main + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { + var failures, looks atomic.Int32 + block := make(chan struct{}) + l := lookerCounting(&failures, &looks, block) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + l.ask() + eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) + for i := 0; i < 20; i++ { + l.ask() + } + block <- struct{}{} // the first look ends + eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) + block <- struct{}{} + time.Sleep(60 * time.Millisecond) + if n := looks.Load(); n != 2 { + t.Errorf("%d looks for 21 asks; want 2", n) + } +} + +func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { + var failures, looks atomic.Int32 + failures.Store(3) // three looks fail, then one is good + l := lookerCounting(&failures, &looks, nil) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + l.ask() + eventually(t, "four looks", func() bool { return looks.Load() == 4 }) + time.Sleep(200 * time.Millisecond) + if n := looks.Load(); n != 4 { + t.Errorf("%d looks; the good one should have ended the retries at 4", n) + } + l.mu.Lock() + armed, backoff := l.retry != nil, l.backoff + l.mu.Unlock() + if armed || backoff != 0 { + t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) + } +} + +func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + + planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) + t.Cleanup(func() { + walkEnds.mu.Lock() + defer walkEnds.mu.Unlock() + if t := walkEnds.timers["walk-1"]; t != nil { + t.Stop() + } + delete(walkEnds.timers, "walk-1") + }) + eventually(t, "a look", func() bool { return looks.Load() == 1 }) + walkEnds.mu.Lock() + _, armed := walkEnds.timers["walk-1"] + walkEnds.mu.Unlock() + if !armed { + t.Error("a walk saved done is not armed to be recorded") + } + askToLook() // no panic, no looker, once it is taken away +} + +func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startRolloutSilent(ctx, js); err != nil { + t.Fatal(err) + } + c := silentClockOf() + w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} + if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if err := c.Reported(ctx, w); err != nil { + t.Fatal(err) + } + time.Sleep(1800 * time.Millisecond) + if n := looks.Load(); n != 0 { + t.Fatalf("a key a report deleted asked for %d looks", n) + } + if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) +} + +func TestAProvisionerClientsWordAsksForALook(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startProvAppliedLooks(ctx, js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) +} + +func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + var failures, looks atomic.Int32 + l := lookerCounting(&failures, &looks, nil) + lookerMu.Lock() + theLooker = l + lookerMu.Unlock() + t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go l.run(ctx) + if err := startPlanMovedLooks(ctx, js); err != nil { + t.Fatal(err) + } + legacy, err := js.Conn().JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) +} diff --git a/cmd/mesh-controller/plan_looks_test.go b/cmd/mesh-controller/plan_looks_test.go index c2076a4b..be81219a 100644 --- a/cmd/mesh-controller/plan_looks_test.go +++ b/cmd/mesh-controller/plan_looks_test.go @@ -7,10 +7,7 @@ import ( "testing" "time" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) func rolloutOf(sentAgo time.Duration, reported bool) (inventory.Plan, []inventory.Reported) { @@ -112,49 +109,9 @@ func eventually(t *testing.T, what string, ok func() bool) { } // Asks made while a look runs are not lost and are one look after it, not one each. -func TestAsksMadeWhileALookRunsAreOneLookAfterIt(t *testing.T) { - var failures, looks atomic.Int32 - block := make(chan struct{}) - l := lookerCounting(&failures, &looks, block) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "the first look starts", func() bool { return looks.Load() == 1 }) - for i := 0; i < 20; i++ { - l.ask() - } - block <- struct{}{} // the first look ends - eventually(t, "the asks made meanwhile are looked at", func() bool { return looks.Load() == 2 }) - block <- struct{}{} - time.Sleep(60 * time.Millisecond) - if n := looks.Load(); n != 2 { - t.Errorf("%d looks for 21 asks; want 2", n) - } -} // A look that failed a step is looked at again, one at a time with a longer wait each time, and a look that fails // nothing ends the retries (novox/hq ADR 0406 §5; the one named retry). -func TestAFailedLookIsLookedAtAgainAndAGoodOneEndsTheRetries(t *testing.T) { - var failures, looks atomic.Int32 - failures.Store(3) // three looks fail, then one is good - l := lookerCounting(&failures, &looks, nil) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - l.ask() - eventually(t, "four looks", func() bool { return looks.Load() == 4 }) - time.Sleep(200 * time.Millisecond) - if n := looks.Load(); n != 4 { - t.Errorf("%d looks; the good one should have ended the retries at 4", n) - } - l.mu.Lock() - armed, backoff := l.retry != nil, l.backoff - l.mu.Unlock() - if armed || backoff != 0 { - t.Errorf("a retry is armed (%v) or the wait kept (%v) after a good look", armed, backoff) - } -} func TestTheRetryWaitDoublesToItsMost(t *testing.T) { l := newLooker(func(context.Context) bool { return true }) @@ -175,35 +132,6 @@ func TestTheRetryWaitDoublesToItsMost(t *testing.T) { } // A plan saved asks for a look, and a walk saved done is armed to be recorded once its end can be known. -func TestAPlanSavedAsksForALookAndADoneWalkIsArmedToBeRecorded(t *testing.T) { - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - - planKept(inventory.Plan{ID: "walk-1", State: inventory.PlanDone}) - t.Cleanup(func() { - walkEnds.mu.Lock() - defer walkEnds.mu.Unlock() - if t := walkEnds.timers["walk-1"]; t != nil { - t.Stop() - } - delete(walkEnds.timers, "walk-1") - }) - eventually(t, "a look", func() bool { return looks.Load() == 1 }) - walkEnds.mu.Lock() - _, armed := walkEnds.timers["walk-1"] - walkEnds.mu.Unlock() - if !armed { - t.Error("a walk saved done is not armed to be recorded") - } - askToLook() // no panic, no looker, once it is taken away -} // The tier-sends a step made are said once each, per machine, with the modules it was sent. func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { @@ -221,77 +149,9 @@ func TestTheSendsAStepMadeAreSaidPerMachine(t *testing.T) { } // A silent key that expires asks for a look; a key a report deleted does not (issue 974, novox/hq ADR 0406 §5). -func TestAnExpiredSilentKeyAsksForALookAndAReportsDeleteDoesNot(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock(); setSilentClock(nil) }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startRolloutSilent(ctx, js); err != nil { - t.Fatal(err) - } - c := silentClockOf() - w := silentWait{Plan: "plan-5", Tier: 0, Node: "ace"} - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := c.Reported(ctx, w); err != nil { - t.Fatal(err) - } - time.Sleep(1800 * time.Millisecond) - if n := looks.Load(); n != 0 { - t.Fatalf("a key a report deleted asked for %d looks", n) - } - if err := c.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - eventually(t, "the expiry asks for a look", func() bool { return looks.Load() == 1 }) -} // A provisioner client's word on a module's files asks for a look, as a send that handed it modules is applied only // once it has said so (issue 974, review 1): no walk waits for its silent key for what the client already said. -func TestAProvisionerClientsWordAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startProvAppliedLooks(ctx, js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(broker.ProvisionerAppliedSubject("ace", "plex"), []byte(`{"status":"applied"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the client's word", func() bool { return looks.Load() >= 1 }) -} type brokenClock struct{ silentClock } @@ -319,33 +179,3 @@ func TestKeysThatCannotBeReadAreWatchedByTimersAndTheLookFails(t *testing.T) { // A plan moved by a process the serving controller did not spawn — a verb run at the control node's terminal — is // looked at: the controller follows its own plan-moved (issue 974, round 1). -func TestAPlanMovedByAnotherProcessAsksForALook(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - var failures, looks atomic.Int32 - l := lookerCounting(&failures, &looks, nil) - lookerMu.Lock() - theLooker = l - lookerMu.Unlock() - t.Cleanup(func() { lookerMu.Lock(); theLooker = nil; lookerMu.Unlock() }) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go l.run(ctx) - if err := startPlanMovedLooks(ctx, js); err != nil { - t.Fatal(err) - } - legacy, err := js.Conn().JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := legacy.Publish(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), []byte(`{}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a look on the plan moved", func() bool { return looks.Load() >= 1 }) -} diff --git a/cmd/mesh-controller/plan_moved_every_writer_integration_test.go b/cmd/mesh-controller/plan_moved_every_writer_integration_test.go new file mode 100644 index 00000000..2d376641 --- /dev/null +++ b/cmd/mesh-controller/plan_moved_every_writer_integration_test.go @@ -0,0 +1,37 @@ +//go:build integration + +package main + +import ( + "testing" + "time" +) + +func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { + open := windowed(t) + everyHeadWaits(t) + asksWithPaths(t) + heard := sayingEverySave(t) + hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) + if _, bs := walks(t, open); len(bs) != 1 { + t.Fatalf("%d batch(es) opened", len(bs)) + } + heard.noUnsaid(t, open) + cutAt(t, open, t0.Add(91*time.Second)) + if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { + t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) + } + heard.noUnsaid(t, open) + counted := 0 + for _, r := range heard.revs { + counted += len(r) + } + cutAt(t, open, t0.Add(92*time.Second)) + again := 0 + for _, r := range heard.revs { + again += len(r) + } + if again != counted { + t.Errorf("a look that changed nothing said %d more move(s)", again-counted) + } +} diff --git a/cmd/mesh-controller/plan_moved_every_writer_test.go b/cmd/mesh-controller/plan_moved_every_writer_test.go index 4be2f8aa..0973ae79 100644 --- a/cmd/mesh-controller/plan_moved_every_writer_test.go +++ b/cmd/mesh-controller/plan_moved_every_writer_test.go @@ -7,7 +7,6 @@ import ( "strings" "sync" "testing" - "time" "github.com/novox/mesh-controller/internal/inventory" ) @@ -110,31 +109,3 @@ func (s *movesSaid) noUnsaid(t *testing.T, open *stores) { // Live, with the store: the moves a merge makes of a plan — the batch it opens, the walk it is cut into, the tier that // is asked — are each said, and a look that changes nothing says nothing (novox/hq ADR 0406 §2). -func TestEveryMoveAMergeAndTheCutMakeOfAPlanIsSaid(t *testing.T) { - open := windowed(t) - everyHeadWaits(t) - asksWithPaths(t) - heard := sayingEverySave(t) - hear(t, open, catalogueMerge("553b7191claude", "app", t0), t0.Add(time.Second)) - if _, bs := walks(t, open); len(bs) != 1 { - t.Fatalf("%d batch(es) opened", len(bs)) - } - heard.noUnsaid(t, open) - cutAt(t, open, t0.Add(91*time.Second)) - if ws, bs := walks(t, open); len(ws) != 1 || len(bs) != 0 { - t.Fatalf("%d walk(s), %d batch(es) after the cut", len(ws), len(bs)) - } - heard.noUnsaid(t, open) - counted := 0 - for _, r := range heard.revs { - counted += len(r) - } - cutAt(t, open, t0.Add(92*time.Second)) - again := 0 - for _, r := range heard.revs { - again += len(r) - } - if again != counted { - t.Errorf("a look that changed nothing said %d more move(s)", again-counted) - } -} diff --git a/cmd/mesh-controller/planner_records_test.go b/cmd/mesh-controller/planner_records_test.go index 9f56ef4c..8bebf2df 100644 --- a/cmd/mesh-controller/planner_records_test.go +++ b/cmd/mesh-controller/planner_records_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/plans_follow_integration_test.go b/cmd/mesh-controller/plans_follow_integration_test.go new file mode 100644 index 00000000..13968d55 --- /dev/null +++ b/cmd/mesh-controller/plans_follow_integration_test.go @@ -0,0 +1,62 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + // Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound + // below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough + // for the wait to have read the plan before it. + was := planWaitEvery + planWaitEvery = 50 * time.Millisecond + t.Cleanup(func() { planWaitEvery = was }) + now := time.Now().UTC() + walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", + Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + moved := make(chan error, 1) + go func() { + time.Sleep(1500 * time.Millisecond) + p, err := open.inventory.PlanByID(ctx, walk.ID) + if err == nil { + p.Tier, p.State = 1, inventory.PlanRolling + err = open.inventory.SavePlan(ctx, &p) + } + moved <- err + }() + began := time.Now() + var got planWaited + out := printed(t, func() error { + return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) + }) + if err := <-moved; err != nil { + t.Fatal(err) + } + if err := json.Unmarshal([]byte(out), &got); err != nil { + t.Fatalf("plans wait did not answer data: %v: %q", err, out) + } + if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { + t.Fatalf("the wait answered %+v", got) + } + if waited := time.Since(began); waited > 3*time.Second { + t.Errorf("it answered after %s, not when the plan moved", waited) + } + out = printed(t, func() error { + return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"}) + }) + if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { + t.Fatalf("a plan that did not move answered %+v (%v)", got, err) + } +} diff --git a/cmd/mesh-controller/plans_follow_test.go b/cmd/mesh-controller/plans_follow_test.go index 0f25870d..0bd63bdc 100644 --- a/cmd/mesh-controller/plans_follow_test.go +++ b/cmd/mesh-controller/plans_follow_test.go @@ -135,56 +135,6 @@ func TestThePlansVerbComposesAWait(t *testing.T) { // Live, with the store: `plans wait` run as the verb runs it answers when the plan reaches its next tier, as // data, and says the bound passed when nothing moved (novox/hq issue 229). -func TestAPlanFollowedFromTheStoreAnswersWhenItMoves(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - // Read every 50ms rather than every second, so the wait answers as soon as the plan moves and the bound - // below passes in a fraction of a second (novox/hq issue 591); the move still waits 1.5s, long enough - // for the wait to have read the plan before it. - was := planWaitEvery - planWaitEvery = 50 * time.Millisecond - t.Cleanup(func() { planWaitEvery = was }) - now := time.Now().UTC() - walk := inventory.Plan{ID: "plan-229", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c229c229", - Created: now, State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {}, "b": {}}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - moved := make(chan error, 1) - go func() { - time.Sleep(1500 * time.Millisecond) - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err == nil { - p.Tier, p.State = 1, inventory.PlanRolling - err = open.inventory.SavePlan(ctx, &p) - } - moved <- err - }() - began := time.Now() - var got planWaited - out := printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--until", "tier", "--bound", "15s"}) - }) - if err := <-moved; err != nil { - t.Fatal(err) - } - if err := json.Unmarshal([]byte(out), &got); err != nil { - t.Fatalf("plans wait did not answer data: %v: %q", err, out) - } - if got.Came != "tier" || got.Plan.Tier != 1 || got.Plan.State != inventory.PlanRolling { - t.Fatalf("the wait answered %+v", got) - } - if waited := time.Since(began); waited > 3*time.Second { - t.Errorf("it answered after %s, not when the plan moved", waited) - } - out = printed(t, func() error { - return plansCommand(ctx, []string{"wait", walk.ID, "--bound", "300ms"}) - }) - if err := json.Unmarshal([]byte(out), &got); err != nil || got.Came != "bound" || got.Until != "moved" { - t.Fatalf("a plan that did not move answered %+v (%v)", got, err) - } -} // The verb's schema says the longest bound in words, and the words are the value (review 75 on #266): the // catalogue cannot read maxPlanWait, so this holds them together. diff --git a/cmd/mesh-controller/plans_one_at_a_time_test.go b/cmd/mesh-controller/plans_one_at_a_time_test.go index 1eb4ec3c..3f3eef97 100644 --- a/cmd/mesh-controller/plans_one_at_a_time_test.go +++ b/cmd/mesh-controller/plans_one_at_a_time_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/predefined_integration_test.go b/cmd/mesh-controller/predefined_integration_test.go new file mode 100644 index 00000000..e0d28433 --- /dev/null +++ b/cmd/mesh-controller/predefined_integration_test.go @@ -0,0 +1,101 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { + withoutAPublishedList(t) + f, manifests := templatedMesh(t) + + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", + notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) + } + for _, c := range []struct{ manifest, list, says string }{ + {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, + {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, + {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, + } { + v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), + "modules/notifier/module.json", catalogue.PredefinedFile) + if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { + t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) + } + } +} + +func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { + withoutAPublishedList(t) + open := aMesh(t) + ctx := t.Context() + + // A manifest naming font-size registers, and its build's list is published. + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, + catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { + t.Fatalf("a module naming font-size was not registered: %v", err) + } + kept, has, err := open.inventory.PredefinedKept(ctx) + if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { + t.Fatalf("the list was not published: %+v %v %v", kept, has, err) + } + if got := catalogue.PublishedPredefined(); len(got) != 2 { + t.Fatalf("the controller judges against %v, not the list published", got) + } + + // One naming a key not on the list, or of another type, is refused and never enters the catalogue. + for i, c := range []struct{ manifest, says string }{ + {notifierFallingBackTo("font-scale", "number"), "not on the list"}, + {notifierFallingBackTo("font-size", "string"), "which is a number"}, + } { + id := []string{"b-2", "b-3"}[i] + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || + !strings.Contains(err.Error(), c.says) { + t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) + } + } + src, err := open.inventory.SourceOf(ctx, "notifier") + if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { + t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) + } + + // A newer build's list, which drops agent-managed-settings, replaces the published one. + smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", + notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { + t.Fatal(err) + } + // The older build's outcome heard later registers nothing new about the list: the newer stands. + if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", + notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { + t.Fatal(err) + } + kept, _, _ = open.inventory.PredefinedKept(ctx) + if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { + t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) + } + + // A build of another repository publishes nothing, and its manifest is judged against the published list. + other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) + other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" + other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, + `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { + t.Fatalf("a module of another repository naming a published key: %v", err) + } + if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { + t.Fatalf("a build of another repository published its list: %+v", kept) + } + other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", + "number"), `"notifier"`, `"chimes"`, 1)) + if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { + t.Fatalf("a key only another repository's list holds was registered: %v", err) + } +} diff --git a/cmd/mesh-controller/predefined_test.go b/cmd/mesh-controller/predefined_test.go index 25f09c50..c9cf3cd6 100644 --- a/cmd/mesh-controller/predefined_test.go +++ b/cmd/mesh-controller/predefined_test.go @@ -95,28 +95,6 @@ func writeTheList(t *testing.T, tree, list string) string { return tree } -func TestTheMergeGateJudgesEachManifestAgainstTheListOfTheTree(t *testing.T) { - withoutAPublishedList(t) - f, manifests := templatedMesh(t) - - v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", - notifierFallingBackTo("font-size", "number"))), theFirstKeys), "modules/notifier/module.json") - if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { - t.Fatalf("a module naming font-size on the tree's list failed the gate (%v):\n%s", v.Modules, v.Report()) - } - for _, c := range []struct{ manifest, list, says string }{ - {notifierFallingBackTo("font-scale", "number"), theFirstKeys, `"font-scale", which is not on the list`}, - {notifierFallingBackTo("font-size", "string"), theFirstKeys, "which is a number"}, - {inlineNotifier, `{"font-size": {"type": "number", "description": "points", "unit": "pt"}}`, `unknown field "unit"`}, - } { - v := gateJudged(t, f, writeTheList(t, aTree(t, withEdit(manifests, "notifier", c.manifest)), c.list), - "modules/notifier/module.json", catalogue.PredefinedFile) - if v.Verdict != "fail" || !strings.Contains(v.Report(), c.says) { - t.Errorf("wanted the gate to fail saying %q:\n%s", c.says, v.Report()) - } - } -} - // catalogueBuild is a trunk build of the catalogue carrying the list at a generation. func catalogueBuild(id, manifest, list string, generation int) link.BuildResult { r := link.BuildResult{ID: id, Repository: "http://forge.internal:20000/novox/mesh-catalog.git", @@ -127,71 +105,3 @@ func catalogueBuild(id, manifest, list string, generation int) link.BuildResult } return r } - -func TestRegistrationPublishesTheNewestTrunkListAndJudgesEveryModuleAgainstIt(t *testing.T) { - withoutAPublishedList(t) - open := aMesh(t) - ctx := t.Context() - - // A manifest naming font-size registers, and its build's list is published. - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, - catalogueBuild("b-1", notifierFallingBackTo("font-size", "number"), theFirstKeys, 10))); err != nil { - t.Fatalf("a module naming font-size was not registered: %v", err) - } - kept, has, err := open.inventory.PredefinedKept(ctx) - if err != nil || !has || kept.Generation != 10 || kept.Build != "b-1" { - t.Fatalf("the list was not published: %+v %v %v", kept, has, err) - } - if got := catalogue.PublishedPredefined(); len(got) != 2 { - t.Fatalf("the controller judges against %v, not the list published", got) - } - - // One naming a key not on the list, or of another type, is refused and never enters the catalogue. - for i, c := range []struct{ manifest, says string }{ - {notifierFallingBackTo("font-scale", "number"), "not on the list"}, - {notifierFallingBackTo("font-size", "string"), "which is a number"}, - } { - id := []string{"b-2", "b-3"}[i] - if _, _, err := takeIn(ctx, open.inventory, catalogueBuild(id, c.manifest, theFirstKeys, 10)); err == nil || - !strings.Contains(err.Error(), c.says) { - t.Errorf("wanted the registration refused saying %q, got %v", c.says, err) - } - } - src, err := open.inventory.SourceOf(ctx, "notifier") - if err != nil || src.BuiltFrom != strings.Repeat("1", 12) { - t.Fatalf("a refused build replaced the registered one: %+v %v", src, err) - } - - // A newer build's list, which drops agent-managed-settings, replaces the published one. - smaller := `{"font-size": {"type": "number", "description": "points", "minimum": 6, "maximum": 72}}` - if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-4", - notifierFallingBackTo("font-size", "number"), smaller, 12)); err != nil { - t.Fatal(err) - } - // The older build's outcome heard later registers nothing new about the list: the newer stands. - if _, _, err := takeIn(ctx, open.inventory, catalogueBuild("b-5", - notifierFallingBackTo("font-size", "number"), theFirstKeys, 11)); err != nil { - t.Fatal(err) - } - kept, _, _ = open.inventory.PredefinedKept(ctx) - if kept.Build != "b-4" || len(kept.Keys) != 1 || len(catalogue.PublishedPredefined()) != 1 { - t.Fatalf("an older build's list heard later replaced the newer one: %+v", kept) - } - - // A build of another repository publishes nothing, and its manifest is judged against the published list. - other := catalogueBuild("b-6", notifierFallingBackTo("font-size", "number"), theFirstKeys, 99) - other.Repository, other.Source.Repository = "http://forge.internal:20000/novox/notifier.git", "novox/notifier" - other.Manifest = json.RawMessage(strings.Replace(notifierFallingBackTo("font-size", "number"), `"notifier"`, - `"chimes"`, 1)) - if _, _, err := takeIn(ctx, open.inventory, asTheOperator(t, open.inventory, other)); err != nil { - t.Fatalf("a module of another repository naming a published key: %v", err) - } - if kept, _, _ = open.inventory.PredefinedKept(ctx); kept.Build != "b-4" { - t.Fatalf("a build of another repository published its list: %+v", kept) - } - other.ID, other.Manifest = "b-7", json.RawMessage(strings.Replace(notifierFallingBackTo("agent-managed-settings", - "number"), `"notifier"`, `"chimes"`, 1)) - if _, _, err := takeIn(ctx, open.inventory, other); err == nil || !strings.Contains(err.Error(), "not on the list") { - t.Fatalf("a key only another repository's list holds was registered: %v", err) - } -} diff --git a/cmd/mesh-controller/proposals_integration_test.go b/cmd/mesh-controller/proposals_integration_test.go new file mode 100644 index 00000000..7a58d0c2 --- /dev/null +++ b/cmd/mesh-controller/proposals_integration_test.go @@ -0,0 +1,135 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestSetOnAWarrantJudgesTheLayerAndKeepsWhoApprovedIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + // y is a preference with a default, so a layer may leave it out; x is the operator's own (ADR 0262). + Settings: map[string]catalogue.SettingDeclaration{"y": {Kind: "preference", Default: "10", Why: "a size"}}, + Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}, + // A trusted file (unmarked), so its keys are the terminal's — and now the warrant's (novox/hq ADR 0277). + {"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "content": "x = ${setting:x}\ny = ${setting:y}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + set := setLayerIn(open) + now := time.Now() + first := map[string]any{"x": "1", "y": "2"} + p := settingsProposal{Module: "notes", Node: "laptop", Values: first, From: "g14/claude-code", At: now, + Digest: layerDigest(first), BeforeDigest: layerDigest(nil)} + changed, err := set(ctx, p, "s9", "approved by the operator, as telegram identity 42 via telegram at 14:05 (ask s9)") + if err != nil || !strings.Contains(changed, "+ x") { + t.Fatalf("%q %v", changed, err) + } + layer, has, err := open.inventory.Layer(ctx, "laptop", "notes") + if err != nil || !has || layer["x"] != "1" { + t.Fatalf("the layer: %v %v %v", layer, has, err) + } + setBy, _, has, err := open.inventory.LayerOrigin(ctx, "laptop", "notes") + if err != nil || !has || !strings.HasPrefix(setBy, "approved by the operator, as telegram identity 42 via telegram") { + t.Fatalf("who set it: %q %v", setBy, err) + } + // Shown by `settings show`, at the terminal and through the verb. + out := captureStdout(t, func() { + if err := atTheTerminal(t, "settings", "show", "notes", "--node", "laptop"); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(out, "approved by the operator, as telegram identity 42 via telegram") { + t.Errorf("settings show says:\n%s", out) + } + // The same proposal again: the layer is no longer the one the operator was shown it against. + if _, err := set(ctx, p, "s9", "approved …"); err == nil || !strings.Contains(err.Error(), "changed since the operator was shown") { + t.Errorf("a stale proposal: %v", err) + } + // A removal not meant is refused; one meant is taken, and the history says who had set the layer. + second := map[string]any{"x": "1"} + q := settingsProposal{Module: "notes", Node: "laptop", Values: second, Before: first, HadLayer: true, + From: "g14/claude-code", At: now, Digest: layerDigest(second), BeforeDigest: layerDigest(first)} + if _, err := set(ctx, q, "s10", "approved …"); err == nil || !strings.Contains(err.Error(), "removal was not meant") { + t.Errorf("a silent removal: %v", err) + } + // A layer the mesh refuses is refused here too, with the same words as at the terminal. + bad := q + bad.Values, bad.Digest = map[string]any{"x": "a\nb", "y": "2"}, layerDigest(map[string]any{"x": "a\nb", "y": "2"}) + if _, err := set(ctx, bad, "s11", "approved …"); err == nil || !strings.Contains(err.Error(), "line break") { + t.Errorf("a line break on a warrant: %v", err) + } + if layer, _, _ := open.inventory.Layer(ctx, "laptop", "notes"); layer["y"] != "2" { + t.Fatalf("a refused act changed the layer: %v", layer) + } + q.Replace = true + if _, err := set(ctx, q, "s10", "approved later"); err != nil { + t.Fatal(err) + } + past, err := open.inventory.SettingsHistory(ctx, "laptop", "notes") + if err != nil || len(past) != 1 || !strings.HasPrefix(past[0].SetBy, "approved by the operator") { + t.Errorf("the history: %+v %v", past, err) + } + // And a clear, keeping who cleared it with the copy. + c := settingsProposal{Module: "notes", Node: "laptop", Clear: true, Before: second, HadLayer: true, From: "x", At: now, + Digest: layerDigest(second), BeforeDigest: layerDigest(second)} + if _, err := set(ctx, c, "s12", "approved by the operator at 15:00"); err != nil { + t.Fatal(err) + } + if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { + t.Error("the layer was not cleared") + } + past, _ = open.inventory.SettingsHistory(ctx, "laptop", "notes") + if len(past) != 2 || !strings.Contains(past[0].SetBy, "cleared approved by the operator at 15:00") { + t.Errorf("the history after a clear: %+v", past) + } +} + +func TestALayerSaysWhoSetIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "trusted": false, + "content": "x = ${setting:x}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"1"}`, "--node", "laptop"); err != nil { + t.Fatal(err) + } + setBy, _, _, _ := open.inventory.LayerOrigin(ctx, "laptop", "notes") + if !strings.HasPrefix(setBy, "set at the controller's terminal by ") { + t.Errorf("at the terminal: %q", setBy) + } + if err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", "values": `{"x":"2"}`}); err != nil { + t.Fatal(err) + } + setBy, _, _, _ = open.inventory.LayerOrigin(ctx, "laptop", "notes") + if !strings.HasPrefix(setBy, "set by ") || !strings.Contains(setBy, "through settings") { + t.Errorf("through the verb: %q", setBy) + } +} + +func TestATrustedSettingThroughAVerbNamesTheProposal(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", + "values": `{"places":{"data":{"path":"/srv/notes","owner":"1001:1001"}}}`}) + if err == nil || !strings.Contains(err.Error(), "issue 339") || !strings.Contains(err.Error(), "propose") { + t.Errorf("%v", err) + } + if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { + t.Error("a layer was kept") + } +} diff --git a/cmd/mesh-controller/proposals_test.go b/cmd/mesh-controller/proposals_test.go index 5a6f759f..c791c9f3 100644 --- a/cmd/mesh-controller/proposals_test.go +++ b/cmd/mesh-controller/proposals_test.go @@ -14,7 +14,6 @@ import ( "git.novox.be/novox/mesh-sdk/go/asks" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/outward" ) @@ -708,132 +707,11 @@ func TestTheSettingsVerbComposesProposeAndProposals(t *testing.T) { // setLayerIn sets the layer as the terminal does — judged, the removal meant, the history kept — with who approved it // beside it; and refuses once the layer is no longer the one the operator was shown the change against. -func TestSetOnAWarrantJudgesTheLayerAndKeepsWhoApprovedIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - // y is a preference with a default, so a layer may leave it out; x is the operator's own (ADR 0262). - Settings: map[string]catalogue.SettingDeclaration{"y": {Kind: "preference", Default: "10", Why: "a size"}}, - Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}, - // A trusted file (unmarked), so its keys are the terminal's — and now the warrant's (novox/hq ADR 0277). - {"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "content": "x = ${setting:x}\ny = ${setting:y}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - set := setLayerIn(open) - now := time.Now() - first := map[string]any{"x": "1", "y": "2"} - p := settingsProposal{Module: "notes", Node: "laptop", Values: first, From: "g14/claude-code", At: now, - Digest: layerDigest(first), BeforeDigest: layerDigest(nil)} - changed, err := set(ctx, p, "s9", "approved by the operator, as telegram identity 42 via telegram at 14:05 (ask s9)") - if err != nil || !strings.Contains(changed, "+ x") { - t.Fatalf("%q %v", changed, err) - } - layer, has, err := open.inventory.Layer(ctx, "laptop", "notes") - if err != nil || !has || layer["x"] != "1" { - t.Fatalf("the layer: %v %v %v", layer, has, err) - } - setBy, _, has, err := open.inventory.LayerOrigin(ctx, "laptop", "notes") - if err != nil || !has || !strings.HasPrefix(setBy, "approved by the operator, as telegram identity 42 via telegram") { - t.Fatalf("who set it: %q %v", setBy, err) - } - // Shown by `settings show`, at the terminal and through the verb. - out := captureStdout(t, func() { - if err := atTheTerminal(t, "settings", "show", "notes", "--node", "laptop"); err != nil { - t.Fatal(err) - } - }) - if !strings.Contains(out, "approved by the operator, as telegram identity 42 via telegram") { - t.Errorf("settings show says:\n%s", out) - } - // The same proposal again: the layer is no longer the one the operator was shown it against. - if _, err := set(ctx, p, "s9", "approved …"); err == nil || !strings.Contains(err.Error(), "changed since the operator was shown") { - t.Errorf("a stale proposal: %v", err) - } - // A removal not meant is refused; one meant is taken, and the history says who had set the layer. - second := map[string]any{"x": "1"} - q := settingsProposal{Module: "notes", Node: "laptop", Values: second, Before: first, HadLayer: true, - From: "g14/claude-code", At: now, Digest: layerDigest(second), BeforeDigest: layerDigest(first)} - if _, err := set(ctx, q, "s10", "approved …"); err == nil || !strings.Contains(err.Error(), "removal was not meant") { - t.Errorf("a silent removal: %v", err) - } - // A layer the mesh refuses is refused here too, with the same words as at the terminal. - bad := q - bad.Values, bad.Digest = map[string]any{"x": "a\nb", "y": "2"}, layerDigest(map[string]any{"x": "a\nb", "y": "2"}) - if _, err := set(ctx, bad, "s11", "approved …"); err == nil || !strings.Contains(err.Error(), "line break") { - t.Errorf("a line break on a warrant: %v", err) - } - if layer, _, _ := open.inventory.Layer(ctx, "laptop", "notes"); layer["y"] != "2" { - t.Fatalf("a refused act changed the layer: %v", layer) - } - q.Replace = true - if _, err := set(ctx, q, "s10", "approved later"); err != nil { - t.Fatal(err) - } - past, err := open.inventory.SettingsHistory(ctx, "laptop", "notes") - if err != nil || len(past) != 1 || !strings.HasPrefix(past[0].SetBy, "approved by the operator") { - t.Errorf("the history: %+v %v", past, err) - } - // And a clear, keeping who cleared it with the copy. - c := settingsProposal{Module: "notes", Node: "laptop", Clear: true, Before: second, HadLayer: true, From: "x", At: now, - Digest: layerDigest(second), BeforeDigest: layerDigest(second)} - if _, err := set(ctx, c, "s12", "approved by the operator at 15:00"); err != nil { - t.Fatal(err) - } - if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { - t.Error("the layer was not cleared") - } - past, _ = open.inventory.SettingsHistory(ctx, "laptop", "notes") - if len(past) != 2 || !strings.Contains(past[0].SetBy, "cleared approved by the operator at 15:00") { - t.Errorf("the history after a clear: %+v", past) - } -} // A layer set at the terminal says so, and one set through a verb names the verb (novox/hq ADR 0277). -func TestALayerSaysWhoSetIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", "trusted": false, - "content": "x = ${setting:x}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"1"}`, "--node", "laptop"); err != nil { - t.Fatal(err) - } - setBy, _, _, _ := open.inventory.LayerOrigin(ctx, "laptop", "notes") - if !strings.HasPrefix(setBy, "set at the controller's terminal by ") { - t.Errorf("at the terminal: %q", setBy) - } - if err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", "values": `{"x":"2"}`}); err != nil { - t.Fatal(err) - } - setBy, _, _, _ = open.inventory.LayerOrigin(ctx, "laptop", "notes") - if !strings.HasPrefix(setBy, "set by ") || !strings.Contains(setBy, "through settings") { - t.Errorf("through the verb: %q", setBy) - } -} // A trusted setting is still refused through the settings verb (novox/hq issue 339), and the refusal now names the // proposal as the way. -func TestATrustedSettingThroughAVerbNamesTheProposal(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "data", "type": "directory", "mode": "0755"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - err := throughVerb(t, "settings", map[string]any{"module": "notes", "node": "laptop", - "values": `{"places":{"data":{"path":"/srv/notes","owner":"1001:1001"}}}`}) - if err == nil || !strings.Contains(err.Error(), "issue 339") || !strings.Contains(err.Error(), "propose") { - t.Errorf("%v", err) - } - if _, has, _ := open.inventory.Layer(ctx, "laptop", "notes"); has { - t.Error("a layer was kept") - } -} // captureStdout runs f and answers what it printed to standard output. func captureStdout(t *testing.T, f func()) string { diff --git a/cmd/mesh-controller/provider_hold_test.go b/cmd/mesh-controller/provider_hold_test.go index d07e9c7b..8597e133 100644 --- a/cmd/mesh-controller/provider_hold_test.go +++ b/cmd/mesh-controller/provider_hold_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/provisioner_account_integration_test.go b/cmd/mesh-controller/provisioner_account_integration_test.go new file mode 100644 index 00000000..75ede30b --- /dev/null +++ b/cmd/mesh-controller/provisioner_account_integration_test.go @@ -0,0 +1,79 @@ +//go:build integration + +package main + +import ( + "slices" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + m := aProvisioner() + register(t, open, m) + if _, err := assign(ctx, open, "laptop", m.Module); err != nil { + t.Fatal(err) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil { + t.Fatal("a push would seal a placeholder for the client's account before it was issued") + } + + if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted { + t.Fatalf("the client's account is not minted: %v %v", minted, err) + } + states, err := inv.SecretsOf(ctx, "laptop", m.Module) + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) { + t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states) + } + if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil { + t.Fatalf("an issued client account is refused: %v", err) + } + if !runsTheProvisionerClient(m) { + t.Fatal("the module is not read as running the client") + } + unread := m + unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} + if runsTheProvisionerClient(unread) { + t.Fatal("a module with nowhere to read the client's account is read as running it") + } +} + +func TestAssigningTheClientIssuesItsAccount(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + client := aClientModule() + register(t, open, client) + out, err := assign(ctx, open, "laptop", client.Module) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "bus credential") { + t.Fatalf("assigning the client said nothing of its account:\n%s", out) + } +} + +// A module that runs the client and declares no broker of its own, as the catalogue's mesh-provisioner-client does, +// may be issued: the client's account alone (issue 917). Before, `module issue` refused it for having no broker, +// while a push refused it for having no account, so no machine could run the client. +func TestAClientWithoutABrokerOfItsOwnMayBeIssued(t *testing.T) { + if err := mayIssue(aClientModule()); err != nil { + t.Fatalf("the client alone is refused an account: %v", err) + } + neither := aBot("bot", false) + if err := mayIssue(neither); err == nil { + t.Fatal("a module with nowhere to read any account may be issued one") + } +} diff --git a/cmd/mesh-controller/provisioner_account_test.go b/cmd/mesh-controller/provisioner_account_test.go index 20dc7b15..7cc212d6 100644 --- a/cmd/mesh-controller/provisioner_account_test.go +++ b/cmd/mesh-controller/provisioner_account_test.go @@ -1,13 +1,8 @@ package main import ( - "slices" - "strings" - "testing" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" ) func aProvisioner() catalogue.Manifest { @@ -23,70 +18,6 @@ func aProvisioner() catalogue.Manifest { // A machine's provisioner client is issued an account of its own, under its own name, sealed to the machine as the // module's own secret its service reads (novox/hq ADR 0349 clause 11, design 57 P3); until it is, a push refuses // to invent one, as it does for a module's `broker`. -func TestTheProvisionerClientsAccountIsIssuedForItsServiceAndNeverInvented(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - m := aProvisioner() - register(t, open, m) - if _, err := assign(ctx, open, "laptop", m.Module); err != nil { - t.Fatal(err) - } - if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err == nil { - t.Fatal("a push would seal a placeholder for the client's account before it was issued") - } - - if err := issueProvisionerClient(ctx, inv, m, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, minted, err := inv.BusUserHash(ctx, "provisioner.laptop"); err != nil || !minted { - t.Fatalf("the client's account is not minted: %v %v", minted, err) - } - states, err := inv.SecretsOf(ctx, "laptop", m.Module) - if err != nil { - t.Fatal(err) - } - if !slices.ContainsFunc(states, func(s inventory.SecretState) bool { return s.Name == broker.ProvisionerSecret }) { - t.Fatalf("the client's account is not sealed to the machine for its service: %+v", states) - } - if err := busCredentialIssued(ctx, inv, "laptop", m.Module, broker.ProvisionerSecret); err != nil { - t.Fatalf("an issued client account is refused: %v", err) - } - if !runsTheProvisionerClient(m) { - t.Fatal("the module is not read as running the client") - } - unread := m - unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} - if runsTheProvisionerClient(unread) { - t.Fatal("a module with nowhere to read the client's account is read as running it") - } -} - -// A module that runs the client and declares no broker of its own, as the catalogue's mesh-provisioner-client does, -// may be issued: the client's account alone (issue 917). Before, `module issue` refused it for having no broker, -// while a push refused it for having no account, so no machine could run the client. -func TestAClientWithoutABrokerOfItsOwnMayBeIssued(t *testing.T) { - if err := mayIssue(aClientModule()); err != nil { - t.Fatalf("the client alone is refused an account: %v", err) - } - neither := aBot("bot", false) - if err := mayIssue(neither); err == nil { - t.Fatal("a module with nowhere to read any account may be issued one") - } -} // Assigning the client tries to issue the client's account, as assigning any module issues its broker (issue 917): // without a bus to issue on, the line says so and names the verb, never nothing. -func TestAssigningTheClientIssuesItsAccount(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - client := aClientModule() - register(t, open, client) - out, err := assign(ctx, open, "laptop", client.Module) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(out, "bus credential") { - t.Fatalf("assigning the client said nothing of its account:\n%s", out) - } -} diff --git a/cmd/mesh-controller/push_recreates_test.go b/cmd/mesh-controller/push_recreates_test.go index 1fc486a0..2b73fd29 100644 --- a/cmd/mesh-controller/push_recreates_test.go +++ b/cmd/mesh-controller/push_recreates_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/queue_integration_test.go b/cmd/mesh-controller/queue_integration_test.go new file mode 100644 index 00000000..91d0d484 --- /dev/null +++ b/cmd/mesh-controller/queue_integration_test.go @@ -0,0 +1,492 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + + said, err := retryPlan(ctx, open, failed.ID) + if err != nil { + t.Fatal(err) + } + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + a := p.Modules["a"] + if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" { + t.Fatalf("after retry the plan is %s and a is %+v", p.State, a) + } + if !strings.Contains(said, a.Build) { + t.Errorf("retry does not say the new id: %q", said) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + + // The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows. + planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1") + if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding { + t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note) + } + asking, _ := link.BuildAskedAt(a.Build) + planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build) + p, err = open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" { + t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"]) + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } +} + +func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + if err := rebuildCommand(ctx, []string{"a"}); err != nil { + t.Fatal(err) + } + p, err := open.inventory.PlanByID(ctx, failed.ID) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" { + t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"]) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } + // b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone. + if err := rebuildCommand(ctx, []string{"b"}); err != nil { + t.Fatal(err) + } + if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil { + t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"]) + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } +} + +func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) { + t.Helper() + url := testbus.URL(t) + t.Setenv(broker.NATSVar, url) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}, + Emits: []string{"started", "built", "log.*", "paused.*"}} + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") + if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{ + link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil { + t.Fatal(err) + } + if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") + _ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine)) + _ = js.Context().PurgeStream(broker.EventsStream) + }) + ask := func(id, repository string) link.BuildRequest { + r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}} + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { + t.Fatal(err) + } + return r + } + return js, ask +} + +func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + twoTiers(t, open) + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + asked, _ := link.BuildAskedAt(id) + plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}} + if err := open.inventory.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id { + t.Fatalf("the queue reads %+v", q) + } + if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") || + strings.Contains(text, "secret-ish") { + t.Errorf("the queue says:\n%s", text) + } + + if err := cancelCommand(ctx, []string{id}); err != nil { + t.Fatal(err) + } + if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { + t.Fatalf("the ask is still queued: %+v", q.Asks) + } + if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled { + t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err) + } + b, found, err := open.inventory.BuildByID(ctx, id) + if err != nil || !found || b.Failed != link.CancelledByHand { + t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err) + } + p, err := open.inventory.PlanByID(ctx, plan.ID) + if err != nil { + t.Fatal(err) + } + if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand { + t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"]) + } + if err := cancelCommand(ctx, []string{id}); err == nil { + t.Error("an ask cancelled already was cancelled again") + } +} + +func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + start := time.Now() + dead := link.NewBuildID(start) + ask(dead, "https://forge.example/novox/dead.git") + deadOne(t, js) + var waiting []string + for i := 1; i <= 2; i++ { + id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond)) + waiting = append(waiting, id) + ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i)) + } + + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 { + t.Fatalf("the queue reads %+v", q) + } + said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") { + t.Errorf("clear said:\n%s", said) + } + q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine) + if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead { + t.Fatalf("after clear the queue is %+v", q.Asks) + } + if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil { + t.Fatal(err) + } + if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { + t.Fatalf("clear --dead left %+v", q.Asks) + } + for _, id := range append(waiting, dead) { + if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand { + t.Errorf("%s is recorded as %+v", id, b) + } + } +} + +func TestCancelRefusesAnAskInFlight(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + a, _ := q.Find(id) + if a.State != link.AskInFlight || a.On != "ace" { + t.Fatalf("the taken ask reads %+v", a) + } + _, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a) + if err == nil || !strings.Contains(err.Error(), "kill "+id) { + t.Fatalf("an ask in flight was cancelled: %v", err) + } + if _, found, _ := open.inventory.BuildByID(ctx, id); found { + t.Error("a refused cancel recorded an outcome") + } +} + +func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { + js, _ := aBuildQueue(t) + ctx := t.Context() + asked := map[string]string{} + handlers := map[string]link.ToolHandler{ + "kill": func(_ context.Context, raw json.RawMessage) (any, error) { + var args struct{ ID string } + _ = json.Unmarshal(raw, &args) + asked["kill"] = args.ID + if args.ID != "build-running" { + return nil, fmt.Errorf("ace is not building %s", args.ID) + } + return map[string]any{"said": "killed " + args.ID}, nil + }, + "pause": func(context.Context, json.RawMessage) (any, error) { + asked["pause"] = "ace" + return map[string]any{"said": "ace is paused"}, nil + }, + } + stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + for _, id := range []string{"build-running", "build-other"} { + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { + t.Fatal(err) + } + } + if err := killCommand(ctx, []string{"build-running"}); err != nil { + t.Fatal(err) + } + if asked["kill"] != "build-running" { + t.Fatalf("the holder was asked %v", asked) + } + if err := killCommand(ctx, []string{"build-other"}); err == nil { + t.Error("the holder's refusal was not the command's") + } + if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") { + t.Errorf("a build nobody started: %v", err) + } + if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" { + t.Fatalf("pause: %v %v", err, asked) + } + if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil { + t.Error("a machine nothing answers on was resumed") + } +} + +func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + var sentTo [][]string + was := sendRollout + sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { + sentTo = append(sentTo, names) + return names, nil + } + t.Cleanup(func() { sendRollout = was }) + + // a records: a person's choice, since the default rolls out (novox/hq ADR 0236). + if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil { + t.Fatal(err) + } + long := time.Now().UTC().Add(-2 * time.Hour) + stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a stopped at its first machine in tier 0: laptop refused what it was sent", + // Asked by the plan, as every module a plan holds built is (novox/hq issue 754). + Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long, + Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} + if err := open.inventory.SavePlan(ctx, &stopped); err != nil { + t.Fatal(err) + } + + // A newer plan holding a refuses it: sending the older build would put it back. + newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour), + State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}} + if err := open.inventory.SavePlan(ctx, &newer); err != nil { + t.Fatal(err) + } + if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") { + t.Fatalf("retried under a newer plan: %v", err) + } + newer.State = inventory.PlanSuperseded + if err := open.inventory.SavePlan(ctx, &newer); err != nil { + t.Fatal(err) + } + + said, err := retryPlan(ctx, open, stopped.ID) + if err != nil { + t.Fatal(err) + } + if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") { + t.Fatalf("sent %v; said %q", sentTo, said) + } + p, err := open.inventory.PlanByID(ctx, stopped.ID) + if err != nil { + t.Fatal(err) + } + a := p.Modules["a"] + if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" { + t.Fatalf("after retry the plan is %s, a %+v", p.State, a) + } + // And it goes on: a records (its policy sends nothing more), so the next tier is asked. + advancePlans(ctx, open) + if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" { + t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"]) + } + if len(*asked) != 1 { + t.Fatalf("asked %v", *asked) + } +} + +func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := time.Now().UTC().Add(-time.Minute) + plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked, + State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}} + if err := open.inventory.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay") + p, _ := open.inventory.PlanByID(ctx, plan.ID) + if p.Modules["a"].State != "asked" { + t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"]) + } + // From the records too: a later build of the module recorded is not the plan's. + recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}} + if settleFromRecords(&p, p.Tiers[0], recorded, nil) { + t.Fatalf("the records settled it with another build: %+v", p.Modules["a"]) + } + planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own") + if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" { + t.Fatalf("its own build did not settle it: %+v", p.Modules["a"]) + } +} + +func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + var refs []string + was := askABuild + askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { + refs = append(refs, ref) + return was(c, source, path, ref) + } + if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a", + Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil { + t.Fatal(err) + } + if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil { + t.Fatal(err) + } + if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 { + t.Fatalf("asked %v at %v", *asked, refs) + } +} + +func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) { + js, ask := aBuildQueue(t) + open := aMesh(t) + ctx := t.Context() + was := holderLooks + holderLooks = 300 * time.Millisecond + t.Cleanup(func() { holderLooks = was }) + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) + sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { + t.Fatal(err) + } + q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) + if err != nil { + t.Fatal(err) + } + a, _ := q.Find(id) + if a.State != link.AskInFlight || a.On != "" { + t.Fatalf("the taken ask reads %+v", a) + } + // The holder that took it says it started, while the cancel waits. + go func() { + time.Sleep(100 * time.Millisecond) + started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) + _, _ = js.Context().Publish(link.BuildStarted(), started) + }() + if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") { + t.Fatalf("a build that started was cancelled: %v", err) + } + if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled { + t.Error("the withdrawn cancel is still marked") + } + if _, found, _ := open.inventory.BuildByID(ctx, id); found { + t.Error("a withdrawn cancel recorded an outcome") + } +} diff --git a/cmd/mesh-controller/queue_test.go b/cmd/mesh-controller/queue_test.go index 0a4a13fd..d7dc6cb2 100644 --- a/cmd/mesh-controller/queue_test.go +++ b/cmd/mesh-controller/queue_test.go @@ -2,8 +2,6 @@ package main import ( "context" - "encoding/json" - "fmt" "reflect" "strings" "testing" @@ -15,8 +13,6 @@ import ( "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // The build queue, controlled by hand (novox/hq ADR 0219), and the plans that follow it. @@ -52,58 +48,6 @@ func twoTiers(t *testing.T, open *stores) { // A failed plan is retried: its failed module asked again under a new id, the plan building at that // tier, and when that build comes in the plan goes on and asks its next tier. -func TestAFailedPlanIsRetriedAndGoesOnThroughItsLaterTiers(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - failed := inventory.Plan{ID: "plan-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.KilledByHand}}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - - said, err := retryPlan(ctx, open, failed.ID) - if err != nil { - t.Fatal(err) - } - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - a := p.Modules["a"] - if p.State != inventory.PlanBuilding || a.State != "asked" || a.Build == "" || a.Build == "build-1" || a.Why != "" { - t.Fatalf("after retry the plan is %s and a is %+v", p.State, a) - } - if !strings.Contains(said, a.Build) { - t.Errorf("retry does not say the new id: %q", said) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - - // The killed build's own late outcome is not this ask's; the new one's is, and tier 1 follows. - planBuilt(ctx, open, "a", "c0ffee", link.KilledByHand, before, "build-1") - if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.State != inventory.PlanBuilding { - t.Fatalf("the old ask's outcome failed the retried plan: %s %q", p.State, p.Note) - } - asking, _ := link.BuildAskedAt(a.Build) - planBuilt(ctx, open, "a", "c0ffee", "", asking, a.Build) - p, err = open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - if p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" || p.Modules["b"].Build == "" { - t.Fatalf("the retried plan did not go on to tier 1: tier %d, %s, b %+v", p.Tier, p.State, p.Modules["b"]) - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } -} // What retry refuses, and says why. func TestRetryRefusesWhatItCannotResume(t *testing.T) { @@ -152,43 +96,6 @@ func TestRetryRefusesWhatItCannotResume(t *testing.T) { } // `rebuild` of a module a failed plan holds joins that plan; one held by nothing runs alone. -func TestARebuildJoinsThePlanHoldingTheModule(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - failed := inventory.Plan{ID: "plan-join", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", Why: link.CancelledByHand}}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := rebuildCommand(ctx, []string{"a"}); err != nil { - t.Fatal(err) - } - p, err := open.inventory.PlanByID(ctx, failed.ID) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanBuilding || p.Modules["a"].State != "asked" || p.Modules["a"].Build == "build-1" { - t.Fatalf("the rebuild did not join the failed plan: %s %+v", p.State, p.Modules["a"]) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } - // b is in a tier not yet reached: nothing holds it in its current tier, so it is asked alone. - if err := rebuildCommand(ctx, []string{"b"}); err != nil { - t.Fatal(err) - } - if p, _ = open.inventory.PlanByID(ctx, failed.ID); p.Modules["b"] != nil { - t.Fatalf("a module the plan has not reached joined it: %+v", p.Modules["b"]) - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } -} // A failed plan an open plan of its repository supersedes is not joined: the open one holds the module. func TestARebuildJoinsTheOpenPlanBeforeASupersededFailedOne(t *testing.T) { @@ -361,43 +268,6 @@ func TestTheQueueVerbsRunTheirCommands(t *testing.T) { // aBuildQueue is the build seat's queue, worker and cancelled set on a real server, the controller // pointed at it, and a function that asks the seat one build. -func aBuildQueue(t *testing.T) (*broker.JetStream, func(id, repository string) link.BuildRequest) { - t.Helper() - url := testbus.URL(t) - t.Setenv(broker.NATSVar, url) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - seat := broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}, - Emits: []string{"started", "built", "log.*", "paused.*"}} - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") - if err := broker.RaiseSeats(js, []broker.DeclaredSeat{seat}, map[string]broker.Holder{ - link.TheBuildMachine: {Node: "anchor", Module: "build-agent"}}); err != nil { - t.Fatal(err) - } - if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - _ = js.Context().DeleteStream("SEAT_NODE_BUILD_AGENT") - _ = js.Context().DeleteKeyValue(broker.CancelledSetName(link.TheBuildMachine)) - _ = js.Context().PurgeStream(broker.EventsStream) - }) - ask := func(id, repository string) link.BuildRequest { - r := link.BuildRequest{ID: id, Repository: repository, Held: map[string]string{"x/y": "secret-ish"}} - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(link.BuildWork(), body); err != nil { - t.Fatal(err) - } - return r - } - return js, ask -} // deadOne takes the oldest ask from the worker and hands it back as often as the worker allows. func deadOne(t *testing.T, js *broker.JetStream) { @@ -425,357 +295,21 @@ func deadOne(t *testing.T, js *broker.JetStream) { // cancel drops a waiting ask from the bus and records it failed, cancelled by hand — and the plan // that asked for it, matched by the id, fails with it; an ask the plan's records name no module for // is still found. -func TestCancelDeletesTheAskAndFailsThePlanThatAskedIt(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - twoTiers(t, open) - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - asked, _ := link.BuildAskedAt(id) - plan := inventory.Plan{ID: "plan-cancel", Repository: "novox/a", Commit: "c0ffee", Created: asked, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: id}}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 1 || q.Asks[0].State != link.AskWaiting || q.Asks[0].ID != id { - t.Fatalf("the queue reads %+v", q) - } - if text := queueText(q, time.Now()); !strings.Contains(text, "1 waiting, 0 in flight, 0 dead") || - strings.Contains(text, "secret-ish") { - t.Errorf("the queue says:\n%s", text) - } - - if err := cancelCommand(ctx, []string{id}); err != nil { - t.Fatal(err) - } - if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { - t.Fatalf("the ask is still queued: %+v", q.Asks) - } - if cancelled, err := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); err != nil || !cancelled { - t.Errorf("the cancelled set does not hold it: %v %v", cancelled, err) - } - b, found, err := open.inventory.BuildByID(ctx, id) - if err != nil || !found || b.Failed != link.CancelledByHand { - t.Fatalf("the cancel is recorded as %+v (%v %v)", b, found, err) - } - p, err := open.inventory.PlanByID(ctx, plan.ID) - if err != nil { - t.Fatal(err) - } - if p.State != inventory.PlanFailed || p.Modules["a"].State != "failed" || p.Modules["a"].Why != link.CancelledByHand { - t.Fatalf("the plan that asked is %s, a %+v", p.State, p.Modules["a"]) - } - if err := cancelCommand(ctx, []string{id}); err == nil { - t.Error("an ask cancelled already was cancelled again") - } -} // clear cancels every waiting ask and leaves the dead ones unless told, and never one in flight. -func TestClearCancelsTheWaitingAndTheDeadOnlyWhenTold(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - start := time.Now() - dead := link.NewBuildID(start) - ask(dead, "https://forge.example/novox/dead.git") - deadOne(t, js) - var waiting []string - for i := 1; i <= 2; i++ { - id := link.NewBuildID(start.Add(time.Duration(i) * time.Millisecond)) - waiting = append(waiting, id) - ask(id, fmt.Sprintf("https://forge.example/novox/w%d.git", i)) - } - - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Of(link.AskDead)) != 1 || len(q.Of(link.AskWaiting)) != 2 || q.MaxDeliver != 5 { - t.Fatalf("the queue reads %+v", q) - } - said, err := clearQueue(ctx, js, open, link.TheBuildMachine, false) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(said, "2 waiting ask(s) cancelled") || !strings.Contains(said, "1 dead, left") { - t.Errorf("clear said:\n%s", said) - } - q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine) - if len(q.Asks) != 1 || q.Asks[0].ID != dead || q.Asks[0].State != link.AskDead { - t.Fatalf("after clear the queue is %+v", q.Asks) - } - if _, err := clearQueue(ctx, js, open, link.TheBuildMachine, true); err != nil { - t.Fatal(err) - } - if q, _ = link.ReadQueue(ctx, js, link.TheBuildMachine); len(q.Asks) != 0 { - t.Fatalf("clear --dead left %+v", q.Asks) - } - for _, id := range append(waiting, dead) { - if b, found, _ := open.inventory.BuildByID(ctx, id); !found || b.Failed != link.CancelledByHand { - t.Errorf("%s is recorded as %+v", id, b) - } - } -} // An ask in flight is not cancelled: kill ends it where it runs. -func TestCancelRefusesAnAskInFlight(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { - t.Fatal(err) - } - started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - a, _ := q.Find(id) - if a.State != link.AskInFlight || a.On != "ace" { - t.Fatalf("the taken ask reads %+v", a) - } - _, err = cancelAsk(ctx, js, open, link.TheBuildMachine, a) - if err == nil || !strings.Contains(err.Error(), "kill "+id) { - t.Fatalf("an ask in flight was cancelled: %v", err) - } - if _, found, _ := open.inventory.BuildByID(ctx, id); found { - t.Error("a refused cancel recorded an outcome") - } -} // kill finds the machine from the build's start and asks that machine's holder; pause asks the // machine named. Each prints what the holder answered, and a refusal is the command's failure. -func TestKillAndPauseAskTheHolderOnTheMachine(t *testing.T) { - js, _ := aBuildQueue(t) - ctx := t.Context() - asked := map[string]string{} - handlers := map[string]link.ToolHandler{ - "kill": func(_ context.Context, raw json.RawMessage) (any, error) { - var args struct{ ID string } - _ = json.Unmarshal(raw, &args) - asked["kill"] = args.ID - if args.ID != "build-running" { - return nil, fmt.Errorf("ace is not building %s", args.ID) - } - return map[string]any{"said": "killed " + args.ID}, nil - }, - "pause": func(context.Context, json.RawMessage) (any, error) { - asked["pause"] = "ace" - return map[string]any{"said": "ace is paused"}, nil - }, - } - stop, err := link.OverNATS{Conn: js.Conn()}.ServeNodeSeatTools(link.TheBuildMachine, "ace", handlers, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - for _, id := range []string{"build-running", "build-other"} { - started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - if _, err := js.Context().Publish(link.BuildStarted(), started); err != nil { - t.Fatal(err) - } - } - if err := killCommand(ctx, []string{"build-running"}); err != nil { - t.Fatal(err) - } - if asked["kill"] != "build-running" { - t.Fatalf("the holder was asked %v", asked) - } - if err := killCommand(ctx, []string{"build-other"}); err == nil { - t.Error("the holder's refusal was not the command's") - } - if err := killCommand(ctx, []string{"build-never"}); err == nil || !strings.Contains(err.Error(), "cancel build-never") { - t.Errorf("a build nobody started: %v", err) - } - if err := pauseCommand(ctx, "pause", []string{"ace"}); err != nil || asked["pause"] != "ace" { - t.Fatalf("pause: %v %v", err, asked) - } - if err := pauseCommand(ctx, "resume", []string{"g14"}); err == nil { - t.Error("a machine nothing answers on was resumed") - } -} // A plan that stopped at its first machine is retried: the module sent to that machine again, the // send recorded as the first anew, and the plan goes on — unless a newer plan holds the module. -func TestAPlanStoppedAtItsFirstMachineIsRetried(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - var sentTo [][]string - was := sendRollout - sendRollout = func(_ context.Context, _ *stores, names []string) ([]string, error) { - sentTo = append(sentTo, names) - return names, nil - } - t.Cleanup(func() { sendRollout = was }) - - // a records: a person's choice, since the default rolls out (novox/hq ADR 0236). - if err := open.inventory.SetUpgradeOf(ctx, "a", inventory.Upgrade{Why: "test"}); err != nil { - t.Fatal(err) - } - long := time.Now().UTC().Add(-2 * time.Hour) - stopped := inventory.Plan{ID: "plan-rollout", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: long, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a stopped at its first machine in tier 0: laptop refused what it was sent", - // Asked by the plan, as every module a plan holds built is (novox/hq issue 754). - Modules: map[string]*inventory.PlanModule{"a": {State: "built", AskedAt: &long, Build: "build-a", BuiltAt: &long, - Commit: "c0ffee", First: []string{"laptop"}, FirstAt: &long, Why: "laptop refused what it was sent"}}} - if err := open.inventory.SavePlan(ctx, &stopped); err != nil { - t.Fatal(err) - } - - // A newer plan holding a refuses it: sending the older build would put it back. - newer := inventory.Plan{ID: "plan-newer", Repository: "novox/other", Commit: "d00d", Created: long.Add(time.Hour), - State: inventory.PlanDone, Tiers: [][]string{{"a"}}, Modules: map[string]*inventory.PlanModule{}} - if err := open.inventory.SavePlan(ctx, &newer); err != nil { - t.Fatal(err) - } - if _, err := retryPlan(ctx, open, stopped.ID); err == nil || !strings.Contains(err.Error(), "plan-newer") { - t.Fatalf("retried under a newer plan: %v", err) - } - newer.State = inventory.PlanSuperseded - if err := open.inventory.SavePlan(ctx, &newer); err != nil { - t.Fatal(err) - } - - said, err := retryPlan(ctx, open, stopped.ID) - if err != nil { - t.Fatal(err) - } - if len(sentTo) != 1 || !reflect.DeepEqual(sentTo[0], []string{"laptop"}) || !strings.Contains(said, "laptop") { - t.Fatalf("sent %v; said %q", sentTo, said) - } - p, err := open.inventory.PlanByID(ctx, stopped.ID) - if err != nil { - t.Fatal(err) - } - a := p.Modules["a"] - if p.State != inventory.PlanRolling || a.FirstAt == nil || !a.FirstAt.After(long) || a.Why != "" { - t.Fatalf("after retry the plan is %s, a %+v", p.State, a) - } - // And it goes on: a records (its policy sends nothing more), so the next tier is asked. - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, stopped.ID); p.Tier != 1 || p.Modules["b"] == nil || p.Modules["b"].State != "asked" { - t.Fatalf("the retried plan did not go on: tier %d %s %+v", p.Tier, p.State, p.Modules["b"]) - } - if len(*asked) != 1 { - t.Fatalf("asked %v", *asked) - } -} // A plan module asked under an id is settled by that id's outcome alone: a replay or a rebuild beside // the plan, asked later, never answers it (novox/hq ADR 0219). -func TestAPlanIsAnsweredOnlyByTheBuildItAskedFor(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := time.Now().UTC().Add(-time.Minute) - plan := inventory.Plan{ID: "plan-own", Repository: "novox/a", Commit: "c0ffee", Created: asked, - State: inventory.PlanBuilding, Tiers: [][]string{{"a"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "asked", AskedAt: &asked, Build: "build-own"}}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - planBuilt(ctx, open, "a", "0ldc0mm1t", "", time.Now().UTC(), "build-replay") - p, _ := open.inventory.PlanByID(ctx, plan.ID) - if p.Modules["a"].State != "asked" { - t.Fatalf("a replay asked after the plan settled it: %+v", p.Modules["a"]) - } - // From the records too: a later build of the module recorded is not the plan's. - recorded := map[string][]inventory.Build{"a": {{ID: "build-replay", Commit: "0ldc0mm1t", Asked: time.Now(), At: time.Now()}}} - if settleFromRecords(&p, p.Tiers[0], recorded, nil) { - t.Fatalf("the records settled it with another build: %+v", p.Modules["a"]) - } - planBuilt(ctx, open, "a", "c0ffee", "", asked, "build-own") - if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Modules["a"].State != "built" || p.Modules["a"].Commit != "c0ffee" { - t.Fatalf("its own build did not settle it: %+v", p.Modules["a"]) - } -} // rebuild of a build made at a commit asks what the module follows now, never the commit. -func TestARebuildOfACommitAsksWhatTheModuleFollows(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - var refs []string - was := askABuild - askABuild = func(c context.Context, source buildSource, path, ref string) (string, error) { - refs = append(refs, ref) - return was(c, source, path, ref) - } - if err := open.inventory.RecordBuild(ctx, inventory.Build{ID: "build-at-commit", Repository: "novox/a", - Ref: "0123456789abcdef0123456789abcdef01234567", Module: "a", Commit: "0123456789abcdef0123456789abcdef01234567"}); err != nil { - t.Fatal(err) - } - if err := rebuildCommand(ctx, []string{"build-at-commit"}); err != nil { - t.Fatal(err) - } - if len(refs) != 1 || refs[0] != "main" || len(*asked) != 1 { - t.Fatalf("asked %v at %v", *asked, refs) - } -} // An ask the worker counts out that no machine said it started is cancelled only if no start comes // while a holder looks: one that does withdraws the cancel, and kill is what ends it. -func TestCancelOfAnAskNobodySaidIsWithdrawnWhenItStarts(t *testing.T) { - js, ask := aBuildQueue(t) - open := aMesh(t) - ctx := t.Context() - was := holderLooks - holderLooks = 300 * time.Millisecond - t.Cleanup(func() { holderLooks = was }) - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - worker, _ := broker.HolderConsumerFor("", "", broker.DeclaredSeat{Name: link.TheBuildMachine, Accepts: []string{"build"}}) - sub, err := js.Context().PullSubscribe(worker.Filters[0], worker.Name, nats.Bind(worker.Stream, worker.Name), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - if _, err := sub.Fetch(1, nats.MaxWait(3*time.Second)); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(ctx, js, link.TheBuildMachine) - if err != nil { - t.Fatal(err) - } - a, _ := q.Find(id) - if a.State != link.AskInFlight || a.On != "" { - t.Fatalf("the taken ask reads %+v", a) - } - // The holder that took it says it started, while the cancel waits. - go func() { - time.Sleep(100 * time.Millisecond) - started, _ := json.Marshal(link.BuildStart{ID: id, On: "ace", At: time.Now().UTC().Format(time.RFC3339Nano)}) - _, _ = js.Context().Publish(link.BuildStarted(), started) - }() - if _, err := cancelAsk(ctx, js, open, link.TheBuildMachine, a); err == nil || !strings.Contains(err.Error(), "started on ace") { - t.Fatalf("a build that started was cancelled: %v", err) - } - if cancelled, _ := link.IsCancelled(js.Conn(), link.TheBuildMachine, id); cancelled { - t.Error("the withdrawn cancel is still marked") - } - if _, found, _ := open.inventory.BuildByID(ctx, id); found { - t.Error("a withdrawn cancel recorded an outcome") - } -} diff --git a/cmd/mesh-controller/reconnects_integration_test.go b/cmd/mesh-controller/reconnects_integration_test.go new file mode 100644 index 00000000..6cf7e7d3 --- /dev/null +++ b/cmd/mesh-controller/reconnects_integration_test.go @@ -0,0 +1,140 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestTheServingControllerLendsItsOwnConnection(t *testing.T) { + bus := testbus.Start(t) + serving, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer serving.Close() + if err := serving.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + was := servingBus.Load() + servingBus.Store(serving) + defer servingBus.Store(was) + t.Setenv(broker.NATSVar, bus.ClientURL()) + + before, _ := bus.Varz(nil) + lent, err := aBus() + if err != nil { + t.Fatal(err) + } + lent.Close() + if !serving.Conn().IsConnected() { + t.Fatal("closing a lent connection closed the serving controller's") + } + if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil { + t.Fatal(err) + } + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`)) + if err != nil { + t.Fatal(err) + } + if a := answer.(verbAnswer); !a.OK || a.Answer == nil { + t.Fatalf("hand-acts answered %+v", a) + } + _, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`)) + after, _ := bus.Varz(nil) + if opened := after.TotalConnections - before.TotalConnections; opened != 0 { + t.Fatalf("the serving controller opened %d connection(s) of its own", opened) + } +} + +func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) { + bus := testbus.Start(t) + setup, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + if err := setup.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + setup.Close() + asAProcess(t, bus.ClientURL()) + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + // A silence acts, so it is still its own process; it dials, and finds no such condition. + _, _ = handlers["conditions"](context.Background(), + json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`)) + names := closedNames(t, bus, "mesh-controller verb conditions") + found := false + for _, n := range names { + found = found || n == "mesh-controller verb conditions" + } + if !found { + t.Fatalf("the verb's connection was named %q", names) + } +} + +func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) { + bus := testbus.Start(t) + conn, err := nats.Connect(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) { + _ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`)) + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = sub.Unsubscribe() }() + read, err := askClosed(context.Background(), conn, "anchor") + if err != nil { + t.Fatal(err) + } + if len(read.Users) != 1 || read.Users[0].Closed != 2 { + t.Fatalf("%+v", read) + } +} + +func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) { + bus := testbus.Start(t) + conn, err := nats.Connect(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + found, err := reconnectsOn(context.Background(), conn, "anchor") + if err != nil || len(found) != 0 { + t.Fatalf("a bus module without the tool: %v %v", found, err) + } +} + +func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) { + bus := testbus.Start(t) + serving, err := broker.Dial(bus.ClientURL()) + if err != nil { + t.Fatal(err) + } + defer serving.Close() + was := servingBus.Load() + servingBus.Store(serving) + defer servingBus.Store(was) + answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"}) + if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") { + t.Fatalf("answered %+v", answer) + } +} diff --git a/cmd/mesh-controller/reconnects_test.go b/cmd/mesh-controller/reconnects_test.go index 4f43f5e3..3c708898 100644 --- a/cmd/mesh-controller/reconnects_test.go +++ b/cmd/mesh-controller/reconnects_test.go @@ -1,94 +1,17 @@ package main import ( - "context" "encoding/json" "strings" "testing" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/conditions" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // The serving controller's own connection serves what it does for a moment: no connection is opened, // and closing what it was lent leaves the serving one open (novox/hq issue 327). -func TestTheServingControllerLendsItsOwnConnection(t *testing.T) { - bus := testbus.Start(t) - serving, err := broker.Dial(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer serving.Close() - if err := serving.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - was := servingBus.Load() - servingBus.Store(serving) - defer servingBus.Store(was) - t.Setenv(broker.NATSVar, bus.ClientURL()) - - before, _ := bus.Varz(nil) - lent, err := aBus() - if err != nil { - t.Fatal(err) - } - lent.Close() - if !serving.Conn().IsConnected() { - t.Fatal("closing a lent connection closed the serving controller's") - } - if err := onTheBus(func(*nats.Conn) error { return nil }); err != nil { - t.Fatal(err) - } - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - answer, err := handlers["hand-acts"](context.Background(), json.RawMessage(`{}`)) - if err != nil { - t.Fatal(err) - } - if a := answer.(verbAnswer); !a.OK || a.Answer == nil { - t.Fatalf("hand-acts answered %+v", a) - } - _, _ = handlers["queue"](context.Background(), json.RawMessage(`{}`)) - after, _ := bus.Varz(nil) - if opened := after.TotalConnections - before.TotalConnections; opened != 0 { - t.Fatalf("the serving controller opened %d connection(s) of its own", opened) - } -} // A verb that still runs as a process of its own says which in the bus's list of connections. -func TestAVerbsOwnProcessNamesItsConnection(t *testing.T) { - bus := testbus.Start(t) - setup, err := broker.Dial(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - if err := setup.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - setup.Close() - asAProcess(t, bus.ClientURL()) - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - // A silence acts, so it is still its own process; it dials, and finds no such condition. - _, _ = handlers["conditions"](context.Background(), - json.RawMessage(`{"silence":"bus.nothing.here","for":"1h","why":"a test"}`)) - names := closedNames(t, bus, "mesh-controller verb conditions") - found := false - for _, n := range names { - found = found || n == "mesh-controller verb conditions" - } - if !found { - t.Fatalf("the verb's connection was named %q", names) - } -} func TestEachProcessNamesItsConnectionsForWhatItIs(t *testing.T) { for _, c := range []struct{ command, verb, want string }{ @@ -134,56 +57,7 @@ func TestAClientReconnectingInALoopIsSaid(t *testing.T) { } // The bus's module is asked on its machine, over the bus. -func TestTheBusModuleIsAskedWhoClosedConnections(t *testing.T) { - bus := testbus.Start(t) - conn, err := nats.Connect(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - sub, err := conn.Subscribe(link.ModuleToolOn("nats", "nats_closed_connections", "anchor"), func(m *nats.Msg) { - _ = m.Respond([]byte(`{"result":{"hours":1,"reaches":true,"users":[{"user":"controller","closed":2,"dropped":0}]}}`)) - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = sub.Unsubscribe() }() - read, err := askClosed(context.Background(), conn, "anchor") - if err != nil { - t.Fatal(err) - } - if len(read.Users) != 1 || read.Users[0].Closed != 2 { - t.Fatalf("%+v", read) - } -} // A bus module older than the tool answers nothing to the question: the probe passes over it quietly. -func TestAnOlderBusModuleIsPassedOverQuietly(t *testing.T) { - bus := testbus.Start(t) - conn, err := nats.Connect(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - found, err := reconnectsOn(context.Background(), conn, "anchor") - if err != nil || len(found) != 0 { - t.Fatalf("a bus module without the tool: %v %v", found, err) - } -} // A verb answered in the serving controller says its flag errors in its answer, not in the controller's log. -func TestAFlagErrorIsSaidInTheAnswer(t *testing.T) { - bus := testbus.Start(t) - serving, err := broker.Dial(bus.ClientURL()) - if err != nil { - t.Fatal(err) - } - defer serving.Close() - was := servingBus.Load() - servingBus.Store(serving) - defer servingBus.Store(was) - answer, read := readHere(context.Background(), []string{"hand-acts", "--bogus"}) - if !read || answer.OK || !strings.Contains(answer.Output, "flag provided but not defined") { - t.Fatalf("answered %+v", answer) - } -} diff --git a/cmd/mesh-controller/recorded_kept_integration_test.go b/cmd/mesh-controller/recorded_kept_integration_test.go new file mode 100644 index 00000000..50b76e46 --- /dev/null +++ b/cmd/mesh-controller/recorded_kept_integration_test.go @@ -0,0 +1,144 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + start := time.Now().Add(-time.Hour) + pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) + mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) + for _, b := range []link.BuildResult{ + aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, + map[string][2]string{"server": {pgImage, ""}}), + aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second), + map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}), + } { + if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { + t.Fatal(err) + } + } + for _, m := range []string{"postgres", "mailu"} { + if _, err := inv.Assign(ctx, "anchor", m); err != nil { + t.Fatal(err) + } + } + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", + map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil { + t.Fatal(err) + } + // The merge: both adopt a health check; the images are the same. + for _, b := range []link.BuildResult{ + aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), + map[string][2]string{"server": {pgImage, "runtime"}}), + aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second), + map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}), + } { + if _, _, err := takeIn(ctx, inv, b); err != nil { + t.Fatal(err) + } + } + + healthOf := func(plan catalogue.Resolution, module, id string) any { + for _, m := range plan.Modules { + for _, r := range m.Resources { + if m.Module == module && r["id"] == id { + return r["health"] + } + } + } + return nil + } + + // A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves. + kept := keepingRecorded(ctx) + plan, settings, err := planFor(kept, open, "anchor") + if err != nil { + t.Fatal(err) + } + if healthOf(plan, "postgres", "server") != nil { + t.Fatal("a send that is not a person's push composed the recorded module's new build") + } + if healthOf(plan, "mailu", "smtp") == nil { + t.Fatal("the module that rolls out was not composed at its new build") + } + if imageOf(t, plan, "postgres", "server") != pgImage { + t.Fatal("the recorded module's container lost its image") + } + gens, err := generators(kept, open) + if err != nil { + t.Fatal(err) + } + declared, err := declarationWith(kept, open, "anchor", plan, settings, gens, Allocating) + if err != nil { + t.Fatal(err) + } + if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" { + t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds) + } + + // A person's push: the recorded module's new build. + plan, _, err = planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if healthOf(plan, "postgres", "server") == nil { + t.Fatal("a person's push did not compose the recorded module's new build") + } + + // The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay. + plan, _, err = planFor(keepingRecorded(ctx, "postgres"), open, "anchor") + if err != nil { + t.Fatal(err) + } + if healthOf(plan, "postgres", "server") == nil { + t.Fatal("the module a send is let move was kept") + } + + // What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves. + if under, err := sendKeeps(ctx, inv); err != nil { + t.Fatal(err) + } else if skip, on := keptExcept(under); !on || len(skip) != 0 { + t.Fatalf("an ordinary send keeps %v %v", on, skip) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2", + Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog", + Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { + t.Fatal(err) + } + if under, err := sendKeeps(withBusStep(ctx), inv); err != nil { + t.Fatal(err) + } else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 { + t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip) + } + + // And a recorded module whose kept build the records no longer hold refuses the send, said. + if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", + map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil { + t.Fatal(err) + } + if _, _, err := planFor(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { + t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err) + } + + // And the gated send for mail says what it recreates: both containers, no new image, at once. + moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}} + sayRecreations(ctx, open, moves) + if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") { + t.Fatalf("the send says %q of mail's containers", moves[0].Recreates) + } + if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") { + t.Fatalf("the plan's note says %q", said) + } +} diff --git a/cmd/mesh-controller/recorded_kept_test.go b/cmd/mesh-controller/recorded_kept_test.go index 07bce8bb..38d626b4 100644 --- a/cmd/mesh-controller/recorded_kept_test.go +++ b/cmd/mesh-controller/recorded_kept_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -58,136 +57,6 @@ func imageOf(t *testing.T, plan catalogue.Resolution, module, id string) string // database's new build to the control node and recreated it, unjudged, with nobody's word. A send that // is not a person's push now composes the database at the build the machine runs and records that it // still carries it; a person's push composes the new one; the bus step moves the bus alone. -func TestARecordedBuildIsCarriedOnlyByAPersonsPush(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - pgImage := "registry.invalid:5000/postgres/server@sha256:" + strings.Repeat("a", 64) - mailImage := "registry.invalid:5000/mailu/smtp@sha256:" + strings.Repeat("c", 64) - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {pgImage, ""}}), - aContainerBuild(t, "mailu", "c1111111", "", start.Add(time.Second), - map[string][2]string{"smtp": {mailImage, ""}, "imap": {mailImage, ""}}), - } { - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, b)); err != nil { - t.Fatal(err) - } - } - for _, m := range []string{"postgres", "mailu"} { - if _, err := inv.Assign(ctx, "anchor", m); err != nil { - t.Fatal(err) - } - } - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c1111111", "mailu": "c1111111"}); err != nil { - t.Fatal(err) - } - // The merge: both adopt a health check; the images are the same. - for _, b := range []link.BuildResult{ - aContainerBuild(t, "postgres", "c2222222", catalogue.PolicyRecord, start.Add(time.Minute), - map[string][2]string{"server": {pgImage, "runtime"}}), - aContainerBuild(t, "mailu", "c2222222", "", start.Add(time.Minute+time.Second), - map[string][2]string{"smtp": {mailImage, "runtime"}, "imap": {mailImage, "runtime"}}), - } { - if _, _, err := takeIn(ctx, inv, b); err != nil { - t.Fatal(err) - } - } - - healthOf := func(plan catalogue.Resolution, module, id string) any { - for _, m := range plan.Modules { - for _, r := range m.Resources { - if m.Module == module && r["id"] == id { - return r["health"] - } - } - } - return nil - } - - // A plan's, a release plan's, a healer's send: the database is kept as it runs, mail moves. - kept := keepingRecorded(ctx) - plan, settings, err := planFor(kept, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") != nil { - t.Fatal("a send that is not a person's push composed the recorded module's new build") - } - if healthOf(plan, "mailu", "smtp") == nil { - t.Fatal("the module that rolls out was not composed at its new build") - } - if imageOf(t, plan, "postgres", "server") != pgImage { - t.Fatal("the recorded module's container lost its image") - } - gens, err := generators(kept, open) - if err != nil { - t.Fatal(err) - } - declared, err := declarationWith(kept, open, "anchor", plan, settings, gens, Allocating) - if err != nil { - t.Fatal(err) - } - if declared.Builds["postgres"] != "c1111111" || declared.Builds["mailu"] != "c2222222" { - t.Fatalf("the send records it carries %v; want postgres still at c1111111 and mailu at c2222222", declared.Builds) - } - - // A person's push: the recorded module's new build. - plan, _, err = planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("a person's push did not compose the recorded module's new build") - } - - // The bus step moves the bus alone: a recorded module it is told it may move moves, the others stay. - plan, _, err = planFor(keepingRecorded(ctx, "postgres"), open, "anchor") - if err != nil { - t.Fatal(err) - } - if healthOf(plan, "postgres", "server") == nil { - t.Fatal("the module a send is let move was kept") - } - - // What a send composes under (sendToEach): every recorded module kept; on the bus step, the bus moves. - if under, err := sendKeeps(ctx, inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || len(skip) != 0 { - t.Fatalf("an ordinary send keeps %v %v", on, skip) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "nats", Version: "2", - Provides: []catalogue.Offer{{Name: "mesh-bus"}}}, inventory.Source{Repository: "novox/mesh-catalog", - Seat: "git", Path: "modules/nats", BuiltFrom: "n2", Head: "n2"}); err != nil { - t.Fatal(err) - } - if under, err := sendKeeps(withBusStep(ctx), inv); err != nil { - t.Fatal(err) - } else if skip, on := keptExcept(under); !on || !skip["nats"] || len(skip) != 1 { - t.Fatalf("the bus step keeps %v %v; want everything recorded but the bus", on, skip) - } - - // And a recorded module whose kept build the records no longer hold refuses the send, said. - if err := inv.RecordSent(ctx, nodeID(t, open, "anchor"), "d-anchor", - map[string]string{"postgres": "c0000000", "mailu": "c2222222"}); err != nil { - t.Fatal(err) - } - if _, _, err := planFor(kept, open, "anchor"); err == nil || !strings.Contains(err.Error(), "push anchor") { - t.Fatalf("a recorded build that cannot be kept did not refuse the send with its remedy: %v", err) - } - - // And the gated send for mail says what it recreates: both containers, no new image, at once. - moves := []inventory.CarriedMove{{Module: "mailu", Node: "anchor", From: "c1111111", To: "c2222222"}} - sayRecreations(ctx, open, moves) - if !strings.Contains(moves[0].Recreates, "recreates 2 of mailu's 2") || !strings.Contains(moves[0].Recreates, "no new image") { - t.Fatalf("the send says %q of mail's containers", moves[0].Recreates) - } - if said := recreationsSaid(moves); !strings.HasPrefix(said, "on anchor recreates") { - t.Fatalf("the plan's note says %q", said) - } -} // The send says what it recreates (ADR 0245): mail's health checks adopted recreate both its // containers, with no new image, every one at once. diff --git a/cmd/mesh-controller/recorded_moves_test.go b/cmd/mesh-controller/recorded_moves_test.go index cc11869a..a464c940 100644 --- a/cmd/mesh-controller/recorded_moves_test.go +++ b/cmd/mesh-controller/recorded_moves_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/registry_verbs_integration_test.go b/cmd/mesh-controller/registry_verbs_integration_test.go new file mode 100644 index 00000000..1de9c6be --- /dev/null +++ b/cmd/mesh-controller/registry_verbs_integration_test.go @@ -0,0 +1,75 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestADryRunCollectAsksOnlyAndChangesNothing(t *testing.T) { + f := &fakeStore{} + store := f.serve(t) + kept := []string{archiveRef("web", "tools", 1)} + eligible := []string{ref("web", "app", 2), archiveRef("web", "tools", 3)} + a := runCollect(context.Background(), nil, store, eligible, kept, false, + sweepBounds{most: 10, budget: 5 * time.Second}) + if !a.DryRun || a.Eligible != 2 || !slices.Equal(a.WouldLetGo, eligible) || len(a.LetGo) != 0 { + t.Errorf("a dry run answered %+v", a) + } + if a.KeptArchives != 1 || a.Held != 0 || a.Unheld != 1 { + t.Errorf("the kept archive is unheld in the fake store, and was said as %+v", a) + } + for _, r := range f.seen() { + if !strings.HasPrefix(r, "HEAD ") { + t.Errorf("a dry run asked the store %s", r) + } + } +} + +func TestARealCollectHoldsEveryKeptArchiveBeforeItLetsAnythingGo(t *testing.T) { + inv := inventory.ForTest(t) + f := &fakeStore{} + store := f.serve(t) + kept := []string{archiveRef("web", "tools", 1)} + eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} + madeBy(t, inv, eligible) + a := runCollect(t.Context(), inv, store, eligible, kept, true, sweepBounds{most: 2, budget: 5 * time.Second}) + if a.DryRun || a.HoldersWritten != 1 || a.Held != 1 { + t.Errorf("the kept archive was not held first: %+v", a) + } + if !slices.Equal(a.LetGo, eligible[:2]) || a.Left != 1 || a.Stopped == "" { + t.Errorf("bounded at two: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) + } + requests := f.seen() + firstPut := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, "/manifests/") }) + firstDelete := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") }) + if firstPut < 0 || firstDelete < 0 || firstPut > firstDelete { + t.Errorf("the holder was not put before the first delete: %v", requests) + } + // What was let go is recorded collected: the sweep no longer offers it. + left, err := inv.ToCollect(t.Context()) + if err != nil { + t.Fatal(err) + } + if !slices.Equal(left, eligible[2:]) { + t.Errorf("after letting go of two, the records offer %v", left) + } +} + +func TestARealCollectStopsAtTheStoresFirstRefusal(t *testing.T) { + inv := inventory.ForTest(t) + f := &fakeStore{refuse: fmt.Sprintf("%064x", 3)} + store := f.serve(t) + eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} + a := runCollect(t.Context(), inv, store, eligible, nil, true, sweepBounds{most: 10, budget: 5 * time.Second}) + if !slices.Equal(a.LetGo, eligible[:1]) || a.Left != 2 || !strings.Contains(a.Stopped, "kept") { + t.Errorf("after a refusal: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) + } +} diff --git a/cmd/mesh-controller/registry_verbs_test.go b/cmd/mesh-controller/registry_verbs_test.go index da61e37c..3c0ea218 100644 --- a/cmd/mesh-controller/registry_verbs_test.go +++ b/cmd/mesh-controller/registry_verbs_test.go @@ -160,26 +160,6 @@ func (f *fakeStore) seen() []string { return slices.Clone(f.requests) } -func TestADryRunCollectAsksOnlyAndChangesNothing(t *testing.T) { - f := &fakeStore{} - store := f.serve(t) - kept := []string{archiveRef("web", "tools", 1)} - eligible := []string{ref("web", "app", 2), archiveRef("web", "tools", 3)} - a := runCollect(context.Background(), nil, store, eligible, kept, false, - sweepBounds{most: 10, budget: 5 * time.Second}) - if !a.DryRun || a.Eligible != 2 || !slices.Equal(a.WouldLetGo, eligible) || len(a.LetGo) != 0 { - t.Errorf("a dry run answered %+v", a) - } - if a.KeptArchives != 1 || a.Held != 0 || a.Unheld != 1 { - t.Errorf("the kept archive is unheld in the fake store, and was said as %+v", a) - } - for _, r := range f.seen() { - if !strings.HasPrefix(r, "HEAD ") { - t.Errorf("a dry run asked the store %s", r) - } - } -} - func TestCollectSaysWhatItDoesNotListAndWithNoStoreAsksNothing(t *testing.T) { var many []string for i := range mostListed + 5 { @@ -192,36 +172,6 @@ func TestCollectSaysWhatItDoesNotListAndWithNoStoreAsksNothing(t *testing.T) { } } -func TestARealCollectHoldsEveryKeptArchiveBeforeItLetsAnythingGo(t *testing.T) { - inv := inventory.ForTest(t) - f := &fakeStore{} - store := f.serve(t) - kept := []string{archiveRef("web", "tools", 1)} - eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} - madeBy(t, inv, eligible) - a := runCollect(t.Context(), inv, store, eligible, kept, true, sweepBounds{most: 2, budget: 5 * time.Second}) - if a.DryRun || a.HoldersWritten != 1 || a.Held != 1 { - t.Errorf("the kept archive was not held first: %+v", a) - } - if !slices.Equal(a.LetGo, eligible[:2]) || a.Left != 1 || a.Stopped == "" { - t.Errorf("bounded at two: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) - } - requests := f.seen() - firstPut := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "PUT ") && strings.Contains(r, "/manifests/") }) - firstDelete := slices.IndexFunc(requests, func(r string) bool { return strings.HasPrefix(r, "DELETE ") }) - if firstPut < 0 || firstDelete < 0 || firstPut > firstDelete { - t.Errorf("the holder was not put before the first delete: %v", requests) - } - // What was let go is recorded collected: the sweep no longer offers it. - left, err := inv.ToCollect(t.Context()) - if err != nil { - t.Fatal(err) - } - if !slices.Equal(left, eligible[2:]) { - t.Errorf("after letting go of two, the records offer %v", left) - } -} - // madeBy records one successful build, of a module the mesh does not hold, that made these images: // eligible, every one. func madeBy(t *testing.T, inv *inventory.Inventory, references []string) { @@ -236,17 +186,6 @@ func madeBy(t *testing.T, inv *inventory.Inventory, references []string) { } } -func TestARealCollectStopsAtTheStoresFirstRefusal(t *testing.T) { - inv := inventory.ForTest(t) - f := &fakeStore{refuse: fmt.Sprintf("%064x", 3)} - store := f.serve(t) - eligible := []string{ref("web", "app", 2), ref("web", "app", 3), ref("web", "app", 4)} - a := runCollect(t.Context(), inv, store, eligible, nil, true, sweepBounds{most: 10, budget: 5 * time.Second}) - if !slices.Equal(a.LetGo, eligible[:1]) || a.Left != 2 || !strings.Contains(a.Stopped, "kept") { - t.Errorf("after a refusal: let go %v, left %d, stopped %q", a.LetGo, a.Left, a.Stopped) - } -} - func TestImagesAreEveryContainerImageOfTheDeclarationAndOfWhatWasSent(t *testing.T) { body := []byte(`{"declaration":1,"resources":[ {"id":"web.server","type":"container","image":"store.invalid/web/server@sha256:aa"}, diff --git a/cmd/mesh-controller/rehearse_test.go b/cmd/mesh-controller/rehearse_test.go index df738810..5e934395 100644 --- a/cmd/mesh-controller/rehearse_test.go +++ b/cmd/mesh-controller/rehearse_test.go @@ -2,11 +2,12 @@ package main import ( "context" - "github.com/novox/mesh-controller/internal/link" "strings" "testing" "time" + "github.com/novox/mesh-controller/internal/link" + "git.novox.be/novox/mesh-sdk/go/asks" ) diff --git a/cmd/mesh-controller/reissue_verified_sender_integration_test.go b/cmd/mesh-controller/reissue_verified_sender_integration_test.go new file mode 100644 index 00000000..3dcc45d4 --- /dev/null +++ b/cmd/mesh-controller/reissue_verified_sender_integration_test.go @@ -0,0 +1,57 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgementPasses(t *testing.T) { + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()} + + failed, router := verifiedSenderRecords(nil) + body, err := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil))) + if err != nil { + t.Fatal(err) + } + if err := bus.PublishMembership(t.Context(), "novox", "messenger", body); err != nil { + t.Fatal(err) + } + issued := func() []byte { + m, err := js.Context().GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject("novox", "messenger")) + if err != nil { + t.Fatal(err) + } + return m.Data + } + if hasVerified(t, issued()) { + t.Fatal("issued with a failed judgement, and the verified sender is kept: the test proves nothing") + } + if n := reissueVerifiedSendersOn(t.Context(), bus, failed, broker.PlacementsOf(failed, nil)); n != 0 { + t.Fatalf("issued %d memberships while the judgement still fails", n) + } + passing, _ := verifiedSenderRecords(map[string]bool{"novox": true}) + where := broker.PlacementsOf(passing, nil) + if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 1 { + t.Fatalf("issued %d memberships after the judgement passed, want 1", n) + } + if !hasVerified(t, issued()) { + t.Fatal("the membership on the bus lacks the verified sender after the judgement passed") + } + if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 0 { + t.Fatalf("a second pass issued %d memberships", n) + } +} diff --git a/cmd/mesh-controller/reissue_verified_sender_test.go b/cmd/mesh-controller/reissue_verified_sender_test.go index 0f00fdaf..57503c88 100644 --- a/cmd/mesh-controller/reissue_verified_sender_test.go +++ b/cmd/mesh-controller/reissue_verified_sender_test.go @@ -9,8 +9,6 @@ import ( "github.com/nats-io/nats.go" "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) func verifiedSenderRecords(free map[string]bool) (broker.Records, broker.Declared) { @@ -124,50 +122,6 @@ func TestAMembershipNeverIssuedIsLeftToItsOwnSend(t *testing.T) { // **On a real bus** (novox/hq issue 930): a router's membership issued without the verified sender because the // judgement failed is, once the judgement passes, published again to the AssignmentsStream with it in its seat // traffic's kinds, and a second pass publishes nothing. -func TestNatsAMembershipIssuedWithoutItsVerifiedSenderIsIssuedAgainWhenTheJudgementPasses(t *testing.T) { - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - bus := link.OverNATS{Conn: js.Conn(), JS: js.Context()} - - failed, router := verifiedSenderRecords(nil) - body, err := json.Marshal(broker.MembershipFor("novox", router, broker.PlacementsOf(failed, nil))) - if err != nil { - t.Fatal(err) - } - if err := bus.PublishMembership(t.Context(), "novox", "messenger", body); err != nil { - t.Fatal(err) - } - issued := func() []byte { - m, err := js.Context().GetLastMsg(broker.AssignmentsStream, broker.MembershipSubject("novox", "messenger")) - if err != nil { - t.Fatal(err) - } - return m.Data - } - if hasVerified(t, issued()) { - t.Fatal("issued with a failed judgement, and the verified sender is kept: the test proves nothing") - } - if n := reissueVerifiedSendersOn(t.Context(), bus, failed, broker.PlacementsOf(failed, nil)); n != 0 { - t.Fatalf("issued %d memberships while the judgement still fails", n) - } - passing, _ := verifiedSenderRecords(map[string]bool{"novox": true}) - where := broker.PlacementsOf(passing, nil) - if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 1 { - t.Fatalf("issued %d memberships after the judgement passed, want 1", n) - } - if !hasVerified(t, issued()) { - t.Fatal("the membership on the bus lacks the verified sender after the judgement passed") - } - if n := reissueVerifiedSendersOn(t.Context(), bus, passing, where); n != 0 { - t.Fatalf("a second pass issued %d memberships", n) - } -} // A push issuing a newer membership between the self-check's read and its write is not overwritten (issue 930). func TestANewerMembershipIssuedMeanwhileIsNotOverwritten(t *testing.T) { diff --git a/cmd/mesh-controller/related_integration_test.go b/cmd/mesh-controller/related_integration_test.go new file mode 100644 index 00000000..b29fb967 --- /dev/null +++ b/cmd/mesh-controller/related_integration_test.go @@ -0,0 +1,52 @@ +//go:build integration + +package main + +import ( + "testing" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) { + edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}} + walk := func(id, repo string, own bool, modules ...string) openWalk { + return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own} + } + one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget") + cand := func(repo, head string, moves ...string) footprint { + return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges) + } + cases := []struct { + name string + fp footprint + own bool + open []openWalk + behind string + why string + }{ + {"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""}, + {"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""}, + {"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"}, + {"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two}, + "plan-2", "both move nzbget"}, + {"a graph edge", cand("novox/x", "fix/a", "nzbget"), + false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3", + "nzbget depends on base-image (stands-on)"}, + {"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1", + "the modules a merge of novox/x moves are not known until its batch is cut"}, + {"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1", + "it is on the controller's own path"}, + {"an open walk on the own path", cand("novox/x", "fix/a", "app"), false, + []openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4", + "plan-4 is on the controller's own path"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + behind, why := relatedOpen(c.fp, c.own, c.open, edges) + if behind != c.behind || why != c.why { + t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why) + } + }) + } +} diff --git a/cmd/mesh-controller/related_test.go b/cmd/mesh-controller/related_test.go index 90a062e8..40f80802 100644 --- a/cmd/mesh-controller/related_test.go +++ b/cmd/mesh-controller/related_test.go @@ -116,48 +116,6 @@ func TestABatchWhoseMovesAreNotKnownIsRelatedToEverything(t *testing.T) { // held behind none (issue 973, ADR 0407 decision 4); one that shares a repository, a module, a graph edge or a // group head is held behind the first walk it shares it with, with the reason; one whose moves are not known // is held behind the first; a walk or change on the controller's own path is related to every other. -func TestRelatedOpenNamesTheFirstOpenWalkAChangeIsRelatedTo(t *testing.T) { - edges := []inventory.Edge{{From: "nzbget", To: "base-image", Kind: inventory.EdgeStandsOn}} - walk := func(id, repo string, own bool, modules ...string) openWalk { - return openWalk{id: id, fp: footprintOfWalk(walkOf(repo, modules...), nil), own: own} - } - one, two := walk("plan-1", "novox/one", false, "notes"), walk("plan-2", "novox/two", false, "nzbget") - cand := func(repo, head string, moves ...string) footprint { - return footprintOfBatch([]inventory.BatchedMerge{heardMerge(repo, head, moves...)}, edges) - } - cases := []struct { - name string - fp footprint - own bool - open []openWalk - behind string - why string - }{ - {"nothing open", cand("novox/x", "fix/a", "app"), false, nil, "", ""}, - {"unrelated to every open walk", cand("novox/x", "fix/a", "app"), false, []openWalk{one, two}, "", ""}, - {"a repository", cand("novox/ONE", "fix/a", "app"), false, []openWalk{two, one}, "plan-1", "both change novox/one"}, - {"a module through its dependents", cand("novox/x", "fix/a", "base-image"), false, []openWalk{one, two}, - "plan-2", "both move nzbget"}, - {"a graph edge", cand("novox/x", "fix/a", "nzbget"), - false, []openWalk{one, walk("plan-3", "novox/y", false, "base-image")}, "plan-3", - "nzbget depends on base-image (stands-on)"}, - {"moves not known", cand("novox/x", "fix/a"), false, []openWalk{one, two}, "plan-1", - "the modules a merge of novox/x moves are not known until its batch is cut"}, - {"the candidate on the own path", cand("novox/x", "fix/a", "app"), true, []openWalk{one}, "plan-1", - "it is on the controller's own path"}, - {"an open walk on the own path", cand("novox/x", "fix/a", "app"), false, - []openWalk{one, walk("plan-4", "novox/mesh-controller", true, "mesh-controller")}, "plan-4", - "plan-4 is on the controller's own path"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - behind, why := relatedOpen(c.fp, c.own, c.open, edges) - if behind != c.behind || why != c.why { - t.Errorf("relatedOpen = %q, %q; want %q, %q", behind, why, c.behind, c.why) - } - }) - } -} // A walk is on the controller's own path when its batch says so or it plans a module of that path (issue 973). func TestAWalkOnTheControllersPathIsKnownByItsFlagOrItsModules(t *testing.T) { diff --git a/cmd/mesh-controller/release_branch_test.go b/cmd/mesh-controller/release_branch_test.go deleted file mode 100644 index 298b1588..00000000 --- a/cmd/mesh-controller/release_branch_test.go +++ /dev/null @@ -1,273 +0,0 @@ -package main - -import ( - "context" - "errors" - "reflect" - "strings" - "testing" - - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" -) - -// **A pull request into a release branch is judged by that branch** (novox/hq ADR 0358, issue 704): a base that is -// not its repository's trunk, where mesh-controller has a branch of that name, is a release branch; the check then -// reads every repository beside it at its branch of that name where it has one, else at its trunk. Any other base is -// checked exactly as before. - -const aRelease = "feat/57-config-and-provisioning" - -// theCore is the mesh's core as a check reads it beside: the controller, the node-engine, the tool runner and the -// catalogue's bus, on the git seat. -func theCore() []inventory.Entry { - onSeat := func(module, repository string) inventory.Entry { - e := fromRepo(module, repository, "") - e.Source.Seat = "git" - return e - } - nats := onSeat("nats", "novox/mesh-catalog") - nats.Source.Path = "modules/nats" - return []inventory.Entry{onSeat("mesh-controller", "novox/mesh-controller"), onSeat("mesh-host", "novox/mesh-host"), - onSeat("node-tools", "novox/mesh-tools"), nats, onSeat("gitea", "novox/mesh-catalog")} -} - -// aForgeWithBranches stands in for the forge: which branches each repository has, and how often it was asked. -func aForgeWithBranches(t *testing.T, branches map[string][]string, failing error) *int { - t.Helper() - asked := 0 - was := forgeHasBranch - t.Cleanup(func() { forgeHasBranch = was }) - forgeHasBranch = func(_ context.Context, owner, repo, branch string) (bool, error) { - asked++ - if failing != nil { - return false, failing - } - for _, b := range branches[owner+"/"+repo] { - if b == branch { - return true, nil - } - } - return false, nil - } - return &asked -} - -func TestABaseIsAReleaseBranchWhenTheControllerHasABranchOfThatName(t *testing.T) { - entries := theCore() - scopeOf := func(p link.PullUpdated) checkScope { return pullScope(p, entries, nil, nil) } - pull := func(repo, base string) link.PullUpdated { - return link.PullUpdated{Owner: "novox", Repo: repo, Number: 286, Base: base, Commit: "abc", - Paths: []string{"modules/gitea/index.ts"}} - } - - asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil) - for _, c := range []struct { - what string - p link.PullUpdated - release string - asks int - }{ - {"a pull request into the trunk asks nothing and is no release", pull("mesh-catalog", "main"), "", 0}, - {"a catalogue pull request into the release branch", pull("mesh-catalog", aRelease), aRelease, 1}, - {"a controller pull request into the release branch", pull("mesh-controller", aRelease), aRelease, 1}, - {"a base mesh-controller has no branch of is checked as today", pull("mesh-catalog", "feat/other"), "", 1}, - {"no base: as today", pull("mesh-catalog", ""), "", 0}, - } { - *asked = 0 - got, err := releaseBranchOf(t.Context(), c.p, scopeOf(c.p), entries) - if err != nil { - t.Fatalf("%s: %v", c.what, err) - } - if got != c.release || *asked != c.asks { - t.Errorf("%s: release %q after %d question(s) to the forge, not %q after %d", c.what, got, *asked, - c.release, c.asks) - } - } - - // The branch a module built from the repository follows is its trunk: never asked about. - photos := fromRepo("photos", "novox/photos", "") - photos.Source.Ref = "nox-mesh" - withPhotos := append(theCore(), photos) - p := link.PullUpdated{Owner: "novox", Repo: "photos", Base: "nox-mesh", Commit: "abc", Paths: []string{"x.js"}} - *asked = 0 - if got, err := releaseBranchOf(t.Context(), p, pullScope(p, withPhotos, nil, nil), withPhotos); err != nil || - got != "" || *asked != 0 { - t.Fatalf("a module's own trunk read as release %q (%v) after %d question(s)", got, err, *asked) - } - - // A forge that cannot answer is an error, never "no release branch". - aForgeWithBranches(t, nil, errors.New("nothing serves gitea.gitea_api")) - q := pull("mesh-catalog", aRelease) - if _, err := releaseBranchOf(t.Context(), q, scopeOf(q), entries); err == nil || - !strings.Contains(err.Error(), "cannot be told") { - t.Fatalf("an unanswered forge was read as an answer: %v", err) - } -} - -func TestBesideAReleaseBranchCheckEachRepositoryIsReadAtThatBranchElseItsTrunk(t *testing.T) { - entries := theCore() - clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil } - running := func(module string) string { return "running-" + module } - - beside, err := besideOf(entries, running, clone, aRelease) - if err != nil { - t.Fatal(err) - } - at := func(repo string, ref, orElse string) link.CheckedOut { - return link.CheckedOut{Repository: "http://git/novox/" + repo + ".git", Ref: ref, Else: orElse} - } - want := map[string]link.CheckedOut{ - // The judge: mesh-controller's release branch, which it has. - "mesh-controller": at("mesh-controller", aRelease, ""), - "mesh-controller-main": at("mesh-controller", "main", ""), - // A controller pull request into the release branch reads mesh-host at that branch where it has one. - "mesh-host": at("mesh-host", aRelease, "main"), - "mesh-tools": at("mesh-tools", aRelease, "main"), - "mesh-catalog": at("mesh-catalog", aRelease, "main"), - // mesh-lab at its trunk, always: its replays run with the container runtime's socket. - "mesh-lab": at("mesh-lab", "main", ""), - "mesh-sdk": at("mesh-sdk", aRelease, "main"), - } - if !reflect.DeepEqual(beside, want) { - t.Fatalf("beside a release-branch check:\n got %v\nwant %v", beside, want) - } - - // Any other base: exactly as before — what the mesh runs, the catalogue's and the lab's main. - beside, err = besideOf(entries, running, clone, "") - if err != nil { - t.Fatal(err) - } - want = map[string]link.CheckedOut{ - "mesh-controller": at("mesh-controller", "running-mesh-controller", ""), - "mesh-controller-main": at("mesh-controller", "main", ""), - "mesh-host": at("mesh-host", "running-mesh-host", ""), - "mesh-tools": at("mesh-tools", "running-node-tools", ""), - "mesh-catalog": at("mesh-catalog", "main", ""), - "mesh-lab": at("mesh-lab", "main", ""), - "mesh-sdk": at("mesh-sdk", sdkPinned(), ""), - } - if !reflect.DeepEqual(beside, want) { - t.Fatalf("beside a trunk check:\n got %v\nwant %v", beside, want) - } -} - -func TestAReleaseBranchVerdictNamesTheBranchTheJudgeAndWhatItRead(t *testing.T) { - asked := &link.CheckRequest{Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: aRelease, Release: aRelease} - c := checkedOf(link.BuildResult{ID: "b", Ref: "aaaaaaaaaaaa0000", Checked: asked, Check: &link.CheckOutcome{ - Verdict: "pass", Head: "aaaaaaaaaaaa", Trunk: "111111111111", Gate: &link.CheckLayer{Verdict: "pass"}, - RepoCheck: &link.CheckLayer{Verdict: "pass"}, Release: aRelease, JudgedBy: "fcdecb436117", - Beside: map[string]string{"mesh-host": "222222222222", "mesh-lab": "333333333333"}, - BesideAt: map[string]string{"mesh-host": aRelease, "mesh-lab": "main"}, - Validator: "mesh-host's validator of " + aRelease + " at 222222222222"}}) - if c.Release != aRelease || c.JudgedBy != "fcdecb436117" { - t.Fatalf("the verdict names release %q judged by %q", c.Release, c.JudgedBy) - } - if c.BesideAt["mesh-host"] != aRelease || c.BesideAt["mesh-lab"] != "main" || c.Beside["mesh-host"] != "222222222222" || - c.Validator != "mesh-host's validator of "+aRelease+" at 222222222222" { - t.Fatalf("the verdict names what it read beside it as %v at %v", c.Beside, c.BesideAt) - } -} - -// **Clause 5** (ADR 0358 §5): a release branch's own pull request into its trunk is no release-branch check: the forge -// is asked nothing, the request names no release branch, and the repositories beside it are read as for any trunk -// pull request. Its delivery group's composed check is asked as for any other (askGroupCheck, unchanged). -func TestAReleaseBranchsPullRequestIntoItsTrunkIsCheckedAsAnyTrunkPullRequest(t *testing.T) { - entries := theCore() - asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease}}, nil) - for _, repo := range []string{"mesh-catalog", "mesh-controller", "mesh-host"} { - p := link.PullUpdated{Owner: "novox", Repo: repo, Number: 400, Head: aRelease, Base: "main", Commit: "abc", - Paths: []string{"modules/gitea/index.ts"}} - got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries) - if err != nil || got != "" || *asked != 0 { - t.Fatalf("%s's release branch into main read as release %q (%v) after %d question(s)", repo, got, err, *asked) - } - } - clone := func(s inventory.Source) (string, error) { return "http://git/" + s.Repository + ".git", nil } - beside, err := besideOf(entries, func(module string) string { return "running-" + module }, clone, "") - if err != nil { - t.Fatal(err) - } - for dir, b := range beside { - if b.Ref == aRelease || b.Else != "" { - t.Fatalf("beside a trunk pull request %s is read at %+v", dir, b) - } - } -} - -// **A forge answer that names no branch is an error**, never "no release branch"; a 404 is "no such branch". -func TestTheForgesAnswerAboutABranchIsReadStrictly(t *testing.T) { - for _, c := range []struct { - what string - answer link.Answer - has bool - fails bool - }{ - {"the branch named", link.Answer{Result: []byte(`{"result":{"name":"` + aRelease + `"}}`)}, true, false}, - {"not found", link.Answer{Error: "Gitea API /repos/novox/mesh-controller/branches/x: 404 {\"message\":\"not found\"}"}, - false, false}, - {"an empty branch name", link.Answer{Result: []byte(`{"result":{"name":""}}`)}, false, true}, - {"no result", link.Answer{Result: []byte(`{}`)}, false, true}, - {"another branch named", link.Answer{Result: []byte(`{"result":{"name":"main"}}`)}, false, true}, - {"unreadable", link.Answer{Result: []byte(`not json`)}, false, true}, - {"another error", link.Answer{Error: "Gitea API: 500 internal"}, false, true}, - } { - has, err := branchAnswered(c.answer, aRelease) - if has != c.has || (err != nil) != c.fails { - t.Errorf("%s: has %v, error %v", c.what, has, err) - } - } -} - -// **The trunk is the repository's own, not a name**: a repository whose trunk is not main, one that follows main, and -// a controller whose trunk is not main. -func TestATrunkIsTheBranchTheRepositoryFollows(t *testing.T) { - asked := aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {"main", aRelease, "master"}}, nil) - legacy := fromRepo("legacy", "novox/legacy", "") - legacy.Source.Ref = "master" - onMain := fromRepo("photos", "novox/photos", "") - onMain.Source.Ref = "main" - entries := append(theCore(), legacy, onMain) - for _, c := range []struct { - what, repo, base, release string - asks int - }{ - {"a repository whose trunk is master, into master", "legacy", "master", "", 0}, - {"a repository whose trunk is master, into the release branch", "legacy", aRelease, aRelease, 1}, - {"a repository whose module follows main, into main", "photos", "main", "", 0}, - {"a repository whose module follows main, into the release branch", "photos", aRelease, aRelease, 1}, - } { - *asked = 0 - p := link.PullUpdated{Owner: "novox", Repo: c.repo, Base: c.base, Commit: "abc", Paths: []string{"x"}} - got, err := releaseBranchOf(t.Context(), p, pullScope(p, entries, nil, nil), entries) - if err != nil || got != c.release || *asked != c.asks { - t.Errorf("%s: release %q (%v) after %d question(s), not %q after %d", c.what, got, err, *asked, c.release, c.asks) - } - } - - // A controller whose trunk is not main: main may be a release branch there, and its own trunk never is. - core := theCore() - core[0].Source.Ref = "trunk" - for _, c := range []struct { - base, release string - asks int - }{{"trunk", "", 0}, {"main", "main", 1}} { - *asked = 0 - p := link.PullUpdated{Owner: "novox", Repo: "hq", Base: c.base, Commit: "abc", Paths: []string{"x"}} - got, err := releaseBranchOf(t.Context(), p, pullScope(p, core, nil, nil), core) - if err != nil || got != c.release || *asked != c.asks { - t.Errorf("hq into %s, the controller following trunk: release %q (%v) after %d question(s)", c.base, got, err, - *asked) - } - } -} - -// trunkOf: the branch a source follows, and main when it records none or records a commit (issue 215's rule). -func TestATrunkIsMainWhenTheSourceRecordsNone(t *testing.T) { - for ref, want := range map[string]string{"": "main", "main": "main", "master": "master", "nox-mesh": "nox-mesh", - "0123456789abcdef0123456789abcdef01234567": "main", " ": "main"} { - if got := trunkOf(inventory.Source{Ref: ref}); got != want { - t.Errorf("a source recording %q follows %q, not %q", ref, got, want) - } - } -} diff --git a/cmd/mesh-controller/release_plan.go b/cmd/mesh-controller/release_plan.go index e10eb7cf..69aa1f8d 100644 --- a/cmd/mesh-controller/release_plan.go +++ b/cmd/mesh-controller/release_plan.go @@ -114,7 +114,7 @@ func reachableFrom(moved []string, edges []inventory.Edge) []string { // packages edge, read from a record made before novox/hq ADR 0267, widens nothing either: // a shared file moves each module whose build source holds it, directly. if e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || e.Kind == inventory.EdgePackages || - e.Kind == edgeGroupOrder { + e.Kind == edgeChangeOrder { continue } if in[e.To] && !in[e.From] { @@ -200,11 +200,15 @@ func gates(p inventory.Plan, edges []inventory.Edge, rollsOut func(string) bool) } seen := map[string]bool{} var out []string + // The plan's own order (ADR 0419) is not in the catalogue's graph: it is kept on the plan. + if p.Delivery != nil { + edges = append(append([]inventory.Edge{}, edges...), p.Delivery.Order...) + } for _, e := range edges { // A delivery group's order inside a walk (novox/hq ADR 0276 decision 5) gates as built-by does: the // member before is running on its machines before the member after is asked. // A built-by edge waits for nothing since novox/hq ADR 0360 §3. - ordered := e.Kind == edgeGroupOrder + ordered := e.Kind == edgeChangeOrder if later[e.From] && inTier[e.To] && ordered && !seen[e.To] && rollsOut(e.To) { seen[e.To] = true out = append(out, e.To) diff --git a/cmd/mesh-controller/release_provisioned_integration_test.go b/cmd/mesh-controller/release_provisioned_integration_test.go new file mode 100644 index 00000000..547e0b73 --- /dev/null +++ b/cmd/mesh-controller/release_provisioned_integration_test.go @@ -0,0 +1,145 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { + t.Helper() + for i := 0; i < n; i++ { + advancePlans(t.Context(), b.open) + } + return b.release(t) +} + +func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { + judgeLeavesAsTheCore(t) + applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} + + t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) + } + }) + t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || + !strings.Contains(p.Note, "provisioning failed (app: no value)") { + t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) + } + }) + t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { + b := aBacklog(t) + handingAnchor(t, b, map[string]string{"app": "t1"}) + p := stepRelease(t, b, 8) + g := p.Release.Gates["anchor"] + if g == nil { + t.Fatalf("no gate: %+v", p.Release) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { + t.Fatal(err) + } + p = stepRelease(t, b, 1) + if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("d both applied: the gate passes", func(t *testing.T) { + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1"}) + says["anchor/app"] = applied + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { + b := aBacklog(t) + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + }) + t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { + // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report + // is none of this delivery's. + b := aBacklog(t) + says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) + says["anchor/app"] = applied + says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} + p := stepRelease(t, b, 8) + if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { + t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) + } + }) +} + +// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and +// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). +func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { + t.Helper() + says := map[string]inventory.ProvReport{} + wasSays := provAppliedOn + provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { + r, ok := says[node+"/"+module] + return r, ok, nil + } + inner := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + got, err := inner(ctx, open, names) + if err != nil { + return got, err + } + for _, n := range names { + if n != "anchor" { + continue + } + inv := open.inventory + s, _, err := inv.SentTo(ctx, n) + if err != nil { + return nil, err + } + builds, _, err := inv.SentBuilds(ctx, n) + if err != nil { + return nil, err + } + plans := map[string]inventory.Provisioning{} + for m, d := range handed { + plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} + } + raw, err := json.Marshal(plans) + if err != nil { + return nil, err + } + id := nodeID(t, open, n) + if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { + return nil, err + } + } + return got, nil + } + t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) + return says +} diff --git a/cmd/mesh-controller/release_provisioned_test.go b/cmd/mesh-controller/release_provisioned_test.go index 52241dcf..da197dfa 100644 --- a/cmd/mesh-controller/release_provisioned_test.go +++ b/cmd/mesh-controller/release_provisioned_test.go @@ -1,9 +1,6 @@ package main import ( - "context" - "encoding/json" - "slices" "strings" "testing" "time" @@ -11,139 +8,9 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// handingAnchor makes every send to anchor hand the provisioner the given modules (module -> templates digest), and -// the clients' words the given reports (key `node/module`), as the stream would give them (issue 840, ADR 0362). -func handingAnchor(t *testing.T, b *backlogMesh, handed map[string]string) map[string]inventory.ProvReport { - t.Helper() - says := map[string]inventory.ProvReport{} - wasSays := provAppliedOn - provAppliedOn = func(_ context.Context, node, module string) (inventory.ProvReport, bool, error) { - r, ok := says[node+"/"+module] - return r, ok, nil - } - inner := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - got, err := inner(ctx, open, names) - if err != nil { - return got, err - } - for _, n := range names { - if n != "anchor" { - continue - } - inv := open.inventory - s, _, err := inv.SentTo(ctx, n) - if err != nil { - return nil, err - } - builds, _, err := inv.SentBuilds(ctx, n) - if err != nil { - return nil, err - } - plans := map[string]inventory.Provisioning{} - for m, d := range handed { - plans[m] = inventory.Provisioning{Templates: inventory.TemplatesArchive{Digest: d}} - } - raw, err := json.Marshal(plans) - if err != nil { - return nil, err - } - id := nodeID(t, open, n) - if err := inv.RecordSentWith(ctx, id, s.Digest, builds, 0, 0, inventory.Send{Node: id, Provisioning: raw}); err != nil { - return nil, err - } - } - return got, nil - } - t.Cleanup(func() { sendRollout, provAppliedOn = inner, wasSays }) - return says -} - -func stepRelease(t *testing.T, b *backlogMesh, n int) inventory.Plan { - t.Helper() - for i := 0; i < n; i++ { - advancePlans(t.Context(), b.open) - } - return b.release(t) -} - // Issue 840 (novox/hq ADR 0362 §1-5): the release's rest gate, which every delivery passes since ADR 0360, counts a // machine's send of a provisioned module applied only once the engine and the machine's provisioner client both said // so; a failed client is a failed apply named with its reason; none past ApplySilentAfter is silent, never done. -func TestTheRestGateWaitsForTheProvisioner(t *testing.T) { - judgeLeavesAsTheCore(t) - applied := inventory.ProvReport{Status: inventory.ProvApplied, Templates: "t1", At: time.Now()} - - t.Run("a engine applied, the client has not said: no verdict, the release stays open", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is judged before its provisioner reported: %s %s %+v", p.State, p.Note, g) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop hands nothing and passes on the engine's report: %+v", p.Release) - } - }) - t.Run("b the client failed: a failed apply named with the reason", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t1", Reason: "no value", At: time.Now()} - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil || g.Verdict != inventory.GateFailed || !slices.Contains(p.Release.Failed, "anchor") || - !strings.Contains(p.Note, "provisioning failed (app: no value)") { - t.Fatalf("the failure is not named: %s %q %+v %+v", p.State, p.Note, g, p.Release) - } - }) - t.Run("c no client report past ApplySilentAfter: silent, never done", func(t *testing.T) { - b := aBacklog(t) - handingAnchor(t, b, map[string]string{"app": "t1"}) - p := stepRelease(t, b, 8) - g := p.Release.Gates["anchor"] - if g == nil { - t.Fatalf("no gate: %+v", p.Release) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := b.open.inventory.SavePlan(t.Context(), &p); err != nil { - t.Fatal(err) - } - p = stepRelease(t, b, 1) - if !slices.Equal(p.Release.Silent, []string{"anchor"}) || slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("anchor is not silent: %s %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("d both applied: the gate passes", func(t *testing.T) { - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1"}) - says["anchor/app"] = applied - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("e a send handing nothing passes on the engine's report", func(t *testing.T) { - b := aBacklog(t) - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - }) - t.Run("f a module the delivery does not carry does not touch its end", func(t *testing.T) { - // anchor's declaration also hands `same`, rebuilt with the very bytes and so not carried: its failed report - // is none of this delivery's. - b := aBacklog(t) - says := handingAnchor(t, b, map[string]string{"app": "t1", "same": "t9"}) - says["anchor/app"] = applied - says["anchor/same"] = inventory.ProvReport{Status: inventory.ProvFailed, Templates: "t9", Reason: "stuck", At: time.Now()} - p := stepRelease(t, b, 8) - if p.State != inventory.PlanDone || !slices.Contains(p.Release.Done, "anchor") || len(p.Release.Failed) > 0 { - t.Fatalf("another module's failure ended this delivery: %s %s %+v", p.State, p.Note, p.Release) - } - }) -} // Issue 840: a walk's end waits on the modules it sent to a machine, not on every module that machine's declaration // hands to the provisioner. diff --git a/cmd/mesh-controller/release_test.go b/cmd/mesh-controller/release_test.go index a2fd2589..b80429be 100644 --- a/cmd/mesh-controller/release_test.go +++ b/cmd/mesh-controller/release_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay164_test.go b/cmd/mesh-controller/replay164_test.go index 34175bda..78f0e6a8 100644 --- a/cmd/mesh-controller/replay164_test.go +++ b/cmd/mesh-controller/replay164_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay201_test.go b/cmd/mesh-controller/replay201_test.go index c852ef20..f7559eee 100644 --- a/cmd/mesh-controller/replay201_test.go +++ b/cmd/mesh-controller/replay201_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay237_test.go b/cmd/mesh-controller/replay237_test.go index 85f7c071..beafe80a 100644 --- a/cmd/mesh-controller/replay237_test.go +++ b/cmd/mesh-controller/replay237_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay239_test.go b/cmd/mesh-controller/replay239_test.go index 46282a41..ed9b9ffa 100644 --- a/cmd/mesh-controller/replay239_test.go +++ b/cmd/mesh-controller/replay239_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay309_test.go b/cmd/mesh-controller/replay309_test.go index f80b948e..c7e46c5d 100644 --- a/cmd/mesh-controller/replay309_test.go +++ b/cmd/mesh-controller/replay309_test.go @@ -7,8 +7,7 @@ import ( "github.com/novox/mesh-controller/internal/inventory" ) -// novox/hq issue 309, replayed with only what orderOf had before its fix, so it can be laid over the older -// commit. The group `feat/a-machine-joins-through-the-tunnel` (2026-10-08): the controller's pull request +// novox/hq issue 309, replayed. The group `feat/a-machine-joins-through-the-tunnel` (2026-10-08): the controller's pull request // moved the controller and the build agent, the node-engine's moved the node-engine, the lab's moved // nothing. The build agent builds the node-engine (built by: the controller's member first), and the // node-engine goes before the controller (engine before controller: the node-engine's member first). Both @@ -34,17 +33,14 @@ func TestReplay309(t *testing.T) { {From: "mesh-host", To: "build-agent", Kind: inventory.EdgeBuiltBy}, {From: "route-proxy", To: "build-agent", Kind: inventory.EdgeBuiltBy}, } - got := orderOf(members, reach, edges) + got := orderOf(members, reach, edges, map[string]moduleChange{"mesh-controller": {Manifest: true}}) if len(got.Cycle) > 0 { t.Fatalf("the group is refused as a cycle %v; pairs %+v", got.Cycle, got.Pairs) } - want := []string{"novox/mesh-controller@8bbfdb53db2b", "novox/mesh-host@86cbebd10f8a", "novox/mesh-lab@e80a4b1642bf"} + // Being built by the build seat's holder orders nothing (ADR 0360 section 3, ADR 0419): the node-engine goes + // first by the rollout default, and no pair puts the controller's member before it. + want := []string{"novox/mesh-host@86cbebd10f8a", "novox/mesh-controller@8bbfdb53db2b", "novox/mesh-lab@e80a4b1642bf"} if !reflect.DeepEqual(got.Order, want) { t.Fatalf("ordered %v, wanted %v; pairs %+v", got.Order, want, got.Pairs) } - for _, p := range got.Pairs { - if p.Before == want[1] && p.After == want[0] { - t.Fatalf("a pair that lost still stands: %+v", p) - } - } } diff --git a/cmd/mesh-controller/replay318_test.go b/cmd/mesh-controller/replay318_test.go index df8a33b6..a84699b8 100644 --- a/cmd/mesh-controller/replay318_test.go +++ b/cmd/mesh-controller/replay318_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay325_test.go b/cmd/mesh-controller/replay325_test.go index d3f0c6f1..c058c2d1 100644 --- a/cmd/mesh-controller/replay325_test.go +++ b/cmd/mesh-controller/replay325_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay327_test.go b/cmd/mesh-controller/replay327_test.go index bac8296d..2410455b 100644 --- a/cmd/mesh-controller/replay327_test.go +++ b/cmd/mesh-controller/replay327_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay335_integration_test.go b/cmd/mesh-controller/replay335_integration_test.go new file mode 100644 index 00000000..98d142a2 --- /dev/null +++ b/cmd/mesh-controller/replay335_integration_test.go @@ -0,0 +1,68 @@ +//go:build integration + +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" +) + +func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + tm := aTierMesh(t, "m01", "m02") + ctx := t.Context() + inv := tm.open.inventory + + advancePlans(ctx, tm.open) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine once, for the tier", tm.sent) + } + p := tm.plan(t) + lead, carried := p.Modules["m01"], p.Modules["m02"] + if lead.Gate == nil || carried.GatedBy != "m01" { + t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried) + } + // The lead passed; the carried module's turn did not come. The first send is past the wait's bound. + sent := time.Now().UTC().Add(-planWaitBound - time.Minute) + judged := sent.Add(2 * time.Minute) + lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent + lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed, + "healthy 3 times over 2m5s", &judged, true + carried.Gate = nil + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + // Another walk's send reached anchor, which has not reported on it. + if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere", + map[string]string{"m01": "c2", "m02": "c2"}); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, tm.open) + p = tm.plan(t) + if p.State == inventory.PlanFailed { + t.Fatalf("the plan failed after its gate passed: %s", p.Note) + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v: the rest once, as the pass said", tm.sent) + } + current, err := inv.CurrentBuilds(ctx) + if err != nil { + t.Fatal(err) + } + for _, m := range []string{"m01", "m02"} { + if current[m].Commit != "c2" { + t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { + t.Errorf("%s's build was marked failed at its gate after the gate passed", m) + } + } + if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed { + t.Errorf("m02 did not take over m01's pass: %+v", g) + } +} diff --git a/cmd/mesh-controller/replay335_test.go b/cmd/mesh-controller/replay335_test.go index 27031906..5cd51649 100644 --- a/cmd/mesh-controller/replay335_test.go +++ b/cmd/mesh-controller/replay335_test.go @@ -77,59 +77,3 @@ func testAWalkStoppedAfterItsGatePassedKeepsThePass(t *testing.T) { // no gate of its own yet. Since then another send reached the first machine and it has not reported on it, and // the first send is older than the wait for a first machine's report. Read before the pass is taken over, the // carried module's step said the first machine never applied it, and the passed build was put back. -func TestACarriedModuleTakesItsLeadsPassBeforeItsStepIsRead(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "m01", "m02") - ctx := t.Context() - inv := tm.open.inventory - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the tier", tm.sent) - } - p := tm.plan(t) - lead, carried := p.Modules["m01"], p.Modules["m02"] - if lead.Gate == nil || carried.GatedBy != "m01" { - t.Fatalf("m02 is not carried by m01's send: lead %+v, carried %+v", lead.Gate, carried) - } - // The lead passed; the carried module's turn did not come. The first send is past the wait's bound. - sent := time.Now().UTC().Add(-planWaitBound - time.Minute) - judged := sent.Add(2 * time.Minute) - lead.FirstAt, carried.FirstAt, lead.Gate.Since = &sent, &sent, &sent - lead.Gate.Verdict, lead.Gate.Why, lead.Gate.JudgedAt, lead.Gate.Kept = inventory.GatePassed, - "healthy 3 times over 2m5s", &judged, true - carried.Gate = nil - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - // Another walk's send reached anchor, which has not reported on it. - if err := inv.RecordSent(ctx, nodeID(t, tm.open, "anchor"), "d-anchor-elsewhere", - map[string]string{"m01": "c2", "m02": "c2"}); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, tm.open) - p = tm.plan(t) - if p.State == inventory.PlanFailed { - t.Fatalf("the plan failed after its gate passed: %s", p.Note) - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: the rest once, as the pass said", tm.sent) - } - current, err := inv.CurrentBuilds(ctx) - if err != nil { - t.Fatal(err) - } - for _, m := range []string{"m01", "m02"} { - if current[m].Commit != "c2" { - t.Errorf("%s was put back to %s after its gate passed", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Errorf("%s's build was marked failed at its gate after the gate passed", m) - } - } - if g := p.Modules["m02"].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Errorf("m02 did not take over m01's pass: %+v", g) - } -} diff --git a/cmd/mesh-controller/replay336_test.go b/cmd/mesh-controller/replay336_test.go index 20fa6dec..7e6b70f3 100644 --- a/cmd/mesh-controller/replay336_test.go +++ b/cmd/mesh-controller/replay336_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay340_test.go b/cmd/mesh-controller/replay340_test.go index 225c0c20..a1b2df52 100644 --- a/cmd/mesh-controller/replay340_test.go +++ b/cmd/mesh-controller/replay340_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay348_test.go b/cmd/mesh-controller/replay348_test.go index 97fcfdb4..30f23918 100644 --- a/cmd/mesh-controller/replay348_test.go +++ b/cmd/mesh-controller/replay348_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay349_test.go b/cmd/mesh-controller/replay349_test.go index fae683a2..ae71bb92 100644 --- a/cmd/mesh-controller/replay349_test.go +++ b/cmd/mesh-controller/replay349_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay401_test.go b/cmd/mesh-controller/replay401_test.go index 0c4fd94b..dfe1135f 100644 --- a/cmd/mesh-controller/replay401_test.go +++ b/cmd/mesh-controller/replay401_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay614_test.go b/cmd/mesh-controller/replay614_test.go index bd256104..411330c8 100644 --- a/cmd/mesh-controller/replay614_test.go +++ b/cmd/mesh-controller/replay614_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/replay867_test.go b/cmd/mesh-controller/replay867_test.go index 5e8d67db..49a5fc0f 100644 --- a/cmd/mesh-controller/replay867_test.go +++ b/cmd/mesh-controller/replay867_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( @@ -365,105 +367,3 @@ func TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn(t *testing.T) { t.Fatalf("mesh-delivery was asked %d times", asked) } } - -// **A delivery group's composed check is judged by the group's own controller** (novox/hq ADR 0239 §14), into a release -// branch too: the controller member and the mesh-host member of the pair are consistent only together (the controller -// calls what only the node-engine change brings), and both merge into a release branch. The release judge (the -// controller of the release branch with the validator of its own) cannot build it; the group's own controller, as -// merged, with the node-engine member's validator, can. It drives askGroupCheck and the build seat's Check for real. -func TestAGroupsCompositionIntoAReleaseBranchIsJudgedByItsOwnController(t *testing.T) { - groupIntoARelease(t, false) -} - -// The shape the live check had: the node-engine member is the group's primary, so the composed request is asked with -// Judge "validator" (a union of modules that includes mesh-host's) and the release branch set; the release judge then -// put this change's validator into the old controller and "judge-with-this-validator" did not build. -func TestAGroupsCompositionJudgedWithTheValidatorIntoAReleaseBranchIsJudgedByItsOwnController(t *testing.T) { - groupIntoARelease(t, true) -} - -func groupIntoARelease(t *testing.T, hostPrimary bool) { - if _, err := exec.LookPath("go"); err != nil { - t.Skip("no Go toolchain on this machine to build a judge with") - } - const release = "feat/r" - r := theGroupOfIssue867Into(t, release) - aForgeWithBranches(t, map[string][]string{"novox/mesh-controller": {release}}, nil) - // The pair renames what the validator says: the node-engine change drops Word for Word2, the controller change - // calls Word2. The controller of the release branch calls Word, so it does not build with the new validator; the - // controller change does not build with the validator it vendors. Only together are they consistent. - r867Git(t, r.hostRepo, "checkout", "--quiet", "-b", "swap", release) - r.hostHead = r867Commit(t, r.hostRepo, map[string]string{"validate/v.go": "package validate\n\nconst Word2 = \"the new validator\"\n"}) - r867Git(t, r.hostRepo, "checkout", "--quiet", "main") - r867Git(t, r.controllerRepo, "checkout", "--quiet", "-b", "swap", release) - swapped := strings.Replace(r867Controller("the controller", "")["cmd/mesh-controller/main.go"], "validate.Word)", "validate.Word2)", 1) - r.controllerHead = r867Commit(t, r.controllerRepo, map[string]string{"cmd/mesh-controller/main.go": swapped}) - r867Git(t, r.controllerRepo, "checkout", "--quiet", "main") - js, _ := aBuildQueue(t) - servingBus.Store(js) - t.Cleanup(func() { servingBus.Store(nil) }) - members := []orderMember{ - {ID: "novox/mesh-controller@" + r.controllerHead[:12], Repository: "novox/mesh-controller", Base: release, - Head: r.controllerHead, Number: 346, Paths: []string{"lock.txt", "cmd/mesh-controller/main.go"}, ModuleDirsSaid: true}, - {ID: "novox/mesh-host@" + r.hostHead[:12], Repository: "novox/mesh-host", Base: release, Head: r.hostHead, Number: 114, - Paths: []string{"validate/v.go"}, ModuleDirsSaid: true}} - if hostPrimary { - members[0], members[1] = members[1], members[0] - } - if _, err := askGroupCheck(t.Context(), r.open, "feat/867", members); err != nil { - t.Fatal(err) - } - q, err := link.ReadQueue(t.Context(), js, link.TheBuildMachine) - if err != nil || len(q.Asks) == 0 { - t.Fatalf("nothing was asked of the build seat: %v", err) - } - request := q.Asks[len(q.Asks)-1].Request - if request.Check.Release != release || request.Check.Group != "feat/867" || len(request.Check.Members) != 1 || len(request.Check.Modules) == 0 { - t.Fatalf("the composed check is asked as release %q, group %q, %d other head(s)", request.Check.Release, - request.Check.Group, len(request.Check.Members)) - } - if hostPrimary { - if request.Check.Repo != "mesh-host" { - t.Fatalf("the primary is %s, not the node-engine", request.Check.Repo) - } - // As the live ask had it (the mesh-host module among the union's): the node-engine's validator judges. - request.Check.Judge = link.JudgeValidator - } - // What the build seat makes of the ask. - var wire map[string]json.RawMessage - body, _ := json.Marshal(request.Check) - _ = json.Unmarshal(body, &wire) - delete(wire, "group") - delete(wire, "members") - body, _ = json.Marshal(wire) - var spec builder.CheckSpec - if err := json.Unmarshal(body, &spec); err != nil { - t.Fatal(err) - } - for _, m := range request.Check.Members { - spec.Group = append(spec.Group, builder.GroupHead{Owner: m.Owner, Repo: m.Repo, Repository: m.Repository, Ref: m.Ref, - Paths: m.Paths, Base: m.Base}) - } - spec.ID, spec.Repository, spec.Ref = request.ID, request.Repository, request.Ref - spec.Toolchain, spec.JudgeCache = "golang", r.judgeCache - v, err := builder.Check(t.Context(), builder.Command, spec, t.TempDir(), r.registry, builder.GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || v.Gate.Verdict != "pass" { - t.Fatalf("the group's composition into %s was not judged by its own controller: %+v\n%s", release, v.Gate, v.Report) - } - if mark := "[release " + release + ", judged by the group's controller " + v.JudgedBy + "] "; !strings.HasPrefix(v.Gate.Summary, mark) || - !strings.Contains(v.Report, "by the group's controller at "+v.JudgedBy) { - t.Fatalf("the statuses and the report do not say the group's controller judged:\n%s\n%s", v.Gate.Summary, v.Report) - } - if !strings.Contains(v.Gate.Summary, "the new validator") { - t.Fatalf("the gate was judged by %q, not by the group's controller with the node-engine member's validator", v.Gate.Summary) - } - if !strings.HasPrefix(v.JudgeFrom, "the group's own controller") || !strings.Contains(v.Validator, "the group's mesh-host merge") { - t.Fatalf("the verdict does not say where the judge and the validator came from: %q / %q", v.JudgeFrom, v.Validator) - } - if v.Release != release || v.JudgedBy == "" { - t.Fatalf("the verdict names release %q judged by %q", v.Release, v.JudgedBy) - } -} diff --git a/cmd/mesh-controller/replays_integration_test.go b/cmd/mesh-controller/replays_integration_test.go new file mode 100644 index 00000000..d0127410 --- /dev/null +++ b/cmd/mesh-controller/replays_integration_test.go @@ -0,0 +1,501 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "slices" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", + Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) + register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", + Requires: []string{"wildcard-resolution"}}) + for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + // The consumer's machine composes first, as a push does: what it is sent is what its provider grants. + for _, node := range []string{"laptop", "anchor"} { + plan, settings, err := planFor(ctx, open, node) + if err != nil { + t.Fatalf("%s does not resolve: %v", node, err) + } + if _, err := declarationFor(ctx, open, node, plan, settings); err != nil { + t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+ + "mesh_laptop_networkmanager (26 characters), refused it: %v", node, err) + } + } +} + +func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { + t.Fatal(err) + } + for _, m := range []catalogue.Manifest{ + {Module: "store", Version: "1", + Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, + Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, + Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, + {Module: "resolver", Version: "1", + Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, + {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, + {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, + } { + register(t, open, m) + } + assignAll := func(pairs ...[2]string) { + for _, a := range pairs { + if _, err := assign(ctx, open, a[0], a[1]); err != nil { + t.Fatalf("assign %s %s: %v", a[0], a[1], err) + } + } + } + // The control node holds the mesh's store and resolver seats; the home server runs its own of each. + assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) + for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { + if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { + t.Fatal(err) + } + } + assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, + [2]string{"laptop", "board"}) + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var board, network string + for _, n := range plan.Needs { + switch { + case n.For == "board" && n.Name == "postgres-database": + board = n.From + case n.For == "network" && n.Name == "wildcard-resolution": + network = n.From + } + } + if board != "laptop" { + t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+ + "database there (issue 273)", board) + } + if network != "anchor" { + t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network) + } +} + +func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { + withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + + crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + + `"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` + + `"streak":5,"restarts":2}]}`) + if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the engine's statement of the crash loop: %v", err) + } + crashLoop = raw + } + // `null` is an engine that states nothing — older than the judging — and its reports carry no health. + var stated map[string]any + if err := json.Unmarshal(crashLoop, &stated); err != nil { + t.Fatal(err) + } + + for _, b := range []inventory.Build{ + {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, + {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", + Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, + } { + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) + b.Manifest = manifest + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + registerAt := func(commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + registerAt("c1", time.Now().Add(-2*time.Hour)) + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + registerAt("c2", time.Now().Add(-time.Minute)) + + // The machine: what it is sent is applied, and its report says what runs is starting. + listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} + sequence := int64(0) + say := func(node, digest, state string) { + t.Helper() + sequence++ + health := map[string]any{} + for k, v := range stated { + health[k] = v + } + health["at"] = time.Now().UTC().Format(time.RFC3339Nano) + if state != "" && stated != nil { + resources := []any{} + for _, r := range stated["resources"].([]any) { + kept := map[string]any{} + for k, v := range r.(map[string]any) { + kept[k] = v + } + kept["state"], kept["reason"] = state, "" + resources = append(resources, kept) + } + health["resources"] = resources + } + said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest, + "report_sequence": sequence} + if stated != nil { + said["health"] = health + } + body, _ := json.Marshal(said) + var report link.Report + if err := json.Unmarshal(body, &report); err != nil { + t.Fatal(err) + } + if _, err := listener.Heard(ctx, report); err != nil { + t.Fatal(err) + } + } + var sent [][]string + last := map[string]string{} + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + sent = append(sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { + return nil, err + } + last[node] = digest + say(node, digest, "starting") + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, 0 + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + + advancePlans(ctx, open) // the first machine is sent the new build, and starts it + if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) { + t.Fatalf("sent %v, not the first machine first", sent) + } + advancePlans(ctx, open) // judged while it starts + // Its container exits at start, and the runtime restarts it: the machine says so, again and again. + for i := 0; i < 6 && len(sent) == 1; i++ { + say("anchor", last["anchor"], "") + advancePlans(ctx, open) + } + gateBound = -time.Second // and the bound passes + advancePlans(ctx, open) + + p, err := inv.PlanByID(ctx, "plan-crashloop") + if err != nil { + t.Fatal(err) + } + gate := p.Modules["app"].Gate + for _, names := range sent { + if slices.Contains(names, "laptop") { + t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate) + } + } + if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed { + t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v", + p.State, p.Note, gate) + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { + t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) + } +} + +func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + + `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + + `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) + if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("the engine's statement of the silent web application: %v", err) + } + silent = raw + } + var h link.Health + if err := json.Unmarshal(silent, &h); err != nil { + t.Fatal(err) + } + if h.Contract == 0 { + t.Fatal("the engine states nothing of the silent web application: it is older than the judging") + } + open1 := func() []conditions.Condition { + t.Helper() + list, err := conditionsFrom.Open(ctx) + if err != nil { + t.Fatal(err) + } + return list + } + for look := 1; look <= 2; look++ { + at := time.Now().Add(time.Duration(look) * time.Second) + h.At = at + if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { + t.Fatal(err) + } + raised := open1() + switch { + case look == 1 && len(raised) != 0: + t.Fatalf("one look raised %v", raised[0].Key) + case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): + t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) + case look == 2: + t.Logf("raised on the second look: %s", raised[0].Summary) + } + } +} + +func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + asked := asksRecorded(t) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa", + Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + advanceHeld(ctx, open) + recent, err := inv.RecentPlans(ctx, 1) + if err != nil || len(recent) != 1 || len(*asked) != 1 { + t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) + } + before := recent[0] + if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { + t.Fatalf("the plan is not waiting on its build: %+v", before) + } + for range 3 { + advanceHeld(ctx, open) + } + after, err := inv.PlanByID(ctx, before.ID) + if err != nil { + t.Fatal(err) + } + if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { + t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", + before.Revision, after.Revision, before.Updated, after.Updated) + } +} + +func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299", + Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}} + if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil { + t.Fatal(err) + } + rows, err := inv.Seats(ctx) + if err != nil { + t.Fatal(err) + } + // The working set as the serving controller holds it: the store's rows, the compiled seats' marks. + // A seat the compiled set does not know keeps its mark only as defined here. + for i := range rows { + if rows[i].Name == catchingUp.Name { + rows[i].Serves = catchingUp.Serves + } + } + catalogue.UseSeats(rows) + t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) + + seats := []string{catchingUp.Name} + if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 { + allOptional := true + for _, v := range uplink.Serves { + allOptional = allOptional && v.Optional + } + if allOptional { + seats = append(seats, "node-uplink") + } else { + t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is") + } + } + + register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"}) + now := time.Now() + var heard []machineFacts + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil { + t.Fatal(err) + } + for _, seat := range seats { + if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil { + t.Fatal(err) + } + } + heard = append(heard, machineFacts{name: node, lastHeard: now}) + } + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}} + + found, err := probeHolders(ctx, d) + if err != nil { + t.Fatal(err) + } + for _, o := range found { + for _, seat := range seats { + if strings.HasPrefix(o.ID, seat+".") { + t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said) + } + } + } +} + +func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + var asked [][3]string + was := askABuild + askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { + asked = append(asked, [3]string{source.Repository, path, ref}) + return "build-" + path, nil + } + t.Cleanup(func() { askABuild = was }) + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", + BuiltFrom: "c0", Head: "c0"}); err != nil { + t.Fatal(err) + } + m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00", + Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"}, + ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true} + if err := (following{open: open}).SourceMoved(ctx, m); err != nil { + t.Fatal(err) + } + want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"} + if len(asked) != 1 || asked[0] != want { + t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want) + } + if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { + t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) + } +} + +func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + start := time.Now().Add(-time.Hour) + image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64) + if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start, + map[string][2]string{"server": {image, ""}}))); err != nil { + t.Fatal(err) + } + for _, node := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, node, "resolver"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil { + t.Fatal(err) + } + } + // The merge: the resolver's new build, registered and held for a person's word. + if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord, + start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil { + t.Fatal(err) + } + + conn := onATestBus(t) + // The operator's yes, machine by machine, as the seat's push records it before it sends. + for _, node := range []string{"anchor", "laptop"} { + set := flag.NewFlagSet("push", flag.ContinueOnError) + why := addHandActFlags(set) + if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node, + "--cause", "split-dns"}); err != nil { + t.Fatal(err) + } + why.record(ctx, "push", []string{node}) + } + acts, err := link.HandActs(ctx, conn, start) + if err != nil { + t.Fatal(err) + } + if len(acts) != 2 { + t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts) + } + f := calm(time.Now()) + f.handActs = acts + for _, o := range watchHandActs(f) { + if strings.Contains(o.ID, "split-dns") { + t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary) + } + } +} diff --git a/cmd/mesh-controller/replays_test.go b/cmd/mesh-controller/replays_test.go index f8a95346..b9c93fe2 100644 --- a/cmd/mesh-controller/replays_test.go +++ b/cmd/mesh-controller/replays_test.go @@ -1,22 +1,11 @@ package main import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" "strings" "testing" "time" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // The replays of the controller's incidents (novox/hq to-be 45 §9, M9): each a scripted replay of what @@ -34,30 +23,6 @@ import ( // consumer's grant — could not compose, and no push to it could go through. The outcome asserted: the // provider's machine composes whatever its consumers are called, and a provision that mints no // credential holds no consumer to a key's length. -func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}}) - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1", - Requires: []string{"wildcard-resolution"}}) - for _, a := range [][2]string{{"anchor", "resolver"}, {"laptop", "networkmanager"}} { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - // The consumer's machine composes first, as a push does: what it is sent is what its provider grants. - for _, node := range []string{"laptop", "anchor"} { - plan, settings, err := planFor(ctx, open, node) - if err != nil { - t.Fatalf("%s does not resolve: %v", node, err) - } - if _, err := declarationFor(ctx, open, node, plan, settings); err != nil { - t.Fatalf("%s cannot be sent anything — the consumer networkmanager on laptop, identified "+ - "mesh_laptop_networkmanager (26 characters), refused it: %v", node, err) - } - } -} // **R273 — a binding to a consumer's data does not move by itself.** On 2026-10-05 a rule written for // the resolver re-bound every database consumer on the home server — which runs its own store, beside @@ -65,65 +30,6 @@ func TestReplay263AnIdentityTooLongNeverRefusesItsProvidersMachine(t *testing.T) // database; five applications ran on empty data for twenty hours. The outcome asserted: a consumer on a // machine running its own store stays bound to it while another machine holds the store's seat; the // resolver, which every holder answers alike, follows its seat. -func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - if _, err := inv.SeedSeats(ctx, catalogue.DefaultSeats()); err != nil { - t.Fatal(err) - } - for _, m := range []catalogue.Manifest{ - {Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "postgres-database", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-store", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"postgres-database": {"port": 5432}}, - Grants: map[string]string{"postgres-database": "/var/lib/mesh/store/grants"}}, - {Module: "resolver", Version: "1", - Provides: []catalogue.Offer{{Name: "wildcard-resolution", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: "mesh-dns-resolver", Scope: catalogue.ScopeMesh}}}, - {Module: "network", Version: "1", Requires: []string{"wildcard-resolution"}}, - {Module: "board", Version: "1", Requires: []string{"postgres-database"}}, - } { - register(t, open, m) - } - assignAll := func(pairs ...[2]string) { - for _, a := range pairs { - if _, err := assign(ctx, open, a[0], a[1]); err != nil { - t.Fatalf("assign %s %s: %v", a[0], a[1], err) - } - } - } - // The control node holds the mesh's store and resolver seats; the home server runs its own of each. - assignAll([2]string{"anchor", "store"}, [2]string{"anchor", "resolver"}) - for _, seat := range [][2]string{{"mesh-store", "store"}, {"mesh-dns-resolver", "resolver"}} { - if err := inv.HoldSeat(ctx, seat[0], catalogue.ScopeMesh, "anchor", seat[1]); err != nil { - t.Fatal(err) - } - } - assignAll([2]string{"laptop", "store"}, [2]string{"laptop", "resolver"}, [2]string{"laptop", "network"}, - [2]string{"laptop", "board"}) - - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var board, network string - for _, n := range plan.Needs { - switch { - case n.For == "board" && n.Name == "postgres-database": - board = n.From - case n.For == "network" && n.Name == "wildcard-resolution": - network = n.From - } - } - if board != "laptop" { - t.Fatalf("the consumer beside its store was bound to the store on %q, which would make it a new, empty "+ - "database there (issue 273)", board) - } - if network != "anchor" { - t.Fatalf("the resolver was bound to %q; its seat is held on anchor (issue 258)", network) - } -} // **R-crashloop — a module that crash-loops after it applied fails its gate on its first machine** (novox/hq // ADR 0240; research 032 §6). On the home server the agent server restarted about a hundred times while the @@ -138,164 +44,6 @@ func TestReplay273AConsumerBesideItsStoreStaysBoundToIt(t *testing.T) { // engine against a real container (mesh-lab replays, R-crashloop), and otherwise what the engine said of // one, kept below. Heard as bytes, so an older controller reads them as it reads any report — this file is // written only with what the controller had before the judging, for the prover to lay over that commit. -func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { - withTheGate(t) // the first-machine gate's own path (novox/hq ADR 0360 keeps it for earlier walks) - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - - crashLoop := []byte(`{"contract":1,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"restarting","since":"2026-10-07T00:00:00Z",` + - `"streak":5,"restarts":2}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the crash loop: %v", err) - } - crashLoop = raw - } - // `null` is an engine that states nothing — older than the judging — and its reports carry no health. - var stated map[string]any - if err := json.Unmarshal(crashLoop, &stated); err != nil { - t.Fatal(err) - } - - for _, b := range []inventory.Build{ - {ID: "build-1", Module: "app", Commit: "c1", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-2 * time.Hour), At: time.Now().Add(-2 * time.Hour)}, - {ID: "build-2", Module: "app", Commit: "c2", Repository: "novox/mesh-catalog", Path: "modules/app", - Asked: time.Now().Add(-time.Minute), At: time.Now().Add(-time.Minute)}, - } { - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: b.Commit}) - b.Manifest = manifest - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - registerAt := func(commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - registerAt("c1", time.Now().Add(-2*time.Hour)) - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - registerAt("c2", time.Now().Add(-time.Minute)) - - // The machine: what it is sent is applied, and its report says what runs is starting. - listener := nudgingListener{Enrolment: link.Enrolment{Inventory: inv}} - sequence := int64(0) - say := func(node, digest, state string) { - t.Helper() - sequence++ - health := map[string]any{} - for k, v := range stated { - health[k] = v - } - health["at"] = time.Now().UTC().Format(time.RFC3339Nano) - if state != "" && stated != nil { - resources := []any{} - for _, r := range stated["resources"].([]any) { - kept := map[string]any{} - for k, v := range r.(map[string]any) { - kept[k] = v - } - kept["state"], kept["reason"] = state, "" - resources = append(resources, kept) - } - health["resources"] = resources - } - said := map[string]any{"node": node, "applied": []string{"app.server"}, "declared": digest, - "report_sequence": sequence} - if stated != nil { - said["health"] = health - } - body, _ := json.Marshal(said) - var report link.Report - if err := json.Unmarshal(body, &report); err != nil { - t.Fatal(err) - } - if _, err := listener.Heard(ctx, report); err != nil { - t.Fatal(err) - } - } - var sent [][]string - last := map[string]string{} - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - sent = append(sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, map[string]string{"app": current["app"].Commit}); err != nil { - return nil, err - } - last[node] = digest - say(node, digest, "starting") - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, 0 - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-crashloop", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", Build: "build-2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - advancePlans(ctx, open) // the first machine is sent the new build, and starts it - if len(sent) == 0 || !slices.Equal(sent[0], []string{"anchor"}) { - t.Fatalf("sent %v, not the first machine first", sent) - } - advancePlans(ctx, open) // judged while it starts - // Its container exits at start, and the runtime restarts it: the machine says so, again and again. - for i := 0; i < 6 && len(sent) == 1; i++ { - say("anchor", last["anchor"], "") - advancePlans(ctx, open) - } - gateBound = -time.Second // and the bound passes - advancePlans(ctx, open) - - p, err := inv.PlanByID(ctx, "plan-crashloop") - if err != nil { - t.Fatal(err) - } - gate := p.Modules["app"].Gate - for _, names := range sent { - if slices.Contains(names, "laptop") { - t.Fatalf("the crash loop passed its gate on anchor and was sent to laptop: sent %v, the gate %+v", sent, gate) - } - } - if gate == nil || gate.Verdict != inventory.GateFailed || p.State != inventory.PlanFailed { - t.Fatalf("a crash-looping build did not fail its gate on the first machine: the plan is %s (%s), its gate %+v", - p.State, p.Note, gate) - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c1" { - t.Fatalf("the module is registered at %s, not put back to c1", current["app"].Commit) - } -} // **R145 — a web application that accepts TCP and answers nothing is raised within two looks** (novox/hq // ADR 0240 rule 4 and Phase B, issue 145). For eleven hours a web application's port was open and its @@ -305,51 +53,6 @@ func TestReplayCrashLoopFailsItsGateOnTheFirstMachine(t *testing.T) { // found looking at such a program; here the controller hears that statement on two looks in a row and // raises the module's condition — the second, never the first. `null` is an engine older than the // readiness check: it states the program alive, and nothing is raised. -func TestReplaySilentWebAppIsRaisedWithinTwoLooks(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - silent := []byte(`{"contract":2,"at":"2026-10-07T00:00:00Z","resources":[{"module":"app","resource":"app.server",` + - `"kind":"container","target":"app-server","state":"unhealthy","reason":"http / on web: no answer within 5s",` + - `"since":"2026-10-07T00:00:00Z","streak":3,"check":"http"}]}`) - if path := os.Getenv("MESH_REPLAY_STATEMENT"); path != "" { - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("the engine's statement of the silent web application: %v", err) - } - silent = raw - } - var h link.Health - if err := json.Unmarshal(silent, &h); err != nil { - t.Fatal(err) - } - if h.Contract == 0 { - t.Fatal("the engine states nothing of the silent web application: it is older than the judging") - } - open1 := func() []conditions.Condition { - t.Helper() - list, err := conditionsFrom.Open(ctx) - if err != nil { - t.Fatal(err) - } - return list - } - for look := 1; look <= 2; look++ { - at := time.Now().Add(time.Duration(look) * time.Second) - h.At = at - if err := stateHealth(ctx, open.inventory, conditionsFrom, "anchor", h, at); err != nil { - t.Fatal(err) - } - raised := open1() - switch { - case look == 1 && len(raised) != 0: - t.Fatalf("one look raised %v", raised[0].Key) - case look == 2 && (len(raised) != 1 || raised[0].Key != "module.app.anchor.unhealthy"): - t.Fatalf("two looks in a row did not raise the module's condition: %v", raised) - case look == 2: - t.Logf("raised on the second look: %s", raised[0].Summary) - } - } -} // **R296 — a plan's clock counts from its tier, not from its last save.** On 2026-10-07 a plan for a // merge to the catalogue, its one module's build asked at 19:48:41 and queued at the build seat for @@ -375,42 +78,6 @@ func TestReplay296APlansLineCountsFromWhenItEnteredItsTier(t *testing.T) { // still waits is advanced as the tick and the mesh's build outcomes advance it, and is not written again — // its revision and its last save stay where its last change left them. Before the fix each advance saved // it, and its revision went from 3 to 6. -func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - asked := asksRecorded(t) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3ad22356aa", - Paths: []string{"modules/app/index.ts"}, ModuleDirs: []string{"modules/app"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - advanceHeld(ctx, open) - recent, err := inv.RecentPlans(ctx, 1) - if err != nil || len(recent) != 1 || len(*asked) != 1 { - t.Fatalf("no plan waiting on its build: %v %v, asked %v", recent, err, *asked) - } - before := recent[0] - if before.State != inventory.PlanBuilding || before.Modules["app"] == nil || before.Modules["app"].State != "asked" { - t.Fatalf("the plan is not waiting on its build: %+v", before) - } - for range 3 { - advanceHeld(ctx, open) - } - after, err := inv.PlanByID(ctx, before.ID) - if err != nil { - t.Fatal(err) - } - if after.Revision != before.Revision || !after.Updated.Equal(before.Updated) { - t.Fatalf("a plan that did not move was saved again: revision %d → %d, saved %s → %s", - before.Revision, after.Revision, before.Updated, after.Updated) - } -} // **R299 — a holder of a seat whose verbs are all optional is not silent.** On 2026-10-07 the // node-uplink seat gained its first verbs, `resolvers` and `links`, both optional while its holders catch @@ -422,75 +89,6 @@ func TestReplay296APlanStandingStillInItsTierIsNotSavedAgain(t *testing.T) { // // The live seat is replayed while its verbs are all optional; beside it, a seat whose verbs are all // optional by definition, so the outcome is still held once node-uplink's verbs are required (step 3). -func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - catchingUp := catalogue.Seat{Name: "node-catching-up", Scope: catalogue.ScopeNode, Decision: "novox/hq issue 299", - Serves: []catalogue.Verb{{Name: "first", Optional: true}, {Name: "second", Optional: true}}} - if _, err := inv.SeedSeats(ctx, append(catalogue.DefaultSeats(), catchingUp)); err != nil { - t.Fatal(err) - } - rows, err := inv.Seats(ctx) - if err != nil { - t.Fatal(err) - } - // The working set as the serving controller holds it: the store's rows, the compiled seats' marks. - // A seat the compiled set does not know keeps its mark only as defined here. - for i := range rows { - if rows[i].Name == catchingUp.Name { - rows[i].Serves = catchingUp.Serves - } - } - catalogue.UseSeats(rows) - t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) - - seats := []string{catchingUp.Name} - if uplink, ok := catalogue.SeatNamed("node-uplink"); ok && len(uplink.Serves) > 0 { - allOptional := true - for _, v := range uplink.Serves { - allOptional = allOptional && v.Optional - } - if allOptional { - seats = append(seats, "node-uplink") - } else { - t.Logf("node-uplink's verbs are required at this commit; the live seat is not replayed, its kind is") - } - } - - register(t, open, catalogue.Manifest{Module: "networkmanager", Version: "1"}) - now := time.Now() - var heard []machineFacts - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "networkmanager"); err != nil { - t.Fatal(err) - } - for _, seat := range seats { - if err := inv.HoldSeat(ctx, seat, catalogue.ScopeNode, node, "networkmanager"); err != nil { - t.Fatal(err) - } - } - heard = append(heard, machineFacts{name: node, lastHeard: now}) - } - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - d := &doctor{open: open, js: js, watchdogs: &watchdogs{last: &signalFacts{now: now, machines: heard}}} - - found, err := probeHolders(ctx, d) - if err != nil { - t.Fatal(err) - } - for _, o := range found { - for _, seat := range seats { - if strings.HasPrefix(o.ID, seat+".") { - t.Errorf("a holder serving none of a seat's optional verbs was said silent: %s (%s)", o.Summary, o.Said) - } - } - } -} // **R300 — a merge that adds a module builds it.** On 2026-10-07 a pull request to the catalogue added // one module, systemd-resolved; its delivery plan said "builds new: modules/systemd-resolved, sent @@ -498,36 +96,6 @@ func TestReplay299AHolderOfASeatWhoseVerbsAreAllOptionalIsNotSilent(t *testing.T // from" and built nothing, so the module was never registered and `assign` refused it until a person // built it by hand. The outcome asserted: the merge asks the build seat for the new module's directory, at // the branch merged into, from the repository as the mesh spells it, and opens no plan for it. -func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - var asked [][3]string - was := askABuild - askABuild = func(_ context.Context, source buildSource, path, ref string) (string, error) { - asked = append(asked, [3]string{source.Repository, path, ref}) - return "build-" + path, nil - } - t.Cleanup(func() { askABuild = was }) - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "networkmanager", Version: "1"}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/networkmanager", Ref: "main", - BuiltFrom: "c0", Head: "c0"}); err != nil { - t.Fatal(err) - } - m := link.SourceMoved{Owner: "novox", Repo: "mesh-catalog", Base: "main", Commit: "3da80a4b00", - Paths: []string{"modules/systemd-resolved/module.json", "modules/systemd-resolved/cmd/main.go"}, - ModuleDirs: []string{"modules/systemd-resolved"}, ModuleDirsSaid: true} - if err := (following{open: open}).SourceMoved(ctx, m); err != nil { - t.Fatal(err) - } - want := [3]string{"novox/mesh-catalog", "modules/systemd-resolved", "main"} - if len(asked) != 1 || asked[0] != want { - t.Fatalf("the merge adding modules/systemd-resolved asked the build seat for %v, not %v", asked, want) - } - if plans, err := inv.OpenPlans(ctx); err != nil || len(plans) != 0 { - t.Fatalf("a merge moving nothing the mesh holds opened a plan: %+v %v", plans, err) - } -} // **R301 — a person's push of a held recorded build is no repair.** On 2026-10-07 the operator approved, // machine by machine, new builds of modules whose upgrade policy records rather than rolls out — the @@ -538,53 +106,3 @@ func TestReplay300AMergeAddingAModuleAsksForItsBuild(t *testing.T) { // days … a healer is wanted"), and the same for `words` and `uplink-verbs`. The outcome asserted: a push // that carries only held recorded builds, recorded as the push records it, wants no healer however // often the person gives that word — through S15 itself, reading the log as the watchdog reads it. -func TestReplay301APersonsPushOfAHeldRecordedBuildIsNoRepair(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - start := time.Now().Add(-time.Hour) - image := "registry.invalid:5000/resolver/server@sha256:" + strings.Repeat("e", 64) - if _, _, err := takeIn(ctx, inv, asTheOperator(t, inv, aContainerBuild(t, "resolver", "c1111111", catalogue.PolicyRecord, start, - map[string][2]string{"server": {image, ""}}))); err != nil { - t.Fatal(err) - } - for _, node := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, node, "resolver"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, node), "d-"+node, map[string]string{"resolver": "c1111111"}); err != nil { - t.Fatal(err) - } - } - // The merge: the resolver's new build, registered and held for a person's word. - if _, _, err := takeIn(ctx, inv, aContainerBuild(t, "resolver", "c2222222", catalogue.PolicyRecord, - start.Add(time.Minute), map[string][2]string{"server": {image, "runtime"}})); err != nil { - t.Fatal(err) - } - - conn := onATestBus(t) - // The operator's yes, machine by machine, as the seat's push records it before it sends. - for _, node := range []string{"anchor", "laptop"} { - set := flag.NewFlagSet("push", flag.ContinueOnError) - why := addHandActFlags(set) - if err := set.Parse([]string{"--why", "operator's yes: the resolver's new build on " + node, - "--cause", "split-dns"}); err != nil { - t.Fatal(err) - } - why.record(ctx, "push", []string{node}) - } - acts, err := link.HandActs(ctx, conn, start) - if err != nil { - t.Fatal(err) - } - if len(acts) != 2 { - t.Fatalf("the two pushes were recorded as %d act(s): %+v", len(acts), acts) - } - f := calm(time.Now()) - f.handActs = acts - for _, o := range watchHandActs(f) { - if strings.Contains(o.ID, "split-dns") { - t.Fatalf("a person's push of a held recorded build was counted as a repair: %s — %s", o.ID, o.Summary) - } - } -} diff --git a/cmd/mesh-controller/rest_at_once_integration_test.go b/cmd/mesh-controller/rest_at_once_integration_test.go new file mode 100644 index 00000000..1aabc267 --- /dev/null +++ b/cmd/mesh-controller/rest_at_once_integration_test.go @@ -0,0 +1,293 @@ +//go:build integration + +package main + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" +) + +func aBacklogOnThree(t *testing.T) *backlogMesh { + t.Helper() + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + if _, err := inv.AddNode(ctx, "desk"); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, "desk", "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + wasHeard := releaseHeard + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { + return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil + } + t.Cleanup(func() { releaseHeard = wasHeard }) + return b +} + +func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + said := events(t) + gateEvery = 0 + for i := 0; i < 12; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") + if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { + t.Fatalf("sends and judgings: %v", *said) + } + if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { + t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) + } + if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } +} + +func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + inv := b.open.inventory + gateEvery = 0 + b.broken["desk"] = true // desk fails what it is sent in the rest's step + advancePlans(ctx, b.open) + for i := 0; i < 12 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { + t.Fatalf("laptop was never sent: %v", b.sent) + } + if !slices.Contains(p.Release.Done, "laptop") { + t.Fatalf("laptop has no verdict of its own: %+v", p.Release) + } + // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. + if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) + } + // app failed on desk: marked, and put back wherever it was sent (ADR 0236). + if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { + t.Fatal("app's build is not marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { + t.Fatalf("desk was not put back: %v", sent) + } +} + +func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once + release := b.release(t) + judging := release.Release.Gate + if judging == nil || judging.Pending || judging.Passes == 0 { + t.Fatalf("the release is not judging anchor: %+v", judging) + } + built := time.Now().UTC() + plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", + Build: "build-app-c2"}}} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again what it runs: %v", b.sent) + } + p, err := inv.PlanByID(ctx, "plan-app") + if err != nil { + t.Fatal(err) + } + g := p.Modules["app"].Gate + if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { + t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) + } + if p.Modules["app"].Previous != "c1" { + t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) + } +} + +func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + // The old controller sends anchor app c2 and is replaced before it keeps the send. + made := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if _, err := made(ctx, open, names); err != nil { + return nil, err + } + return nil, context.Canceled + } + advancePlans(ctx, g.open) + sendRollout = made + if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { + t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) + } + last, _, err := g.open.inventory.SentTo(ctx, "anchor") + if err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if len(g.sent) != 1 { + t.Fatalf("the send was made again: %v", g.sent) + } + s := g.plan(t).Modules["app"] + if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { + t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) + } + // What anchor ran before was kept before the send, so a failure still puts it back (§5). + if s.Previous != "c1" || s.Gate.From != "c1" { + t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) + } +} + +func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + p, err := releaseBacklog(ctx, b.open, "") + if err != nil || p == nil { + t.Fatalf("no release opened: %v", err) + } + if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { + t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) + } + if err := inv.SavePlan(ctx, p); err != nil { + t.Fatal(err) + } + // The send is made, and the controller is replaced before it keeps it. + if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + if len(b.sent) != 1 { + t.Fatalf("anchor was sent again: %v", b.sent) + } + g := b.release(t).Release.Gate + if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { + t.Fatalf("anchor is not judged on the send made: %+v", g) + } +} + +func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { + withTheGate(t) + b := aBacklogOnThree(t) + ctx := t.Context() + gateEvery = 0 + b.broken["anchor"] = true + advancePlans(ctx, b.open) + for i := 0; i < 6 && b.release(t).Open(); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + for _, s := range b.sent { + if !slices.Equal(s, []string{"anchor"}) { + t.Fatalf("sent %v: the rest after a failed first machine", b.sent) + } + } +} + +func TestTheControlNodeIsSentLastInAStep(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklogOnThree(t) + ctx := t.Context() + register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) + if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { + t.Fatal(err) + } + said := events(t) + gateEvery = 0 + for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { + advancePlans(ctx, b.open) + } + if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { + t.Fatalf("the order is %v", p.Release.Order) + } + laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") + if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { + t.Fatalf("the control node is not sent last in the rest's step: %v", *said) + } +} + +func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { + withTheGate(t) + judgeLeavesAsTheCore(t) + b := aBacklog(t) + ctx := t.Context() + inv := b.open.inventory + reporting := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing + b.sent = append(b.sent, names) + current, _ := open.inventory.CurrentBuilds(ctx) + return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", + map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) + } + return reporting(ctx, open, names) + } + t.Cleanup(func() { sendRollout = reporting }) + gateEvery = 0 + for i := 0; i < 8; i++ { + advancePlans(ctx, b.open) + } + p := b.release(t) + g := p.Release.Gates["laptop"] + if !p.Open() || g == nil || g.Verdict != "" || g.Silent { + t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) + } + since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) + g.Since = &since + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + advancePlans(ctx, b.open) + p = b.release(t) + if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { + t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) + } + if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { + t.Fatal("a silent machine's build was marked failed") + } + if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { + t.Fatalf("laptop was put back: %v", sent) + } + obs, err := silentObservation(ctx, inv) + if err != nil { + t.Fatal(err) + } + if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { + t.Fatalf("the silence is not said: %+v", obs) + } +} diff --git a/cmd/mesh-controller/rest_at_once_test.go b/cmd/mesh-controller/rest_at_once_test.go index 39a0cb55..ea332777 100644 --- a/cmd/mesh-controller/rest_at_once_test.go +++ b/cmd/mesh-controller/rest_at_once_test.go @@ -5,7 +5,6 @@ import ( "slices" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" @@ -20,27 +19,6 @@ import ( // had just applied, and judged it from its first reading. // aBacklogOnThree is the backlog fixture with a third machine, desk, running app at c1 like the others. -func aBacklogOnThree(t *testing.T) *backlogMesh { - t.Helper() - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - if _, err := inv.AddNode(ctx, "desk"); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, "desk", "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, b.open, "desk"), "d-desk", map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - wasHeard := releaseHeard - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { - return map[string]bool{"anchor": true, "laptop": true, "desk": true}, nil - } - t.Cleanup(func() { releaseHeard = wasHeard }) - return b -} // events records every send and every judging, in order. func events(t *testing.T) *[]string { @@ -61,177 +39,18 @@ func events(t *testing.T) *[]string { // Once the first machine passed, the rest are both sent before either is judged (ADR 0353 §1); before, the second // was judged to its verdict before the third was sent. -func TestAReleaseSendsTheRestAtOnceAfterTheFirstPasses(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - said := events(t) - gateEvery = 0 - for i := 0; i < 12; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanDone || len(p.Release.Done) != 3 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - desk, laptop := slices.Index(*said, "send desk"), slices.Index(*said, "send laptop") - if desk < 0 || laptop < 0 || slices.Index(*said, "send anchor") > min(desk, laptop) { - t.Fatalf("sends and judgings: %v", *said) - } - if slices.Contains((*said)[min(desk, laptop):max(desk, laptop)], "judge") { - t.Fatalf("the rest were sent one at a time, each judged before the next: %v", *said) - } - if v, found, err := b.open.inventory.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } -} // A machine of the rest whose gate fails is put back at once, and the other machine of the step goes on to its own // verdict (ADR 0353 §3); before, the walk stopped at the failure and the machine after it was never sent. -func TestARestMachineThatFailsIsPutBackAloneAndTheOthersAreJudgedOn(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - inv := b.open.inventory - gateEvery = 0 - b.broken["desk"] = true // desk fails what it is sent in the rest's step - advancePlans(ctx, b.open) - for i := 0; i < 12 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Failed, []string{"desk"}) { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if !slices.ContainsFunc(b.sent, func(s []string) bool { return slices.Equal(s, []string{"laptop"}) }) { - t.Fatalf("laptop was never sent: %v", b.sent) - } - if !slices.Contains(p.Release.Done, "laptop") { - t.Fatalf("laptop has no verdict of its own: %+v", p.Release) - } - // late waited on laptop alone and was healthy there: its pass is kept, whatever desk did. - if v, found, err := inv.GateOf(ctx, "build-late-c2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("late's pass on laptop was not kept: %+v %v %v", v, found, err) - } - // app failed on desk: marked, and put back wherever it was sent (ADR 0236). - if failed, _ := inv.GateFailed(ctx, "build-app-c2"); !failed { - t.Fatal("app's build is not marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "desk"); sent["app"] != "c1" { - t.Fatalf("desk was not put back: %v", sent) - } -} // A walk whose first machine already runs its build under another walk's gate sends nothing and goes on from that // gate's readings (ADR 0353 §7); before, it sent the machine again and judged from its first reading. -func TestAWalkTakesOverTheReadingsOfAGateJudgingTheSameBuildThere(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - advancePlans(ctx, b.open) // the release sends anchor app c2, and reads it once - release := b.release(t) - judging := release.Release.Gate - if judging == nil || judging.Pending || judging.Passes == 0 { - t.Fatalf("the release is not judging anchor: %+v", judging) - } - built := time.Now().UTC() - plan := inventory.Plan{ID: "plan-app", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {State: "built", BuiltAt: &built, Commit: "c2", - Build: "build-app-c2"}}} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again what it runs: %v", b.sent) - } - p, err := inv.PlanByID(ctx, "plan-app") - if err != nil { - t.Fatal(err) - } - g := p.Modules["app"].Gate - if g == nil || g.Since == nil || !g.Since.Equal(*judging.Since) || g.Passes < judging.Passes || g.TookOver != release.ID { - t.Fatalf("the walk's gate starts from zero: %+v (the release's: %+v)", g, judging) - } - if p.Modules["app"].Previous != "c1" { - t.Fatalf("what anchor ran before is %q, not c1", p.Modules["app"].Previous) - } -} // A walk's first send that was made and not kept — the controller replaced between the two — is not made again // by the controller that takes over: the gate counts from that send (ADR 0353 §5, §6). -func TestAFirstSendMadeAndNotKeptIsNotSentAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - // The old controller sends anchor app c2 and is replaced before it keeps the send. - made := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if _, err := made(ctx, open, names); err != nil { - return nil, err - } - return nil, context.Canceled - } - advancePlans(ctx, g.open) - sendRollout = made - if len(g.sent) != 1 || g.plan(t).Modules["app"].FirstAt != nil { - t.Fatalf("the fixture's send was not made, or was kept: %v", g.sent) - } - last, _, err := g.open.inventory.SentTo(ctx, "anchor") - if err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if len(g.sent) != 1 { - t.Fatalf("the send was made again: %v", g.sent) - } - s := g.plan(t).Modules["app"] - if s.FirstAt == nil || s.Gate == nil || s.Gate.Since == nil || !s.Gate.Since.Equal(last.At) { - t.Fatalf("the gate does not count from the send made: %+v %+v", s, s.Gate) - } - // What anchor ran before was kept before the send, so a failure still puts it back (§5). - if s.Previous != "c1" || s.Gate.From != "c1" { - t.Fatalf("what anchor ran before is %q (gate from %q), not c1", s.Previous, s.Gate.From) - } -} // A release's send kept before it was made, and made, is judged by the controller that takes over without being // made again (ADR 0353 §8). -func TestAReleaseSendKeptAndMadeIsNotMadeAgain(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - p, err := releaseBacklog(ctx, b.open, "") - if err != nil || p == nil { - t.Fatalf("no release opened: %v", err) - } - if _, err := advanceRelease(ctx, b.open, p); err != nil || p.Release.Gate == nil || !p.Release.Gate.Pending { - t.Fatalf("the send was not kept before it was made: %+v %v", p.Release, err) - } - if err := inv.SavePlan(ctx, p); err != nil { - t.Fatal(err) - } - // The send is made, and the controller is replaced before it keeps it. - if _, err := sendRollout(ctx, b.open, []string{"anchor"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - if len(b.sent) != 1 { - t.Fatalf("anchor was sent again: %v", b.sent) - } - g := b.release(t).Release.Gate - if g == nil || g.Pending || g.Since == nil || len(g.Carried) == 0 { - t.Fatalf("anchor is not judged on the send made: %+v", g) - } -} // A build no gate has passed goes to one machine of a step, its first; a machine that would be its second waits // for the next step, and one whose every build passed goes now (ADR 0353 §2). @@ -283,100 +102,9 @@ func TestTheNodeEngineAndTheResolverGoToTheRestOneAtATime(t *testing.T) { } // A first machine that fails still stops the walk (ADR 0236 §5, kept by ADR 0353 §3): the rest are never sent. -func TestAFirstMachineThatFailsStillStopsTheWalk(t *testing.T) { - withTheGate(t) - b := aBacklogOnThree(t) - ctx := t.Context() - gateEvery = 0 - b.broken["anchor"] = true - advancePlans(ctx, b.open) - for i := 0; i < 6 && b.release(t).Open(); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); p.State != inventory.PlanFailed || p.Release.Rest { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - for _, s := range b.sent { - if !slices.Equal(s, []string{"anchor"}) { - t.Fatalf("sent %v: the rest after a failed first machine", b.sent) - } - } -} // The control node is sent last in a step of the rest (ADR 0353 §1): the machine holding the controller is // ordered after every other. -func TestTheControlNodeIsSentLastInAStep(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklogOnThree(t) - ctx := t.Context() - register(t, b.open, catalogue.Manifest{Module: catalogue.ControllerSeatName, Version: "1"}) - if _, err := b.open.inventory.Assign(ctx, "anchor", catalogue.ControllerSeatName); err != nil { - t.Fatal(err) - } - said := events(t) - gateEvery = 0 - for i := 0; i < 12 && (len(b.sent) == 0 || b.release(t).Open()); i++ { - advancePlans(ctx, b.open) - } - if p := b.release(t); !slices.Equal(p.Release.Order, []string{"desk", "laptop", "anchor"}) { - t.Fatalf("the order is %v", p.Release.Order) - } - laptop, anchor := slices.Index(*said, "send laptop"), slices.Index(*said, "send anchor") - if laptop < 0 || anchor < laptop || slices.Contains((*said)[laptop:anchor], "judge") { - t.Fatalf("the control node is not sent last in the rest's step: %v", *said) - } -} // A machine of the rest that says nothing on its send is left silent after fifteen minutes: not failed, not put // back; a rest that went silent whole ends failed, said as a condition (ADR 0353 §4). -func TestARestMachineSilentPastItsBoundIsLeftAndSaid(t *testing.T) { - withTheGate(t) - judgeLeavesAsTheCore(t) - b := aBacklog(t) - ctx := t.Context() - inv := b.open.inventory - reporting := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - if slices.Equal(names, []string{"laptop"}) { // laptop sleeps: sent, and says nothing - b.sent = append(b.sent, names) - current, _ := open.inventory.CurrentBuilds(ctx) - return names, open.inventory.RecordSent(ctx, nodeID(t, open, "laptop"), "d-laptop-asleep", - map[string]string{"app": current["app"].Commit, "late": current["late"].Commit, "same": current["same"].Commit}) - } - return reporting(ctx, open, names) - } - t.Cleanup(func() { sendRollout = reporting }) - gateEvery = 0 - for i := 0; i < 8; i++ { - advancePlans(ctx, b.open) - } - p := b.release(t) - g := p.Release.Gates["laptop"] - if !p.Open() || g == nil || g.Verdict != "" || g.Silent { - t.Fatalf("laptop is judged before fifteen minutes: %s %s %+v", p.State, p.Note, g) - } - since := time.Now().Add(-inventory.ApplySilentAfter - time.Minute) - g.Since = &since - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - advancePlans(ctx, b.open) - p = b.release(t) - if p.State != inventory.PlanFailed || !slices.Equal(p.Release.Silent, []string{"laptop"}) || len(p.Release.Failed) > 0 { - t.Fatalf("the release is %s: %s %+v", p.State, p.Note, p.Release) - } - if failed, _ := inv.GateFailed(ctx, "build-late-c2"); failed { - t.Fatal("a silent machine's build was marked failed") - } - if sent, _, _ := inv.SentBuilds(ctx, "laptop"); sent["late"] != "c2" { - t.Fatalf("laptop was put back: %v", sent) - } - obs, err := silentObservation(ctx, inv) - if err != nil { - t.Fatal(err) - } - if len(obs) != 1 || obs[0].Kind != "release-silent" || !slices.Equal(obs[0].Also, []string{"laptop"}) { - t.Fatalf("the silence is not said: %+v", obs) - } -} diff --git a/cmd/mesh-controller/retirement_integration_test.go b/cmd/mesh-controller/retirement_integration_test.go new file mode 100644 index 00000000..62c80b86 --- /dev/null +++ b/cmd/mesh-controller/retirement_integration_test.go @@ -0,0 +1,191 @@ +//go:build integration + +package main + +import ( + "slices" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "pg", Version: "1", + Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}}) + if _, err := assign(ctx, open, "anchor", "pg"); err != nil { + t.Fatal(err) + } + r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }} + for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} { + if err := r.Retired(ctx, w); err != nil { + t.Fatal(err) + } + } + if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{ + cleanupObservation(providerInstance{Node: "anchor", Module: "gone"}, + []link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()), + }); err != nil { + t.Fatal(err) + } + all, _ := conditionsFrom.Open(ctx) + if len(all) != 3 { + t.Fatalf("%+v", all) + } + if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil { + t.Fatal(err) + } + left := openKeys(t, conditionsFrom) + if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" { + t.Fatalf("only the assigned provider's waiting should stay: %+v", left) + } +} + +func onATestBus(t *testing.T) *nats.Conn { + t.Helper() + url := testbus.URL(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + return js.Conn() +} + +func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) { + conn := onATestBus(t) + fake := &fakeProvider{} + fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6} + fake.serve(t, conn, "pg-approve", "anchor") + p := providerInstance{Node: "anchor", Module: "pg-approve"} + said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) }) + if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") { + t.Fatalf("approved %v; said %s", fake.approved, said) + } + acts := handActsBy(t, conn, "retire approve") + if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting || + acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" { + t.Fatalf("the approval is not in the hand-act log: %+v", acts) + } + + // Reject, on another provider: the same set, and nothing approved. + other := &fakeProvider{state: fake.state} + other.serve(t, conn, "pg-reject", "anchor") + printed(t, func() error { + return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false, + whyFlags(t, "still moving")) + }) + if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil { + t.Fatalf("rejected %v approved %v", other.rejected, other.approved) + } + + // Nothing waiting: refused, nothing asked but the question. + idle := &fakeProvider{} + idle.serve(t, conn, "pg-idle", "anchor") + if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true, + whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") { + t.Fatalf("%v", err) + } + if len(idle.asked) != 1 { + t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked) + } +} + +func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { + conn := onATestBus(t) + fake := &fakeProvider{} + fake.state.Held = []string{"active"} + fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)} + fake.serve(t, conn, "pg-clean", "anchor") + p := providerInstance{Node: "anchor", Module: "pg-clean"} + said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) }) + if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") { + t.Fatalf("deleted %v; said %s", fake.deleted, said) + } + // An active consumer, or one it does not hold, is refused before the provider is asked to delete. + for _, name := range []string{"active", "nobody"} { + if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil || + !strings.Contains(err.Error(), "only a retired consumer is deleted") { + t.Fatalf("%s: %v", name, err) + } + } + if !slices.Equal(fake.deleted, []string{"old"}) { + t.Fatalf("more was deleted: %v", fake.deleted) + } + if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" { + t.Fatalf("the deletion is not in the hand-act log: %+v", acts) + } + + // Older than: listed, and nothing deleted without confirm; with it, only the old one. + listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) + if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 { + t.Fatalf("%+v", listing) + } + fake.deleted = nil + said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) }) + if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || + strings.Contains(said, "young") { + t.Fatalf("deleted %v; said %s", fake.deleted, said) + } + printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) }) + if !slices.Equal(fake.deleted, []string{"old"}) { + t.Fatalf("confirmed, deleted %v", fake.deleted) + } +} + +func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) { + conn := onATestBus(t) + old := &fakeProvider{} + old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)} + old.serve(t, conn, "pg-d11-old", "anchor") + young := &fakeProvider{} + young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)} + young.serve(t, conn, "pg-d11-young", "anchor") + instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"}, + // Nothing serves this one: a provider older than the question is passed over, not a failure. + {Node: "anchor", Module: "pg-d11-predates"}} + found, err := retiredTooLong(t.Context(), conn, instances, time.Now()) + if err != nil { + t.Fatal(err) + } + if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting || + found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") || + strings.Contains(found[0].Summary, "fresh") { + t.Fatalf("%+v", found) + } +} + +func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) { + conn := onATestBus(t) + fake := &fakeProvider{} + kept := retiredDaysAgo("ledger", 3) + kept.Access = "kept" + fake.state.Retired = []link.RetiredConsumer{kept} + fake.state.RetiresBy = "mark-only" + fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2} + fake.serve(t, conn, "vault-mark", "anchor") + p := providerInstance{Node: "anchor", Module: "vault-mark"} + listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) + said := printed(t, func() error { return printRetired(listing, false) }) + if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" { + t.Fatalf("%s %+v", said, listing) + } + said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) }) + if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") { + t.Fatal(said) + } +} diff --git a/cmd/mesh-controller/retirement_test.go b/cmd/mesh-controller/retirement_test.go index 0402eaab..7d6a17c6 100644 --- a/cmd/mesh-controller/retirement_test.go +++ b/cmd/mesh-controller/retirement_test.go @@ -13,12 +13,8 @@ import ( "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" - - "github.com/novox/mesh-controller/internal/testbus" ) // A consumer the mesh stops asking for is retired, not withdrawn, and deleted only by a person @@ -151,39 +147,6 @@ func TestARetirementWordIsKeptAsItsConditionNamingTheEmitterFromTheSubject(t *te } } -func TestARetirementConditionOfAnUnassignedProviderClears(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "pg", Version: "1", - Receives: map[string]string{"postgres-database": "/var/lib/mesh/pg/mesh.json"}}) - if _, err := assign(ctx, open, "anchor", "pg"); err != nil { - t.Fatal(err) - } - r := retirements{keeper: func() *conditions.Keeper { return conditionsFrom }} - for _, w := range []link.Retirement{waitingWord("pg", "anchor", "a", "b", "c", "d"), waitingWord("gone", "anchor", "a", "b", "c", "d")} { - if err := r.Retired(ctx, w); err != nil { - t.Fatal(err) - } - } - if err := conditionsFrom.Reconcile(ctx, "D11", []conditions.Observation{ - cleanupObservation(providerInstance{Node: "anchor", Module: "gone"}, - []link.RetiredConsumer{{Consumer: "a", RetiredAt: time.Now().Add(-40 * 24 * time.Hour).Format(time.RFC3339)}}, time.Now()), - }); err != nil { - t.Fatal(err) - } - all, _ := conditionsFrom.Open(ctx) - if len(all) != 3 { - t.Fatalf("%+v", all) - } - if err := unassignedProviders(ctx, open.inventory, conditionsFrom, providerConditions(all)); err != nil { - t.Fatal(err) - } - left := openKeys(t, conditionsFrom) - if len(left) != 1 || left["provider.pg.anchor.retire"].Key == "" { - t.Fatalf("only the assigned provider's waiting should stay: %+v", left) - } -} - // fakeProvider answers the retirement tools on one machine, over a real bus, keeping what it was asked. type fakeProvider struct { mu sync.Mutex @@ -260,23 +223,6 @@ func (f *fakeProvider) serve(t *testing.T, conn *nats.Conn, module, node string) } } -func onATestBus(t *testing.T) *nats.Conn { - t.Helper() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - return js.Conn() -} - func whyFlags(t *testing.T, why string) handActFlags { t.Helper() set := flag.NewFlagSet("t", flag.ContinueOnError) @@ -308,109 +254,6 @@ func retiredDaysAgo(name string, days int) link.RetiredConsumer { RetiredAt: time.Now().Add(-time.Duration(days) * 24 * time.Hour).UTC().Format(time.RFC3339), SizeBytes: &size} } -func TestNatsApproveSendsTheExactSetTheProviderWaitsWith(t *testing.T) { - conn := onATestBus(t) - fake := &fakeProvider{} - fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "d"}, {Consumer: "b"}, {Consumer: "a"}, {Consumer: "c"}}, Held: 6} - fake.serve(t, conn, "pg-approve", "anchor") - p := providerInstance{Node: "anchor", Module: "pg-approve"} - said := printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "moved them")) }) - if !slices.Equal(fake.approved, []string{"a", "b", "c", "d"}) || !strings.Contains(said, "retired d, b, a, c") { - t.Fatalf("approved %v; said %s", fake.approved, said) - } - acts := handActsBy(t, conn, "retire approve") - if len(acts) == 0 || acts[len(acts)-1].Cause != kindRetireWaiting || - acts[len(acts)-1].Condition != "provider.pg-approve.anchor.retire" { - t.Fatalf("the approval is not in the hand-act log: %+v", acts) - } - - // Reject, on another provider: the same set, and nothing approved. - other := &fakeProvider{state: fake.state} - other.serve(t, conn, "pg-reject", "anchor") - printed(t, func() error { - return answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-reject"}, false, - whyFlags(t, "still moving")) - }) - if !slices.Equal(other.rejected, []string{"a", "b", "c", "d"}) || other.approved != nil { - t.Fatalf("rejected %v approved %v", other.rejected, other.approved) - } - - // Nothing waiting: refused, nothing asked but the question. - idle := &fakeProvider{} - idle.serve(t, conn, "pg-idle", "anchor") - if err := answerRetirement(t.Context(), conn, providerInstance{Node: "anchor", Module: "pg-idle"}, true, - whyFlags(t, "x")); err == nil || !strings.Contains(err.Error(), "waits for nobody") { - t.Fatalf("%v", err) - } - if len(idle.asked) != 1 { - t.Fatalf("an idle provider was asked more than its state: %+v", idle.asked) - } -} - -func TestNatsCleanupDeletesOnlyTheNamedRetiredConsumer(t *testing.T) { - conn := onATestBus(t) - fake := &fakeProvider{} - fake.state.Held = []string{"active"} - fake.state.Retired = []link.RetiredConsumer{retiredDaysAgo("old", 40), retiredDaysAgo("young", 2)} - fake.serve(t, conn, "pg-clean", "anchor") - p := providerInstance{Node: "anchor", Module: "pg-clean"} - said := printed(t, func() error { return deleteRetired(t.Context(), conn, p, "old", whyFlags(t, "not needed")) }) - if !slices.Equal(fake.deleted, []string{"old"}) || !strings.Contains(said, "deleted old (1.0 KB freed)") { - t.Fatalf("deleted %v; said %s", fake.deleted, said) - } - // An active consumer, or one it does not hold, is refused before the provider is asked to delete. - for _, name := range []string{"active", "nobody"} { - if err := deleteRetired(t.Context(), conn, p, name, whyFlags(t, "x")); err == nil || - !strings.Contains(err.Error(), "only a retired consumer is deleted") { - t.Fatalf("%s: %v", name, err) - } - } - if !slices.Equal(fake.deleted, []string{"old"}) { - t.Fatalf("more was deleted: %v", fake.deleted) - } - if acts := handActsBy(t, conn, "cleanup delete"); len(acts) == 0 || acts[len(acts)-1].Args[2] != "old" { - t.Fatalf("the deletion is not in the hand-act log: %+v", acts) - } - - // Older than: listed, and nothing deleted without confirm; with it, only the old one. - listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) - if len(listing.Retired) != 2 || listing.Retired[0].Consumer != "old" || listing.Retired[0].AgeDays != 40 { - t.Fatalf("%+v", listing) - } - fake.deleted = nil - said = printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, false, whyFlags(t, "tidy")) }) - if fake.deleted != nil || !strings.Contains(said, "nothing was deleted: add --confirm") || !strings.Contains(said, "old") || - strings.Contains(said, "young") { - t.Fatalf("deleted %v; said %s", fake.deleted, said) - } - printed(t, func() error { return deleteFrom(t.Context(), conn, nil, listing, 30, true, whyFlags(t, "tidy")) }) - if !slices.Equal(fake.deleted, []string{"old"}) { - t.Fatalf("confirmed, deleted %v", fake.deleted) - } -} - -func TestNatsD11SaysCleanupWaitsAfterThirtyDays(t *testing.T) { - conn := onATestBus(t) - old := &fakeProvider{} - old.state.Retired = []link.RetiredConsumer{retiredDaysAgo("mesh_a_letta", 31), retiredDaysAgo("fresh", 1)} - old.serve(t, conn, "pg-d11-old", "anchor") - young := &fakeProvider{} - young.state.Retired = []link.RetiredConsumer{retiredDaysAgo("x", 29)} - young.serve(t, conn, "pg-d11-young", "anchor") - instances := []providerInstance{{Node: "anchor", Module: "pg-d11-old"}, {Node: "anchor", Module: "pg-d11-young"}, - // Nothing serves this one: a provider older than the question is passed over, not a failure. - {Node: "anchor", Module: "pg-d11-predates"}} - found, err := retiredTooLong(t.Context(), conn, instances, time.Now()) - if err != nil { - t.Fatal(err) - } - if len(found) != 1 || found[0].Key() != "provider.pg-d11-old.anchor.cleanup" || found[0].Kind != kindCleanupWaiting || - found[0].Severity != conditions.Warning || !strings.Contains(found[0].Summary, "mesh_a_letta (31 days") || - strings.Contains(found[0].Summary, "fresh") { - t.Fatalf("%+v", found) - } -} - func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) { for _, c := range []struct { verb string @@ -459,23 +302,3 @@ func TestTheRetireAndCleanupVerbsComposeTheirCommandLines(t *testing.T) { // A mark-only provider's retired consumer keeps its access; the listing and the approval say so // rather than claiming it was disabled (ADR 0230). -func TestNatsAMarkOnlyRetirementIsSaidAsSuch(t *testing.T) { - conn := onATestBus(t) - fake := &fakeProvider{} - kept := retiredDaysAgo("ledger", 3) - kept.Access = "kept" - fake.state.Retired = []link.RetiredConsumer{kept} - fake.state.RetiresBy = "mark-only" - fake.state.Waiting = &link.RetirementWaiting{Consumers: []link.RetiredConsumer{{Consumer: "a"}, {Consumer: "b"}}, Held: 2} - fake.serve(t, conn, "vault-mark", "anchor") - p := providerInstance{Node: "anchor", Module: "vault-mark"} - listing := retiredOf(t.Context(), conn, []providerInstance{p}, time.Now()) - said := printed(t, func() error { return printRetired(listing, false) }) - if !strings.Contains(said, "MARK ONLY") || listing.Retired[0].Access != "kept" { - t.Fatalf("%s %+v", said, listing) - } - said = printed(t, func() error { return answerRetirement(t.Context(), conn, p, true, whyFlags(t, "gone")) }) - if !strings.Contains(said, "MARK ONLY") || strings.Contains(said, "access disabled") { - t.Fatal(said) - } -} diff --git a/cmd/mesh-controller/rollout_events_test.go b/cmd/mesh-controller/rollout_events_test.go index f3809bba..a90d359c 100644 --- a/cmd/mesh-controller/rollout_events_test.go +++ b/cmd/mesh-controller/rollout_events_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/rollout_silent_bus_integration_test.go b/cmd/mesh-controller/rollout_silent_bus_integration_test.go new file mode 100644 index 00000000..e3de4c3b --- /dev/null +++ b/cmd/mesh-controller/rollout_silent_bus_integration_test.go @@ -0,0 +1,92 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + api, err := jetstream.New(conn) + if err != nil { + t.Fatal(err) + } + heard := make(chan silentWait, 16) + ctx, stop := context.WithCancel(context.Background()) + t.Cleanup(stop) + s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) + if err != nil { + t.Fatal(err) + } + return s, heard +} + +func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { + s, heard := silentOnABus(t) + want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} + if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { + t.Fatalf("heard %+v (%v); want %+v", got, ok, want) + } +} + +func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { + s, heard := silentOnABus(t) + ctx := t.Context() + w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} + if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if err := s.Reported(ctx, w); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { + t.Fatalf("%+v was named silent after it reported", got) + } + // Reporting twice, or on a send that put no key, is no error. + if err := s.Reported(ctx, w); err != nil { + t.Fatalf("a second report: %v", err) + } +} + +func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { + s, heard := silentOnABus(t) + ctx := t.Context() + w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} + if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { + t.Fatal(err) + } + time.Sleep(1200 * time.Millisecond) + if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { + t.Fatal(err) + } + if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { + t.Fatalf("%+v expired at the first send's time, not the second's", got) + } + if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { + t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) + } +} diff --git a/cmd/mesh-controller/rollout_silent_bus_test.go b/cmd/mesh-controller/rollout_silent_bus_test.go index fef01437..aaf3aa85 100644 --- a/cmd/mesh-controller/rollout_silent_bus_test.go +++ b/cmd/mesh-controller/rollout_silent_bus_test.go @@ -1,46 +1,11 @@ package main import ( - "context" "testing" "time" - - "github.com/nats-io/nats.go" - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // silentOnABus is the rollout's silent keys on a real bus, and the waits its watcher heard expire. -func silentOnABus(t *testing.T) (*busSilent, <-chan silentWait) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - api, err := jetstream.New(conn) - if err != nil { - t.Fatal(err) - } - heard := make(chan silentWait, 16) - ctx, stop := context.WithCancel(context.Background()) - t.Cleanup(stop) - s, err := newBusSilent(ctx, api, func(w silentWait) { heard <- w }) - if err != nil { - t.Fatal(err) - } - return s, heard -} func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (silentWait, bool) { t.Helper() @@ -54,56 +19,10 @@ func hearsSilent(t *testing.T, heard <-chan silentWait, within time.Duration) (s // A machine that never reports on its send is named silent when its key expires (issue 974, novox/hq ADR 0406 §5): // the watcher names the plan, the tier and the machine, which a plan id or a machine name with a dot does not blur. -func TestAKeyLeftAloneExpiresAndTheWatcherNamesTheMachineSilent(t *testing.T) { - s, heard := silentOnABus(t) - want := silentWait{Plan: "plan.7", Tier: 2, Node: "ace"} - if err := s.Arm(t.Context(), want, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 10*time.Second); !ok || got != want { - t.Fatalf("heard %+v (%v); want %+v", got, ok, want) - } -} // A machine that reports in time is never named silent: the report deletes the key, and a delete is not an expiry. -func TestAReportDeletesTheKeyAndNoOneIsNamedSilent(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-1", Tier: 0, Node: "g14"} - if err := s.Arm(ctx, w, time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if err := s.Reported(ctx, w); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 2500*time.Millisecond); ok { - t.Fatalf("%+v was named silent after it reported", got) - } - // Reporting twice, or on a send that put no key, is no error. - if err := s.Reported(ctx, w); err != nil { - t.Fatalf("a second report: %v", err) - } -} // A send made again to a machine puts its key again with the full time, and only the last send's key can expire. -func TestASendMadeAgainPutsTheKeyAgain(t *testing.T) { - s, heard := silentOnABus(t) - ctx := t.Context() - w := silentWait{Plan: "plan-3", Tier: 1, Node: "novox"} - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - time.Sleep(1200 * time.Millisecond) - if err := s.Arm(ctx, w, 2*time.Second, time.Now()); err != nil { - t.Fatal(err) - } - if got, ok := hearsSilent(t, heard, 1500*time.Millisecond); ok { - t.Fatalf("%+v expired at the first send's time, not the second's", got) - } - if got, ok := hearsSilent(t, heard, 5*time.Second); !ok || got != w { - t.Fatalf("heard %+v (%v); want %+v after the second send's time", got, ok, w) - } -} func TestASilentKeyIsParsedFromTheRightOfTheNode(t *testing.T) { for _, w := range []silentWait{{"plan-1", 0, "ace"}, {"a.b", 12, "g14"}} { diff --git a/cmd/mesh-controller/rollout_test.go b/cmd/mesh-controller/rollout_test.go index ba5ac2f5..409eb28e 100644 --- a/cmd/mesh-controller/rollout_test.go +++ b/cmd/mesh-controller/rollout_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/roster_failure_test.go b/cmd/mesh-controller/roster_failure_test.go index 2aa035d6..2479f38c 100644 --- a/cmd/mesh-controller/roster_failure_test.go +++ b/cmd/mesh-controller/roster_failure_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/runtime_health_test.go b/cmd/mesh-controller/runtime_health_test.go index 8bc63491..3856441c 100644 --- a/cmd/mesh-controller/runtime_health_test.go +++ b/cmd/mesh-controller/runtime_health_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/same_program.go b/cmd/mesh-controller/same_program.go index 20b7598d..6c79c26b 100644 --- a/cmd/mesh-controller/same_program.go +++ b/cmd/mesh-controller/same_program.go @@ -157,7 +157,7 @@ func withDependentsOf(same map[string]string, plan *inventory.Plan, moved []stri var bases []string for _, e := range edges { ordering := e.Kind == inventory.EdgeBuiltBy || e.Kind == inventory.EdgeWorkerOf || - e.Kind == inventory.EdgePackages || e.Kind == edgeGroupOrder + e.Kind == inventory.EdgePackages || e.Kind == edgeChangeOrder if !ordering && e.From == n && same[e.To] != "" && !slices.Contains(bases, e.To) { bases = append(bases, e.To) } diff --git a/cmd/mesh-controller/same_program_integration_test.go b/cmd/mesh-controller/same_program_integration_test.go new file mode 100644 index 00000000..294ae773 --- /dev/null +++ b/cmd/mesh-controller/same_program_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { + open := windowed(t) + endsItsLine(t) + asksRecorded(t) + ctx := t.Context() + inv := open.inventory + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", + Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), + Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}, + Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { + t.Fatal(err) + } + // What runs: the module registered with that build's image (RegisteredHeld reads the module table). + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ + {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, + inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { + t.Fatal(err) + } + pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7} + was := checkedBuildsOf + checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { + return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k", + Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}}, + Same: "unchanged: its program is the same as running build one aaaa", + SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil + } + t.Cleanup(func() { checkedBuildsOf = was }) + code := repoMerge("one", "one1aaaaaaaa", t0) + code.Number, code.HeadSHA = 7, "headaaaaaaaa1111" + hear(t, open, code, t0.Add(time.Second)) + cutAt(t, open, t0.Add(2*time.Minute)) + ws, _ := walks(t, open) + if len(ws) != 1 { + t.Fatalf("walks: %+v", ws) + } + w := ws[0] + if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { + t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) + } + if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") { + t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) + } +} diff --git a/cmd/mesh-controller/same_program_test.go b/cmd/mesh-controller/same_program_test.go index 2d47d6bd..11afe41d 100644 --- a/cmd/mesh-controller/same_program_test.go +++ b/cmd/mesh-controller/same_program_test.go @@ -1,12 +1,9 @@ package main import ( - "context" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -96,46 +93,3 @@ func TestADependentOfABaseThatCameOutTheSameIsLeftOut(t *testing.T) { // Against the store, the path a real merge takes (novox/hq issues 721 and 611): a merge whose check built its one // module the same as the build the mesh holds is cut into a walk done with nothing to walk, whose record names the // module unchanged and why — the walk mesh-delivery reads as delivered, never stranded. -func TestAWalkWhoseEveryBuildCameOutTheSameIsDone(t *testing.T) { - open := windowed(t) - endsItsLine(t) - asksRecorded(t) - ctx := t.Context() - inv := open.inventory - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-one", Repository: "novox/one", Ref: "main", - Module: "one", Commit: "c0", On: "builder", Asked: t0.Add(-time.Hour), - Made: []inventory.Artifact{{Name: "one", Kind: "image", Reference: "store/one@sha256:aaaa"}}, - Sources: []inventory.BuildSource{{Paths: []string{"main.go", "go.mod"}}}}); err != nil { - t.Fatal(err) - } - // What runs: the module registered with that build's image (RegisteredHeld reads the module table). - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": "store/one@sha256:aaaa"}}}, - inventory.Source{Repository: "novox/one", Ref: "main", BuiltFrom: "c0", Asked: time.Now()}); err != nil { - t.Fatal(err) - } - pull := link.PullRef{Owner: "novox", Repo: "one", Number: 7} - was := checkedBuildsOf - checkedBuildsOf = func(context.Context, []link.PullRef) ([]link.CheckedBuild, error) { - return []link.CheckedBuild{{Pull: pull, Head: "headaaaaaaaa", Built: link.CheckBuilt{Module: "one", Key: "k", - Made: []link.MadeArtifact{{Name: "one", Kind: "image", Reference: "scratch/one@sha256:bbbb"}}, - Same: "unchanged: its program is the same as running build one aaaa", - SameAs: map[string]string{"one": "10.0.0.1:5000/one@sha256:aaaa"}}}}, nil - } - t.Cleanup(func() { checkedBuildsOf = was }) - code := repoMerge("one", "one1aaaaaaaa", t0) - code.Number, code.HeadSHA = 7, "headaaaaaaaa1111" - hear(t, open, code, t0.Add(time.Second)) - cutAt(t, open, t0.Add(2*time.Minute)) - ws, _ := walks(t, open) - if len(ws) != 1 { - t.Fatalf("walks: %+v", ws) - } - w := ws[0] - if w.State != inventory.PlanDone || len(w.Tiers) != 0 || len(w.Modules) != 0 { - t.Fatalf("the walk reads %s, tiers %v, modules %v; want done with nothing to walk", w.State, w.Tiers, w.Modules) - } - if w.Delivery == nil || !strings.Contains(w.Delivery.Unchanged["one"], "its program is the same as running build") { - t.Fatalf("the walk does not say one matched what runs: %+v", w.Delivery) - } -} diff --git a/cmd/mesh-controller/same_source_test.go b/cmd/mesh-controller/same_source_test.go index 01abd454..695e2244 100644 --- a/cmd/mesh-controller/same_source_test.go +++ b/cmd/mesh-controller/same_source_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/sayable_integration_test.go b/cmd/mesh-controller/sayable_integration_test.go new file mode 100644 index 00000000..508cf740 --- /dev/null +++ b/cmd/mesh-controller/sayable_integration_test.go @@ -0,0 +1,150 @@ +//go:build integration + +package main + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "sync/atomic" + "testing" + + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/outward" +) + +func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) { + var unanswered, broken atomic.Bool + held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong", + Machine: "anchor", Severity: conditions.Urgent, Confirm: true, + Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"} + withProbes(t, + probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if unanswered.Load() { + return []conditions.Observation{held}, nil + } + return nil, nil + }}, + probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1, + run: func(context.Context, *doctor) ([]conditions.Observation, error) { + if broken.Load() { + return nil, fmt.Errorf("a question timed out") + } + return nil, nil + }}, + ) + store := conditions.NewInMemory() + k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) + defer k.Close(context.Background()) + d := &doctor{keeper: k, host: "anchor"} + keys := func() []string { + open, err := k.Open(t.Context()) + if err != nil { + t.Fatal(err) + } + var out []string + for _, c := range open { + out = append(out, c.Key) + } + sort.Strings(out) + return out + } + + unanswered.Store(true) + broken.Store(true) + run := d.runOnce(t.Context(), "a test") + if open := keys(); len(open) != 0 { + t.Fatalf("one look raised %v", open) + } + if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun { + t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes) + } + + // Seen twice in a row: raised, and so is the probe that could not run twice. + run = d.runOnce(t.Context(), "a test") + if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" { + t.Fatalf("two looks in a row left open %v", open) + } + if run.Probes[0].Verdict != verdictFail { + t.Fatalf("%+v", run.Probes[0]) + } + + // Open, and seen again: kept, never cleared and raised again. + d.runOnce(t.Context(), "a test") + if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 { + t.Fatalf("the open condition was not kept as it was: %+v", c) + } + + // Answered again: cleared. Then one look alone raises nothing again. + unanswered.Store(false) + broken.Store(false) + d.runOnce(t.Context(), "a test") + if open := keys(); len(open) != 0 { + t.Fatalf("a passing run left open %v", open) + } + unanswered.Store(true) + d.runOnce(t.Context(), "a test") + if open := keys(); len(open) != 0 { + t.Fatalf("one look after a pass raised %v", open) + } +} + +// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277, +// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row +// suppressed past its bound, every probe's finding, every event's condition — is held to the operator +// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's +// shape fails the suite, naming its source. The keeper would say such a summary in words at run time; +// this is where the producer is made to say it rightly in the first place. +func TestMain(m *testing.M) { + // **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test + // binary as a verb's command line, and a binary built with the race detector sleeps a second before it + // exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still + // run under the race detector and still fail on a race they find; they only stop waiting at their end. + // Set before any child starts, and only where nobody chose GORACE. + if _, chosen := os.LookupEnv("GORACE"); !chosen { + _ = os.Setenv("GORACE", "atexit_sleep_ms=0") + } + // The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and + // ends (meshcli_test.go). + // The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does + // (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go). + if want := os.Getenv(secretAcceptWants); want != "" { + os.Exit(readAsSecretAccept(want, os.Args[1:])) + } + if os.Getenv(echoEnvironment) != "" { + fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal()) + if os.Getenv(echoFails) != "" { + os.Exit(3) + } + os.Exit(0) + } + conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { + unsaid.note(o, field, r) + } + conditions.Unworded = func(o conditions.Observation, why string) { + if o.Source != "test" { // a kind a test makes up to exercise the keeper + unworded.add(o, why) + } + } + code := m.Run() + if said := unworded.all(); len(said) > 0 { + fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+ + "words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+ + "reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n ")) + if code == 0 { + code = 1 + } + } + if said := unsaid.all(); len(said) > 0 { + fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+ + "(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n", + len(said), strings.Join(said, "\n ")) + if code == 0 { + code = 1 + } + } + os.Exit(code) +} diff --git a/cmd/mesh-controller/sayable_test.go b/cmd/mesh-controller/sayable_test.go index 8e92dda6..1d51afa6 100644 --- a/cmd/mesh-controller/sayable_test.go +++ b/cmd/mesh-controller/sayable_test.go @@ -1,10 +1,8 @@ package main import ( - "context" "fmt" "net" - "os" "sort" "strings" "sync" @@ -19,63 +17,6 @@ import ( "github.com/novox/mesh-controller/internal/outward" ) -// **Every condition this suite raises says itself in machine names and words** (novox/hq issue 277, -// ADR 0234 §6). Every observation a keeper takes in any test of this package — every signals row -// suppressed past its bound, every probe's finding, every event's condition — is held to the operator -// channel's content rule; one whose summary or key carries an address, a domain, a path or a secret's -// shape fails the suite, naming its source. The keeper would say such a summary in words at run time; -// this is where the producer is made to say it rightly in the first place. -func TestMain(m *testing.M) { - // **A command line this suite runs ends when it is done** (novox/hq issue 591). Many tests run this test - // binary as a verb's command line, and a binary built with the race detector sleeps a second before it - // exits (GORACE's atexit_sleep_ms), which was more than the rest of each such test. The children still - // run under the race detector and still fail on a race they find; they only stop waiting at their end. - // Set before any child starts, and only where nobody chose GORACE. - if _, chosen := os.LookupEnv("GORACE"); !chosen { - _ = os.Setenv("GORACE", "atexit_sleep_ms=0") - } - // The process a mesh-cli test runs as a command line: it says the verb and the caller it was given, and - // ends (meshcli_test.go). - // The process a mesh-cli test runs as `secret accept`: it reads its value exactly as `secret accept` does - // (valueFor) and says whether it is the one the test gave, never the value (meshcli_stdin_test.go). - if want := os.Getenv(secretAcceptWants); want != "" { - os.Exit(readAsSecretAccept(want, os.Args[1:])) - } - if os.Getenv(echoEnvironment) != "" { - fmt.Printf("verb=%q caller=%q terminal=%v\n", os.Getenv("MESH_VERB"), os.Getenv("MESH_CALLER"), startedAtTheTerminal()) - if os.Getenv(echoFails) != "" { - os.Exit(3) - } - os.Exit(0) - } - conditions.Unsayable = func(o conditions.Observation, field string, r outward.Refusal) { - unsaid.note(o, field, r) - } - conditions.Unworded = func(o conditions.Observation, why string) { - if o.Source != "test" { // a kind a test makes up to exercise the keeper - unworded.add(o, why) - } - } - code := m.Run() - if said := unworded.all(); len(said) > 0 { - fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests have no plain words of their own, or "+ - "words that are not plain (novox/hq ADR 0253: a headline, an explanation and a resolved line the operator "+ - "reads at a glance — plain_words.go):\n %s\n", len(said), strings.Join(said, "\n ")) - if code == 0 { - code = 1 - } - } - if said := unsaid.all(); len(said) > 0 { - fmt.Fprintf(os.Stderr, "FAIL: %d condition(s) raised in these tests say what the operator's channel withholds "+ - "(an address, a domain, a path or a secret's shape belongs in the evidence, not the summary):\n %s\n", - len(said), strings.Join(said, "\n ")) - if code == 0 { - code = 1 - } - } - os.Exit(code) -} - // unsaid collects, across the suite, every finding whose words the operator's channel would withhold. var unsaid unsayable @@ -284,79 +225,3 @@ func TestAResolverAnsweringWronglyIsSaidAtOnceInMachineNames(t *testing.T) { // **The self-check raises a finding one look can be wrong about on the second run in a row**, keeps it // open while it is seen, and clears it when it is not — and a probe that cannot run is said as a // condition only when the next run cannot run it either. Neither is ever a pass in the verdict. -func TestASingleLookIsHeldAndTheSecondInARowRaises(t *testing.T) { - var unanswered, broken atomic.Bool - held := conditions.Observation{Scope: conditions.ScopeSeat, ID: "mesh-dns-resolver.anchor", Token: "wrong", - Machine: "anchor", Severity: conditions.Urgent, Confirm: true, - Summary: "the mesh's resolver on anchor does not answer", Said: "no answer from 10.77.0.1"} - withProbes(t, - probe{ID: "P1", Asserts: "asks over the network", Kind: "resolver-wrong", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if unanswered.Load() { - return []conditions.Observation{held}, nil - } - return nil, nil - }}, - probe{ID: "P2", Asserts: "sometimes cannot run", Kind: "x", Phase: 1, - run: func(context.Context, *doctor) ([]conditions.Observation, error) { - if broken.Load() { - return nil, fmt.Errorf("a question timed out") - } - return nil, nil - }}, - ) - store := conditions.NewInMemory() - k := conditions.NewKeeper(t.Context(), conditions.Options{Store: store, History: store}) - defer k.Close(context.Background()) - d := &doctor{keeper: k, host: "anchor"} - keys := func() []string { - open, err := k.Open(t.Context()) - if err != nil { - t.Fatal(err) - } - var out []string - for _, c := range open { - out = append(out, c.Key) - } - sort.Strings(out) - return out - } - - unanswered.Store(true) - broken.Store(true) - run := d.runOnce(t.Context(), "a test") - if open := keys(); len(open) != 0 { - t.Fatalf("one look raised %v", open) - } - if run.Probes[0].Verdict != verdictPass || len(run.Probes[0].Unconfirmed) != 1 || run.Probes[1].Verdict != verdictFailedToRun { - t.Fatalf("the first run's verdict hides what it saw: %+v", run.Probes) - } - - // Seen twice in a row: raised, and so is the probe that could not run twice. - run = d.runOnce(t.Context(), "a test") - if open := keys(); strings.Join(open, " ") != "probe.P2.failed seat.mesh-dns-resolver.anchor.wrong" { - t.Fatalf("two looks in a row left open %v", open) - } - if run.Probes[0].Verdict != verdictFail { - t.Fatalf("%+v", run.Probes[0]) - } - - // Open, and seen again: kept, never cleared and raised again. - d.runOnce(t.Context(), "a test") - if c, open, _ := k.Get(t.Context(), "seat.mesh-dns-resolver.anchor.wrong"); !open || c.Count != 1 || c.Observations != 2 { - t.Fatalf("the open condition was not kept as it was: %+v", c) - } - - // Answered again: cleared. Then one look alone raises nothing again. - unanswered.Store(false) - broken.Store(false) - d.runOnce(t.Context(), "a test") - if open := keys(); len(open) != 0 { - t.Fatalf("a passing run left open %v", open) - } - unanswered.Store(true) - d.runOnce(t.Context(), "a test") - if open := keys(); len(open) != 0 { - t.Fatalf("one look after a pass raised %v", open) - } -} diff --git a/cmd/mesh-controller/seat_dependencies_integration_test.go b/cmd/mesh-controller/seat_dependencies_integration_test.go new file mode 100644 index 00000000..c54f7c41 --- /dev/null +++ b/cmd/mesh-controller/seat_dependencies_integration_test.go @@ -0,0 +1,119 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, serviceManagerHolder()) + register(t, open, packageManagerHolder()) + register(t, open, aDaemon()) + + _, err := assign(ctx, open, "laptop", "sshd") + if err == nil { + t.Fatal("sshd went onto a machine nothing holds the service manager of") + } + for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q:\n%v", want, err) + } + } + assigned, err := open.inventory.Assigned(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if contains(assigned, "sshd") { + t.Fatalf("a refused assignment was kept: %v", assigned) + } + + // The holders depend on each other, so neither goes on alone — and both go on in one act. + if _, err := assign(ctx, open, "laptop", "systemd"); err == nil { + t.Fatal("systemd went on alone though its package needs a package manager") + } + if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil { + t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said) + } + if said, err := assign(ctx, open, "laptop", "sshd"); err != nil { + t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said) + } +} + +func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, serviceManagerHolder()) + register(t, open, packageManagerHolder()) + register(t, open, aDaemon()) + if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil { + t.Fatal(err) + } + + _, err := unassign(ctx, open, "laptop", "systemd") + if err == nil { + t.Fatal("the service manager came off a machine still running services") + } + for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not name %q:\n%v", want, err) + } + } + assigned, _ := open.inventory.Assigned(ctx, "laptop") + if !contains(assigned, "systemd") { + t.Fatalf("a refused unassignment took the module off anyway: %v", assigned) + } + if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil { + t.Fatalf("a dependent could not come off: %v", err) + } +} + +func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) { + // The mesh as it ran before the switch (novox/hq ADR 0207 §4). + defer catalogue.EnforcingSeatDependencies(false)() + open := aMesh(t) + ctx := t.Context() + register(t, open, serviceManagerHolder()) + register(t, open, aDaemon()) + // Assigned straight into the store: a machine whose modules predate the rule, which is every + // machine on the day it ships. + if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil { + t.Fatal(err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if _, refused := asked.refused["laptop"]; refused { + t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"]) + } + if asked.well() { + t.Error("a mesh with an unheld dependency reads as all well") + } + got := printed(t, func() error { return printStatus(asked) }) + for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} { + if !strings.Contains(got, want) { + t.Errorf("status does not say %q:\n%s", want, got) + } + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Unheld []catalogue.Unheld `json:"unheld"` + } + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat { + t.Errorf("the document's unheld is %+v", doc.Unheld) + } +} diff --git a/cmd/mesh-controller/seat_dependencies_test.go b/cmd/mesh-controller/seat_dependencies_test.go index 9d47f4af..2a61cf56 100644 --- a/cmd/mesh-controller/seat_dependencies_test.go +++ b/cmd/mesh-controller/seat_dependencies_test.go @@ -1,7 +1,6 @@ package main import ( - "encoding/json" "strings" "testing" @@ -31,42 +30,6 @@ func aDaemon() catalogue.Manifest { Resources: []map[string]any{{"id": "sshd", "type": "service", "unit": "sshd.service"}}} } -func TestAnAssignmentWithoutItsHolderIsRefusedAndNotKept(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, serviceManagerHolder()) - register(t, open, packageManagerHolder()) - register(t, open, aDaemon()) - - _, err := assign(ctx, open, "laptop", "sshd") - if err == nil { - t.Fatal("sshd went onto a machine nothing holds the service manager of") - } - for _, want := range []string{catalogue.ServiceManagerSeat, "systemd", "ADR 0207"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q:\n%v", want, err) - } - } - assigned, err := open.inventory.Assigned(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if contains(assigned, "sshd") { - t.Fatalf("a refused assignment was kept: %v", assigned) - } - - // The holders depend on each other, so neither goes on alone — and both go on in one act. - if _, err := assign(ctx, open, "laptop", "systemd"); err == nil { - t.Fatal("systemd went on alone though its package needs a package manager") - } - if said, err := assign(ctx, open, "laptop", "systemd", "pacman"); err != nil { - t.Fatalf("the two holders assigned together were refused: %v\n%s", err, said) - } - if said, err := assign(ctx, open, "laptop", "sshd"); err != nil { - t.Fatalf("sshd beside its holder was refused: %v\n%s", err, said) - } -} - func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) { argv, err := argvFor("assign", map[string]any{"node": "laptop", "module": "systemd, pacman"}) if err != nil { @@ -76,75 +39,3 @@ func TestTheControllerSeatsAssignTakesSeveralModulesAsOneAct(t *testing.T) { t.Errorf("the seat's assign became %v", argv) } } - -func TestUnassigningTheLastHolderUnderItsDependentsIsRefused(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, serviceManagerHolder()) - register(t, open, packageManagerHolder()) - register(t, open, aDaemon()) - if _, err := assign(ctx, open, "laptop", "systemd", "pacman", "sshd"); err != nil { - t.Fatal(err) - } - - _, err := unassign(ctx, open, "laptop", "systemd") - if err == nil { - t.Fatal("the service manager came off a machine still running services") - } - for _, want := range []string{catalogue.ServiceManagerSeat, "sshd", "pacman"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not name %q:\n%v", want, err) - } - } - assigned, _ := open.inventory.Assigned(ctx, "laptop") - if !contains(assigned, "systemd") { - t.Fatalf("a refused unassignment took the module off anyway: %v", assigned) - } - if _, err := unassign(ctx, open, "laptop", "sshd"); err != nil { - t.Fatalf("a dependent could not come off: %v", err) - } -} - -func TestStatusReportsAnUnheldDependencyWithoutRefusingTheMachine(t *testing.T) { - // The mesh as it ran before the switch (novox/hq ADR 0207 §4). - defer catalogue.EnforcingSeatDependencies(false)() - open := aMesh(t) - ctx := t.Context() - register(t, open, serviceManagerHolder()) - register(t, open, aDaemon()) - // Assigned straight into the store: a machine whose modules predate the rule, which is every - // machine on the day it ships. - if _, err := open.inventory.Assign(ctx, "laptop", "sshd"); err != nil { - t.Fatal(err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if _, refused := asked.refused["laptop"]; refused { - t.Fatalf("an unmet dependency refused the machine before the switch: %s", asked.refused["laptop"]) - } - if asked.well() { - t.Error("a mesh with an unheld dependency reads as all well") - } - got := printed(t, func() error { return printStatus(asked) }) - for _, want := range []string{"unheld", "laptop", "sshd", catalogue.ServiceManagerSeat, "systemd"} { - if !strings.Contains(got, want) { - t.Errorf("status does not say %q:\n%s", want, got) - } - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Unheld []catalogue.Unheld `json:"unheld"` - } - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Unheld) != 1 || doc.Unheld[0].Module != "sshd" || doc.Unheld[0].Seat != catalogue.ServiceManagerSeat { - t.Errorf("the document's unheld is %+v", doc.Unheld) - } -} diff --git a/cmd/mesh-controller/seatverbs_integration_test.go b/cmd/mesh-controller/seatverbs_integration_test.go new file mode 100644 index 00000000..84e0dea8 --- /dev/null +++ b/cmd/mesh-controller/seatverbs_integration_test.go @@ -0,0 +1,164 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/link" +) + +func TestToolsAnswersTheSeatsRecords(t *testing.T) { + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + if len(behind) != 0 { + t.Fatalf("this build cannot run %v of its own seat's verbs", behind) + } + if len(handlers) != len(catalogue.ControllerVerbs) { + t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs)) + } + answer := seatTools() + seats, _ := answer["seats"].([]map[string]any) + var found bool + for _, s := range seats { + if s["seat"] == catalogue.ControllerSeatName { + found = true + tools, _ := s["tools"].([]map[string]any) + if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil { + t.Fatalf("the controller seat's tools are not listed in full: %v", tools) + } + } + } + if !found { + t.Fatal("the mesh-controller seat is not in the listing") + } + // And what a verb replaces travels with it (novox/hq ADR 0245): the console and the agent's + // instructions read it from here. + var journal []string + for _, s := range seats { + if s["seat"] != catalogue.ServiceManagerSeat { + continue + } + tools, _ := s["tools"].([]map[string]any) + for _, tool := range tools { + if tool["name"] == "journal" { + journal, _ = tool["replaces"].([]string) + } + } + } + if len(journal) == 0 || journal[0] != "journalctl" { + t.Fatalf("the service manager's journal does not say it replaces journalctl: %v", journal) + } +} + +func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) { + jsonVerbs["-test.run"] = true + t.Cleanup(func() { delete(jsonVerbs, "-test.run") }) + answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"}) + if err != nil { + t.Fatal(err) + } + if !answer.OK { + t.Fatalf("the command failed: %s", answer.Output) + } + if !strings.Contains(answer.Output, "PASS") { + t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output) + } +} + +func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) { + seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName) + if !known { + t.Fatal("no controller seat") + } + // The row as a newer control plane would have written it: every verb this build knows, and one + // it does not. + widened := seat + widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...), + catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"}) + rows := catalogue.DefaultSeats() + for i := range rows { + if rows[i].Name == catalogue.ControllerSeatName { + rows[i] = widened + } + } + catalogue.UseSeats(rows) + t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) + + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatalf("a row with one unknown verb refused to serve at all: %v", err) + } + if len(behind) != 1 || behind[0] != "teleport" { + t.Fatalf("the verbs this build cannot run were reported as %v", behind) + } + if len(handlers) != len(widened.Serves) { + t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves)) + } + for _, known := range []string{"status", "nodes", "push"} { + if handlers[known] == nil { + t.Errorf("%s is not served although this build knows it", known) + } + } + _, err = handlers["teleport"](context.Background(), nil) + if err == nil { + t.Fatal("the unknown verb answered as though it had run") + } + for _, want := range []string{"teleport", "cannot run it", "behind"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the answer does not say %q: %v", want, err) + } + } +} + +func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) { + handlers, _, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + info := seatAnnouncement(handlers) + if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" { + t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity) + } + if len(info.Endpoints) != len(handlers) { + t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers)) + } + for _, e := range info.Endpoints { + verb := e.Metadata["tool"] + if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb { + t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb) + } + if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName { + t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata) + } + if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName { + t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup) + } + if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" { + t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata) + } + } +} + +func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) { + for _, argv := range [][]string{ + {"operator", "key", "set"}, {"operator", "issue"}, {"identity"}, {"token", "issue"}, {"broker", "users"}, + {"api"}, {"licence"}, {"secret", "export", "x"}, {"secret", "recover", "x"}, {"secret", "accept", "x"}, {"secret"}, + } { + if err := terminalOnly(argv); err == nil { + t.Errorf("%v passed", argv) + } + } + if err := terminalOnly([]string{"secret", "rotate", "n", "m", "s"}); err != nil { + t.Errorf("rotating seals to the machine that uses the secret, and stays a verb's: %v", err) + } + // A policy refusal is not a verb this binary is behind on: every verb is still served. + if _, behind, err := seatToolHandlers(); err != nil || len(behind) != 0 { + t.Fatalf("behind %v: %v", behind, err) + } +} diff --git a/cmd/mesh-controller/seatverbs_schema_integration_test.go b/cmd/mesh-controller/seatverbs_schema_integration_test.go new file mode 100644 index 00000000..88eb3882 --- /dev/null +++ b/cmd/mesh-controller/seatverbs_schema_integration_test.go @@ -0,0 +1,185 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "reflect" + "sort" + "strconv" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func servedSchemas(t *testing.T) map[string]catalogue.Verb { + t.Helper() + handlers, behind, err := seatToolHandlers() + if err != nil { + t.Fatal(err) + } + if len(behind) != 0 { + t.Fatalf("this build cannot run %v of its own seat's verbs", behind) + } + served := map[string]catalogue.Verb{} + for _, e := range seatAnnouncement(handlers).Endpoints { + var input map[string]any + if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil { + t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err) + } + served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input} + } + if len(served) != len(catalogue.ControllerVerbs) { + t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs)) + } + return served +} + +func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) { + for name, v := range servedSchemas(t) { + if inProcess[name] { + continue + } + names, switches := declaredArguments(v) + for _, subset := range subsetsOf(names) { + args := argumentsFor(subset, switches) + argv, err := argvFor(name, args) + if err != nil { + if strings.Contains(err.Error(), "does not declare") { + t.Errorf("%s %v: %v", name, args, err) + } + continue + } + for _, dropped := range subset { + fewer := map[string]any{} + for k, val := range args { + if k != dropped { + fewer[k] = val + } + } + without, err := argvFor(name, fewer) + if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped { + t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv) + } + } + } + } +} + +func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { + for name, v := range servedSchemas(t) { + names, _ := declaredArguments(v) + strangers := []string{"no-such-argument"} + if !contains(names, "node") { + strangers = append(strangers, "node") + } + for _, stranger := range strangers { + args := sampleArguments(v) + args[stranger] = "x" + _, err := argvFor(name, args) + if inProcess[name] { + _, err = readArguments(name, args) + } + if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) { + t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err) + } + } + } + if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil { + t.Error("a switch took a word that is neither true nor false, and would have read it as false") + } + if _, err := argvFor("node", map[string]any{"node": 7}); err == nil { + t.Error("a number was taken as a machine's name, or as no machine") + } + if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" { + t.Errorf("a switch given as JSON true: %v %v", argv, err) + } +} + +func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) { + flags := commandFlags(t) + if len(flags["push"]) == 0 || len(flags["plan"]) == 0 { + t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags) + } + reached := map[string]map[string]bool{} // verb → flag sets its command lines reach + served := servedSchemas(t) + for name, v := range served { + if inProcess[name] || name == "command" { + continue + } + names, switches := declaredArguments(v) + for _, subset := range subsetsOf(names) { + argv, err := argvFor(name, argumentsFor(subset, switches)) + if err != nil { + continue + } + // The flag set is named by the longest run of leading words that names one. + var words []string + for _, w := range argv { + if strings.HasPrefix(w, "-") { + break + } + words = append(words, w) + } + for n := len(words); n > 0; n-- { + if _, has := flags[strings.Join(words[:n], " ")]; has { + if reached[name] == nil { + reached[name] = map[string]bool{} + } + reached[name][strings.Join(words[:n], " ")] = true + break + } + } + } + } + used := map[string]map[string]bool{} + verbs := make([]string, 0, len(reached)) + for verb := range reached { + verbs = append(verbs, verb) + } + sort.Strings(verbs) + for _, verb := range verbs { + declared, _ := declaredArguments(served[verb]) + for set := range reached[verb] { + if used[set] == nil { + used[set] = map[string]bool{} + } + for _, flagName := range flags[set] { + why, accounted := accountedFlags[set][flagName] + switch { + case accounted && strings.HasPrefix(why, "="): + used[set][flagName] = true + if !contains(declared, strings.TrimPrefix(why, "=")) { + t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare", + verb, flagName, set, strings.TrimPrefix(why, "=")) + } + case accounted: + used[set][flagName] = true + case contains(declared, flagName): + default: + t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+ + "or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName) + } + } + } + } + for set, fs := range accountedFlags { + for flagName := range fs { + if !used[set][flagName] { + t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set) + } + } + } +} + +func TestEveryRequiredArgumentIsDescribed(t *testing.T) { + for name, v := range servedSchemas(t) { + names, _ := declaredArguments(v) + for k := range sampleArguments(v) { + if !contains(names, k) { + t.Errorf("%s requires %q and does not describe it", name, k) + } + } + } +} diff --git a/cmd/mesh-controller/seatverbs_schema_test.go b/cmd/mesh-controller/seatverbs_schema_test.go index a5c489dd..a14f1041 100644 --- a/cmd/mesh-controller/seatverbs_schema_test.go +++ b/cmd/mesh-controller/seatverbs_schema_test.go @@ -1,44 +1,17 @@ package main import ( - "encoding/json" "go/ast" "go/parser" "go/token" "path/filepath" - "reflect" - "sort" "strconv" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" ) // The schemas the controller serves, verb by verb, as the console receives them: from the // announcement, not the table — what is checked is what a caller is shown (novox/hq issue 244). -func servedSchemas(t *testing.T) map[string]catalogue.Verb { - t.Helper() - handlers, behind, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - if len(behind) != 0 { - t.Fatalf("this build cannot run %v of its own seat's verbs", behind) - } - served := map[string]catalogue.Verb{} - for _, e := range seatAnnouncement(handlers).Endpoints { - var input map[string]any - if err := json.Unmarshal([]byte(e.Metadata["schema"]), &input); err != nil { - t.Fatalf("%s announces a schema that is not JSON: %v", e.Name, err) - } - served[e.Metadata["tool"]] = catalogue.Verb{Name: e.Metadata["tool"], Input: input} - } - if len(served) != len(catalogue.ControllerVerbs) { - t.Fatalf("%d verbs served for %d in the table", len(served), len(catalogue.ControllerVerbs)) - } - return served -} // subsetsOf is every subset of the names, the empty one included. func subsetsOf(names []string) [][]string { @@ -78,69 +51,10 @@ var saidOutright = map[string]string{ // composes a command line that each given argument changed: taking any one away changes the line // or makes it refused. An argument the line is the same without is one the verb ignored — the // shape of the push that named a machine and pushed every machine behind. -func TestNoArgumentAVerbIsGivenIsPassedOver(t *testing.T) { - for name, v := range servedSchemas(t) { - if inProcess[name] { - continue - } - names, switches := declaredArguments(v) - for _, subset := range subsetsOf(names) { - args := argumentsFor(subset, switches) - argv, err := argvFor(name, args) - if err != nil { - if strings.Contains(err.Error(), "does not declare") { - t.Errorf("%s %v: %v", name, args, err) - } - continue - } - for _, dropped := range subset { - fewer := map[string]any{} - for k, val := range args { - if k != dropped { - fewer[k] = val - } - } - without, err := argvFor(name, fewer) - if err == nil && reflect.DeepEqual(argv, without) && saidOutright[name] != dropped { - t.Errorf("%s ignores %q given with %v: %v either way", name, dropped, subset, argv) - } - } - } - } -} // **An argument a verb does not declare is refused, naming it — never dropped.** Every verb, with // what it requires and one argument more; and `node` in particular, for every verb whose schema // does not take a machine. -func TestAnArgumentAVerbDoesNotDeclareIsRefused(t *testing.T) { - for name, v := range servedSchemas(t) { - names, _ := declaredArguments(v) - strangers := []string{"no-such-argument"} - if !contains(names, "node") { - strangers = append(strangers, "node") - } - for _, stranger := range strangers { - args := sampleArguments(v) - args[stranger] = "x" - _, err := argvFor(name, args) - if inProcess[name] { - _, err = readArguments(name, args) - } - if err == nil || !strings.Contains(err.Error(), strconv.Quote(stranger)) { - t.Errorf("%s took %q, which it does not declare: %v", name, stranger, err) - } - } - } - if _, err := argvFor("clear", map[string]any{"dead": "yes"}); err == nil { - t.Error("a switch took a word that is neither true nor false, and would have read it as false") - } - if _, err := argvFor("node", map[string]any{"node": 7}); err == nil { - t.Error("a number was taken as a machine's name, or as no machine") - } - if argv, err := argvFor("clear", map[string]any{"dead": true}); err != nil || strings.Join(argv, " ") != "clear --dead" { - t.Errorf("a switch given as JSON true: %v %v", argv, err) - } -} // The push that was the cause: a machine named is that machine; none named is the whole mesh, and // naming one beside behind is refused rather than one of the two guessed. @@ -297,91 +211,6 @@ var accountedFlags = map[string]map[string]string{ // fails this until somebody decides, in writing, how a caller reaches it (novox/hq issue 244). And // a flag accounted for that no longer exists fails too, so the table cannot rot into a list nobody // reads. -func TestEveryFlagOfAVerbsCommandIsAnArgumentOrAccountedFor(t *testing.T) { - flags := commandFlags(t) - if len(flags["push"]) == 0 || len(flags["plan"]) == 0 { - t.Fatalf("reading the commands' flags found nothing for push or plan: %v", flags) - } - reached := map[string]map[string]bool{} // verb → flag sets its command lines reach - served := servedSchemas(t) - for name, v := range served { - if inProcess[name] || name == "command" { - continue - } - names, switches := declaredArguments(v) - for _, subset := range subsetsOf(names) { - argv, err := argvFor(name, argumentsFor(subset, switches)) - if err != nil { - continue - } - // The flag set is named by the longest run of leading words that names one. - var words []string - for _, w := range argv { - if strings.HasPrefix(w, "-") { - break - } - words = append(words, w) - } - for n := len(words); n > 0; n-- { - if _, has := flags[strings.Join(words[:n], " ")]; has { - if reached[name] == nil { - reached[name] = map[string]bool{} - } - reached[name][strings.Join(words[:n], " ")] = true - break - } - } - } - } - used := map[string]map[string]bool{} - verbs := make([]string, 0, len(reached)) - for verb := range reached { - verbs = append(verbs, verb) - } - sort.Strings(verbs) - for _, verb := range verbs { - declared, _ := declaredArguments(served[verb]) - for set := range reached[verb] { - if used[set] == nil { - used[set] = map[string]bool{} - } - for _, flagName := range flags[set] { - why, accounted := accountedFlags[set][flagName] - switch { - case accounted && strings.HasPrefix(why, "="): - used[set][flagName] = true - if !contains(declared, strings.TrimPrefix(why, "=")) { - t.Errorf("%s: --%s of `%s` is said to be carried by %q, which the schema does not declare", - verb, flagName, set, strings.TrimPrefix(why, "=")) - } - case accounted: - used[set][flagName] = true - case contains(declared, flagName): - default: - t.Errorf("%s runs `%s`, which takes --%s, and the verb's schema has no %q: declare it, "+ - "or say in accountedFlags why a caller does not reach it", verb, set, flagName, flagName) - } - } - } - } - for set, fs := range accountedFlags { - for flagName := range fs { - if !used[set][flagName] { - t.Errorf("accountedFlags names --%s of `%s`, which no verb's command takes any more", flagName, set) - } - } - } -} // Every verb's required arguments are properties of its schema: a schema that requires what it // does not describe is the uncallable verb of issue 244 from the other side. -func TestEveryRequiredArgumentIsDescribed(t *testing.T) { - for name, v := range servedSchemas(t) { - names, _ := declaredArguments(v) - for k := range sampleArguments(v) { - if !contains(names, k) { - t.Errorf("%s requires %q and does not describe it", name, k) - } - } - } -} diff --git a/cmd/mesh-controller/seatverbs_test.go b/cmd/mesh-controller/seatverbs_test.go index c6bf7120..361fdc49 100644 --- a/cmd/mesh-controller/seatverbs_test.go +++ b/cmd/mesh-controller/seatverbs_test.go @@ -1,16 +1,12 @@ package main import ( - "context" "errors" "fmt" - "github.com/novox/mesh-controller/internal/link" "os" "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" ) // `builds` given a build's id reads that build's log from the bus rather than listing builds @@ -182,68 +178,10 @@ func TestActsDoNotBlockTheCall(t *testing.T) { } // What `tools` answers is the seats' records, with each verb's schema. -func TestToolsAnswersTheSeatsRecords(t *testing.T) { - handlers, behind, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - if len(behind) != 0 { - t.Fatalf("this build cannot run %v of its own seat's verbs", behind) - } - if len(handlers) != len(catalogue.ControllerVerbs) { - t.Fatalf("%d handlers for %d verbs", len(handlers), len(catalogue.ControllerVerbs)) - } - answer := seatTools() - seats, _ := answer["seats"].([]map[string]any) - var found bool - for _, s := range seats { - if s["seat"] == catalogue.ControllerSeatName { - found = true - tools, _ := s["tools"].([]map[string]any) - if len(tools) != len(catalogue.ControllerVerbs) || tools[0]["input"] == nil { - t.Fatalf("the controller seat's tools are not listed in full: %v", tools) - } - } - } - if !found { - t.Fatal("the mesh-controller seat is not in the listing") - } - // And what a verb replaces travels with it (novox/hq ADR 0245): the console and the agent's - // instructions read it from here. - var journal []string - for _, s := range seats { - if s["seat"] != catalogue.ServiceManagerSeat { - continue - } - tools, _ := s["tools"].([]map[string]any) - for _, tool := range tools { - if tool["name"] == "journal" { - journal, _ = tool["replaces"].([]string) - } - } - } - if len(journal) == 0 || journal[0] != "journalctl" { - t.Fatalf("the service manager's journal does not say it replaces journalctl: %v", journal) - } -} // A JSON verb's answer is parsed from what the command wrote to standard output alone; a warning it // printed beside the document does not take the document away. The test binary stands in for the // controller: `-test.run` with a name that matches nothing prints `ok` and a warning about no tests. -func TestAJSONVerbsAnswerIsItsStandardOutput(t *testing.T) { - jsonVerbs["-test.run"] = true - t.Cleanup(func() { delete(jsonVerbs, "-test.run") }) - answer, err := runVerb(t.Context(), []string{"-test.run", "TestAnswerEcho", "-test.v"}) - if err != nil { - t.Fatal(err) - } - if !answer.OK { - t.Fatalf("the command failed: %s", answer.Output) - } - if !strings.Contains(answer.Output, "PASS") { - t.Fatalf("stderr and stdout are both what the command said: %s", answer.Output) - } -} // `command` is the generic verb: the command line as given, split as a shell would, nothing added // (novox/hq ADR 0154, ADR 0175). It once carried an operator's `node account g14 jochen` too; a node's @@ -274,50 +212,6 @@ func TestCommandRunsTheLineAsGiven(t *testing.T) { // 0185). One unknown word cost the mesh ten minutes of silence on 2026-10-02, recoverable only by // a person running the binary by hand — the push that would have repaired it needs the control // plane that was down. -func TestARowAheadOfThisBuildIsServedAnyway(t *testing.T) { - seat, known := catalogue.SeatNamed(catalogue.ControllerSeatName) - if !known { - t.Fatal("no controller seat") - } - // The row as a newer control plane would have written it: every verb this build knows, and one - // it does not. - widened := seat - widened.Serves = append(append([]catalogue.Verb{}, seat.Serves...), - catalogue.Verb{Name: "teleport", Description: "a verb from a build that does not exist yet"}) - rows := catalogue.DefaultSeats() - for i := range rows { - if rows[i].Name == catalogue.ControllerSeatName { - rows[i] = widened - } - } - catalogue.UseSeats(rows) - t.Cleanup(func() { catalogue.UseSeats(catalogue.DefaultSeats()) }) - - handlers, behind, err := seatToolHandlers() - if err != nil { - t.Fatalf("a row with one unknown verb refused to serve at all: %v", err) - } - if len(behind) != 1 || behind[0] != "teleport" { - t.Fatalf("the verbs this build cannot run were reported as %v", behind) - } - if len(handlers) != len(widened.Serves) { - t.Fatalf("%d handlers for %d verbs in the row", len(handlers), len(widened.Serves)) - } - for _, known := range []string{"status", "nodes", "push"} { - if handlers[known] == nil { - t.Errorf("%s is not served although this build knows it", known) - } - } - _, err = handlers["teleport"](context.Background(), nil) - if err == nil { - t.Fatal("the unknown verb answered as though it had run") - } - for _, want := range []string{"teleport", "cannot run it", "behind"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the answer does not say %q: %v", want, err) - } - } -} // novox/hq ADR 0195: the console's discovery reads the machines and the modules; they answer as JSON, // as status and seats do, so nothing parses a printed column. @@ -335,34 +229,6 @@ func TestTheNodesAndModulesVerbsAnswerAsJSON(t *testing.T) { // novox/hq ADR 0197: the controller announces exactly the verbs it serves, each on the subject and // queue it serves it on, with the seat's own description and schema, in NATS's services format. -func TestTheControllerAnnouncesTheVerbsItServes(t *testing.T) { - handlers, _, err := seatToolHandlers() - if err != nil { - t.Fatal(err) - } - info := seatAnnouncement(handlers) - if info.Name != catalogue.ControllerSeatName || info.ID == "" || info.Version == "" { - t.Fatalf("the service is not named for the seat: %+v", info.ServiceIdentity) - } - if len(info.Endpoints) != len(handlers) { - t.Fatalf("%d endpoints announced for %d verbs served", len(info.Endpoints), len(handlers)) - } - for _, e := range info.Endpoints { - verb := e.Metadata["tool"] - if _, served := handlers[verb]; !served || e.Name != catalogue.ControllerSeatName+"__"+verb { - t.Errorf("%s (%s) is announced and not served under that name", e.Name, verb) - } - if e.Metadata["kind"] != "seat" || e.Metadata["seat"] != catalogue.ControllerSeatName { - t.Errorf("%s is not announced as the seat's verb: %v", e.Name, e.Metadata) - } - if e.Subject != link.SeatToolSubject(catalogue.ControllerSeatName, verb) || e.QueueGroup != "seat."+catalogue.ControllerSeatName { - t.Errorf("%s is announced on %s/%s, not where it is served", e.Name, e.Subject, e.QueueGroup) - } - if e.Metadata["description"] == "" || e.Metadata["schema"] == "" || e.Metadata["scope"] != "mesh" { - t.Errorf("%s is announced without its description, schema or scope: %v", e.Name, e.Metadata) - } - } -} // The generic verb only reads (novox/hq ADR 0266): an allow list of reading forms, and every line that // writes, issues, sets a key or reveals a secret refused — the chain a review found ran through it: set the @@ -402,23 +268,6 @@ func TestTheCommandVerbOnlyReads(t *testing.T) { } // What hands a caller a key, a credential or a secret is refused whichever verb composed it. -func TestNoVerbSetsTheOperatorsKeyOrRevealsASecret(t *testing.T) { - for _, argv := range [][]string{ - {"operator", "key", "set"}, {"operator", "issue"}, {"identity"}, {"token", "issue"}, {"broker", "users"}, - {"api"}, {"licence"}, {"secret", "export", "x"}, {"secret", "recover", "x"}, {"secret", "accept", "x"}, {"secret"}, - } { - if err := terminalOnly(argv); err == nil { - t.Errorf("%v passed", argv) - } - } - if err := terminalOnly([]string{"secret", "rotate", "n", "m", "s"}); err != nil { - t.Errorf("rotating seals to the machine that uses the secret, and stays a verb's: %v", err) - } - // A policy refusal is not a verb this binary is behind on: every verb is still served. - if _, behind, err := seatToolHandlers(); err != nil || len(behind) != 0 { - t.Fatalf("behind %v: %v", behind, err) - } -} // A value with a quote in it can be said on a `command` line, as in a shell, and a line whose quote // is left open is refused naming where it opened and how a quote is written, quoting none of it (novox/hq issue 294: an diff --git a/cmd/mesh-controller/secret_integration_test.go b/cmd/mesh-controller/secret_integration_test.go new file mode 100644 index 00000000..b3595be5 --- /dev/null +++ b/cmd/mesh-controller/secret_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" +) + +func TestTheArgumentsAreRequired(t *testing.T) { + for _, args := range [][]string{ + {}, + {"accept"}, + {"accept", "anchor"}, + {"accept", "anchor", "umami"}, + {"give", "anchor", "umami", "database"}, + } { + if err := secretCommand(t.Context(), args); err == nil { + t.Errorf("%v was accepted", args) + } + } +} + +func TestTheCommandItselfSeesTheFlag(t *testing.T) { + err := secretCommand(t.Context(), + []string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"}) + if err == nil { + t.Fatal("a missing file was accepted") + } + if strings.Contains(err.Error(), "secret accept ") { + t.Fatalf("the command did not see its flag and complained about usage instead: %v", err) + } +} diff --git a/cmd/mesh-controller/secret_test.go b/cmd/mesh-controller/secret_test.go index 7753cf94..f90ec4e1 100644 --- a/cmd/mesh-controller/secret_test.go +++ b/cmd/mesh-controller/secret_test.go @@ -3,7 +3,6 @@ package main import ( "os" "path/filepath" - "strings" "testing" ) @@ -55,19 +54,6 @@ func TestAMissingFileIsRefused(t *testing.T) { } // The command refuses what it cannot act on, rather than acting on part of it. -func TestTheArgumentsAreRequired(t *testing.T) { - for _, args := range [][]string{ - {}, - {"accept"}, - {"accept", "anchor"}, - {"accept", "anchor", "umami"}, - {"give", "anchor", "umami", "database"}, - } { - if err := secretCommand(t.Context(), args); err == nil { - t.Errorf("%v was accepted", args) - } - } -} // The invocation that actually gets used, which the first version of these tests never tried. // @@ -92,16 +78,6 @@ func TestTheArgumentsAndTheFlagAreBothSeen(t *testing.T) { // Testing `split` alone left the command able to ignore it entirely — removing the call changed // no test. This reaches secretCommand: with a `--from` naming a file that is not there, the // complaint must be about the file. A complaint about usage would mean the flag was never seen. -func TestTheCommandItselfSeesTheFlag(t *testing.T) { - err := secretCommand(t.Context(), - []string{"accept", "anchor", "umami", "database", "--from", "/nonexistent/nowhere"}) - if err == nil { - t.Fatal("a missing file was accepted") - } - if strings.Contains(err.Error(), "secret accept ") { - t.Fatalf("the command did not see its flag and complained about usage instead: %v", err) - } -} func assert(t *testing.T, ok bool, format string, args ...any) { t.Helper() diff --git a/cmd/mesh-controller/selfexec_integration_test.go b/cmd/mesh-controller/selfexec_integration_test.go new file mode 100644 index 00000000..6b74d0d6 --- /dev/null +++ b/cmd/mesh-controller/selfexec_integration_test.go @@ -0,0 +1,138 @@ +//go:build integration + +package main + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestSelfExecServer(t *testing.T) { + if os.Getenv(selfExecRole) != "server" { + t.Skip("run by TestAVerbRunsWhileItsBuildIsMovedOrDeleted") + } + line, _ := bufio.NewReader(os.Stdin).ReadString('\n') + if strings.TrimSpace(line) != "go" { + t.Fatalf("told %q", line) + } + if err := os.Setenv(selfExecRole, "verb"); err != nil { + t.Fatal(err) + } + answer, err := runVerb(t.Context(), []string{"-test.run", "^TestSelfExecVerb$", "-test.v"}) + said := map[string]any{"ok": answer.OK, "output": answer.Output} + if err != nil { + said["error"] = err.Error() + } + body, _ := json.Marshal(said) + fmt.Println("ANSWER " + string(body)) +} + +func TestAVerbRunsWhileItsBuildIsMovedOrDeleted(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("the image is named through /proc on Linux only") + } + self, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, how := range []string{"moved into a closed directory", "deleted"} { + t.Run(how, func(t *testing.T) { + root := t.TempDir() + placed := filepath.Join(root, "mesh-controller", "mesh-controller") + if err := os.MkdirAll(filepath.Dir(placed), 0o755); err != nil { + t.Fatal(err) + } + copyFile(t, self, placed) + + server := exec.Command(placed, "-test.run", "^TestSelfExecServer$", "-test.v") + server.Env = append(os.Environ(), selfExecRole+"=server") + stdin, err := server.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := server.StdoutPipe() + if err != nil { + t.Fatal(err) + } + server.Stderr = os.Stderr + if err := server.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = server.Process.Kill(); _ = server.Wait() }) + + // What the witness does to a running build, once the process runs. + switch how { + case "deleted": + if err := os.RemoveAll(filepath.Dir(placed)); err != nil { + t.Fatal(err) + } + default: + kept := filepath.Join(root, ".witness", "mesh-controller", "previous") + if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { + t.Fatal(err) + } + if err := os.Rename(filepath.Dir(placed), kept); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(root, ".witness"), 0); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(filepath.Join(root, ".witness"), 0o700) }) + } + if _, err := io.WriteString(stdin, "go\n"); err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(stdout) + _ = server.Wait() + var said struct { + OK bool `json:"ok"` + Output string `json:"output"` + Error string `json:"error"` + } + found := false + for _, line := range strings.Split(string(out), "\n") { + if rest, ok := strings.CutPrefix(line, "ANSWER "); ok { + found = json.Unmarshal([]byte(rest), &said) == nil + } + } + if !found { + t.Fatalf("the controller standing in answered nothing:\n%s", out) + } + if said.Error != "" || !said.OK || !strings.Contains(said.Output, "the verb ran") { + t.Fatalf("the verb did not run from the controller's own image after its build was %s: %+v", how, said) + } + }) + } +} + +func TestAVerbThatCannotRunIsRefusedAsAHandover(t *testing.T) { + closed := filepath.Join(t.TempDir(), "closed") + if err := os.MkdirAll(closed, 0o700); err != nil { + t.Fatal(err) + } + was := ownImage + ownImage = func() string { return filepath.Join(closed, "gone", "mesh-controller") } + t.Cleanup(func() { ownImage = was }) + _, err := runVerb(t.Context(), []string{"status"}) + if !errors.Is(err, link.ErrHandingOver) { + t.Fatalf("a build that is not there was answered %v, not a handover", err) + } + + ownImage = was + handingOver.Store(true) + t.Cleanup(func() { handingOver.Store(false) }) + if _, err := runVerb(t.Context(), []string{"-test.run", "^$"}); !errors.Is(err, link.ErrHandingOver) { + t.Fatalf("a controller stopping ran a verb: %v", err) + } +} diff --git a/cmd/mesh-controller/selfexec_test.go b/cmd/mesh-controller/selfexec_test.go index 05d8102b..45bf4c28 100644 --- a/cmd/mesh-controller/selfexec_test.go +++ b/cmd/mesh-controller/selfexec_test.go @@ -1,19 +1,10 @@ package main import ( - "bufio" - "encoding/json" - "errors" "fmt" "io" "os" - "os/exec" - "path/filepath" - "runtime" - "strings" "testing" - - "github.com/novox/mesh-controller/internal/link" ) // The roles a copy of this test binary plays in TestAVerbRunsWhileItsBuildIsMovedOrDeleted. @@ -21,25 +12,6 @@ const selfExecRole = "MESH_CONTROLLER_SELFEXEC_ROLE" // TestSelfExecServer is a controller standing in, when run as one: it waits until its build has been // moved, then runs a verb as the seat runs one, and prints what came of it as JSON. -func TestSelfExecServer(t *testing.T) { - if os.Getenv(selfExecRole) != "server" { - t.Skip("run by TestAVerbRunsWhileItsBuildIsMovedOrDeleted") - } - line, _ := bufio.NewReader(os.Stdin).ReadString('\n') - if strings.TrimSpace(line) != "go" { - t.Fatalf("told %q", line) - } - if err := os.Setenv(selfExecRole, "verb"); err != nil { - t.Fatal(err) - } - answer, err := runVerb(t.Context(), []string{"-test.run", "^TestSelfExecVerb$", "-test.v"}) - said := map[string]any{"ok": answer.OK, "output": answer.Output} - if err != nil { - said["error"] = err.Error() - } - body, _ := json.Marshal(said) - fmt.Println("ANSWER " + string(body)) -} // TestSelfExecVerb is the verb, when run as one. func TestSelfExecVerb(t *testing.T) { @@ -56,106 +28,9 @@ func TestSelfExecVerb(t *testing.T) { // // A copy of this test binary is the controller: started from one place, moved into a directory closed // to everyone (or deleted), and only then asked to run a verb. -func TestAVerbRunsWhileItsBuildIsMovedOrDeleted(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("the image is named through /proc on Linux only") - } - self, err := os.Executable() - if err != nil { - t.Fatal(err) - } - for _, how := range []string{"moved into a closed directory", "deleted"} { - t.Run(how, func(t *testing.T) { - root := t.TempDir() - placed := filepath.Join(root, "mesh-controller", "mesh-controller") - if err := os.MkdirAll(filepath.Dir(placed), 0o755); err != nil { - t.Fatal(err) - } - copyFile(t, self, placed) - - server := exec.Command(placed, "-test.run", "^TestSelfExecServer$", "-test.v") - server.Env = append(os.Environ(), selfExecRole+"=server") - stdin, err := server.StdinPipe() - if err != nil { - t.Fatal(err) - } - stdout, err := server.StdoutPipe() - if err != nil { - t.Fatal(err) - } - server.Stderr = os.Stderr - if err := server.Start(); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = server.Process.Kill(); _ = server.Wait() }) - - // What the witness does to a running build, once the process runs. - switch how { - case "deleted": - if err := os.RemoveAll(filepath.Dir(placed)); err != nil { - t.Fatal(err) - } - default: - kept := filepath.Join(root, ".witness", "mesh-controller", "previous") - if err := os.MkdirAll(filepath.Dir(kept), 0o700); err != nil { - t.Fatal(err) - } - if err := os.Rename(filepath.Dir(placed), kept); err != nil { - t.Fatal(err) - } - if err := os.Chmod(filepath.Join(root, ".witness"), 0); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = os.Chmod(filepath.Join(root, ".witness"), 0o700) }) - } - if _, err := io.WriteString(stdin, "go\n"); err != nil { - t.Fatal(err) - } - out, _ := io.ReadAll(stdout) - _ = server.Wait() - var said struct { - OK bool `json:"ok"` - Output string `json:"output"` - Error string `json:"error"` - } - found := false - for _, line := range strings.Split(string(out), "\n") { - if rest, ok := strings.CutPrefix(line, "ANSWER "); ok { - found = json.Unmarshal([]byte(rest), &said) == nil - } - } - if !found { - t.Fatalf("the controller standing in answered nothing:\n%s", out) - } - if said.Error != "" || !said.OK || !strings.Contains(said.Output, "the verb ran") { - t.Fatalf("the verb did not run from the controller's own image after its build was %s: %+v", how, said) - } - }) - } -} // A verb the controller cannot start from its own build is refused as a handover — marked so its // caller asks again — never a permission error; and so is one arriving once the controller is stopping. -func TestAVerbThatCannotRunIsRefusedAsAHandover(t *testing.T) { - closed := filepath.Join(t.TempDir(), "closed") - if err := os.MkdirAll(closed, 0o700); err != nil { - t.Fatal(err) - } - was := ownImage - ownImage = func() string { return filepath.Join(closed, "gone", "mesh-controller") } - t.Cleanup(func() { ownImage = was }) - _, err := runVerb(t.Context(), []string{"status"}) - if !errors.Is(err, link.ErrHandingOver) { - t.Fatalf("a build that is not there was answered %v, not a handover", err) - } - - ownImage = was - handingOver.Store(true) - t.Cleanup(func() { handingOver.Store(false) }) - if _, err := runVerb(t.Context(), []string{"-test.run", "^$"}); !errors.Is(err, link.ErrHandingOver) { - t.Fatalf("a controller stopping ran a verb: %v", err) - } -} func copyFile(t *testing.T, from, to string) { t.Helper() diff --git a/cmd/mesh-controller/sendable_integration_test.go b/cmd/mesh-controller/sendable_integration_test.go new file mode 100644 index 00000000..e1817f27 --- /dev/null +++ b/cmd/mesh-controller/sendable_integration_test.go @@ -0,0 +1,354 @@ +//go:build integration + +package main + +import ( + "bytes" + "encoding/json" + "io" + "os" + "reflect" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/overlay" +) + +func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, helloWeb()) + if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil { + t.Fatal(err) + } + declared := composed(t, open, "laptop") + if declared.Adoption != nil { + t.Fatal("a converged node was given an adoption envelope") + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + if string(body) != convergedBefore { + t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s", + body, convergedBefore) + } + if bytes.Contains(body, []byte(`"adoption"`)) { + t.Fatal("a converged declaration names adoption; an older host would refuse it") + } +} + +func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, helloWeb()) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + + declared := composed(t, open, "anchor") + if declared.Adoption == nil { + t.Fatal("an adopted node's declaration does not say it is adopted") + } + untaken := declared.Adoption.Untaken["hello-web"] + // Every kind — its directory too (novox/hq ADR 0103). + if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server", + "hello-web.served"}) { + t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption) + } + if len(declared.Adoption.Taken) != 0 { + t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken) + } + + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + var envelope map[string]any + if err := json.Unmarshal(body, &envelope); err != nil { + t.Fatal(err) + } + adoption, _ := envelope["adoption"].(map[string]any) + if _, ok := adoption["taken"].([]any); !ok { + t.Fatalf("taken is not a list on the wire, even empty: %s", body) + } + + // The digest the mesh compares is the digest of what is sent: status and push agree. + would, err := wouldSend(ctx, open, mustNodes(t, open)) + if err != nil { + t.Fatal(err) + } + if would["anchor"] != digestOf(body) { + t.Fatal("the digest the mesh compares is not of the declaration push sends") + } + + // plan --json prints that same envelope. + printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) + var compact bytes.Buffer + if err := json.Compact(&compact, []byte(printed)); err != nil { + t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) + } + if digestOf(compact.Bytes()) != digestOf(body) { + t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) + } + + // Taking the module moves its resources out of untaken. + if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { + t.Fatal(err) + } + declared = composed(t, open, "anchor") + if _, still := declared.Adoption.Untaken["hello-web"]; still { + t.Fatalf("a taken module is still untaken: %v", declared.Adoption) + } + if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) { + t.Fatalf("taken is %v", declared.Adoption.Taken) + } +} + +func stdoutOf(t *testing.T, run func() error) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + saved := os.Stdout + os.Stdout = w + done := make(chan string) + go func() { + all, _ := io.ReadAll(r) + done <- string(all) + }() + runErr := run() + os.Stdout = saved + w.Close() + out := <-done + if runErr != nil { + t.Fatalf("%v\n%s", runErr, out) + } + return out +} + +func TestConsumersAreToldTheGivenPort(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}, + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}}) + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { + t.Fatal(err) + } + + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var told any + for _, n := range plan.Needs { + if n.Name == "database" { + told = n.Serves["port"] + } + } + if told != 5433 { + t.Fatalf("the consumer is told the store is on %v", told) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) { + t.Fatalf("the store publishes %v", r["ports"]) + } + } + + // A port for the whole mesh is refused where it is set, not stored to refuse every node's + // declaration afterwards. + if err := open.inventory.SetSettings(ctx, "", "store", + map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil || + !strings.Contains(err.Error(), "per node") { + t.Fatalf("a port given for the whole mesh was not refused: %v", err) + } + plan, settings, err := planFor(ctx, open, "anchor") + if err != nil { + t.Fatal(err) + } + if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil { + t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err) + } +} + +func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "store", Version: "1", + Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, + Guards: []int{5432}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", + "ports": []any{"5432:5432"}, + "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "store"); err != nil { + t.Fatal(err) + } + if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) { + t.Fatal("an untaken store is guarded") + } + if err := open.inventory.Take(ctx, "anchor", "store"); err != nil { + t.Fatal(err) + } + for _, r := range composed(t, open, "anchor").Resources { + if r["id"] == catalogue.GuardID() { + if r["content"] != catalogue.AsGuard([]int{5432}) { + t.Fatalf("the taken store's guard is:\n%s", r["content"]) + } + return + } + } + t.Fatal("a taken store is not guarded") +} + +func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "forge", Version: "1", + Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}}, + Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}}, + Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}}, + Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge", + "ports": []any{"2222:22"}, + "image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}}) + register(t, open, catalogue.Manifest{Module: "app", Version: "1", + Requires: []string{"git-over-ssh"}}) + if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "anchor", "forge"); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", "app"); err != nil { + t.Fatal(err) + } + if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "anchor", "forge", + map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil { + t.Fatal(err) + } + + resources := composed(t, open, "anchor").Resources + var container, opening map[string]any + for _, r := range resources { + switch r["id"] { + case "forge.server": + container = r + case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded): + opening = r + } + if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") { + t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id) + } + } + if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) { + t.Fatalf("the forge's container publishes %v", container["ports"]) + } + if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh { + t.Fatalf("no opening for the port this node gave the forge: %v", opening) + } + var guard map[string]any + for _, r := range resources { + if r["id"] == catalogue.GuardID() { + guard = r + } + } + if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) { + t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"]) + } + + // And the consumer on the other machine dials the same number. + plan, _, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + var told any + for _, n := range plan.Needs { + if n.Name == "git-over-ssh" { + told = n.Serves["port"] + } + } + if told != 222 { + t.Fatalf("the consumer is told the forge answers on %v", told) + } +} + +func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + web := helloWeb() + web.Resources[1]["ports"] = []any{"8080"} + register(t, open, web) + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}}) + for _, m := range []string{"hello-web", "notes"} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + // Judged where it is stored: a port the module does not publish is refused by name. + err := open.inventory.SetSettings(ctx, "laptop", "hello-web", + map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}}) + if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") { + t.Fatalf("an impossible setting was stored: %v", err) + } + if err := open.inventory.SetSettings(ctx, "laptop", "hello-web", + map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil { + t.Fatal(err) + } + if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 { + t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut) + } + + // The definition moves: the container publishes another port now. + web.Version = "2" + web.Resources[1]["ports"] = []any{"9090"} + register(t, open, web) + declared := composed(t, open, "laptop") + if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" { + t.Fatalf("hello-web is not left out: %v", declared.LeftOut) + } + if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") { + t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy) + } + if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") { + t.Fatalf("the machine was not told everything else: %v", declared.Resources) + } + body, err := declared.Body() + if err != nil { + t.Fatal(err) + } + var env map[string]any + if err := json.Unmarshal(body, &env); err != nil { + t.Fatal(err) + } + left, _ := env["left_out"].([]any) + if len(left) != 1 || left[0] != "hello-web" { + t.Fatalf("the envelope does not say what was left out: %v", env) + } +} diff --git a/cmd/mesh-controller/sendable_test.go b/cmd/mesh-controller/sendable_test.go index 9b218695..47224eea 100644 --- a/cmd/mesh-controller/sendable_test.go +++ b/cmd/mesh-controller/sendable_test.go @@ -1,17 +1,12 @@ package main import ( - "bytes" "encoding/json" - "io" - "os" - "reflect" "strings" "testing" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/overlay" ) // novox/hq ADR 0100: every declaration an adopted node is sent says it is adopted and which modules @@ -54,103 +49,6 @@ func composed(t *testing.T, open *stores, node string) sendable { // this guard permits; a field it would refuse is the one it exists to catch. const convergedBefore = `{"declaration":1,"resources":[{"content":"hello","id":"hello-web.page","path":"/var/lib/hello-web/index.html","type":"file"},{"id":"hello-web.server","image":"registry.example/hello@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","name":"hello-web","type":"container"},{"id":"hello-web.served","path":"/var/lib/hello-web","type":"directory"}]}` -func TestAConvergedDeclarationIsByteForByteWhatItWas(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, helloWeb()) - if _, err := unassign(ctx, open, "laptop", overlay.Name); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "hello-web"); err != nil { - t.Fatal(err) - } - declared := composed(t, open, "laptop") - if declared.Adoption != nil { - t.Fatal("a converged node was given an adoption envelope") - } - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - if string(body) != convergedBefore { - t.Fatalf("a converged declaration changed; an older host parses this strictly:\n%s\n%s", - body, convergedBefore) - } - if bytes.Contains(body, []byte(`"adoption"`)) { - t.Fatal("a converged declaration names adoption; an older host would refuse it") - } -} - -func TestAnAdoptedDeclarationCarriesItsModeAndWhatWasTaken(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, helloWeb()) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "hello-web"); err != nil { - t.Fatal(err) - } - - declared := composed(t, open, "anchor") - if declared.Adoption == nil { - t.Fatal("an adopted node's declaration does not say it is adopted") - } - untaken := declared.Adoption.Untaken["hello-web"] - // Every kind — its directory too (novox/hq ADR 0103). - if !reflect.DeepEqual(untaken, []string{"hello-web.page", "hello-web.server", - "hello-web.served"}) { - t.Fatalf("hello-web's resources are not all named untaken: %v", declared.Adoption) - } - if len(declared.Adoption.Taken) != 0 { - t.Fatalf("nothing was taken, and the declaration says %v", declared.Adoption.Taken) - } - - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - var envelope map[string]any - if err := json.Unmarshal(body, &envelope); err != nil { - t.Fatal(err) - } - adoption, _ := envelope["adoption"].(map[string]any) - if _, ok := adoption["taken"].([]any); !ok { - t.Fatalf("taken is not a list on the wire, even empty: %s", body) - } - - // The digest the mesh compares is the digest of what is sent: status and push agree. - would, err := wouldSend(ctx, open, mustNodes(t, open)) - if err != nil { - t.Fatal(err) - } - if would["anchor"] != digestOf(body) { - t.Fatal("the digest the mesh compares is not of the declaration push sends") - } - - // plan --json prints that same envelope. - printed := stdoutOf(t, func() error { return planCommand(ctx, []string{"anchor", "--json"}) }) - var compact bytes.Buffer - if err := json.Compact(&compact, []byte(printed)); err != nil { - t.Fatalf("plan --json is not JSON: %v\n%s", err, printed) - } - if digestOf(compact.Bytes()) != digestOf(body) { - t.Fatalf("plan --json shows something other than what push sends:\n%s", printed) - } - - // Taking the module moves its resources out of untaken. - if err := open.inventory.Take(ctx, "anchor", "hello-web"); err != nil { - t.Fatal(err) - } - declared = composed(t, open, "anchor") - if _, still := declared.Adoption.Untaken["hello-web"]; still { - t.Fatalf("a taken module is still untaken: %v", declared.Adoption) - } - if !reflect.DeepEqual(declared.Adoption.Taken, []string{"hello-web"}) { - t.Fatalf("taken is %v", declared.Adoption.Taken) - } -} - func mustNodes(t *testing.T, open *stores) []inventory.Node { t.Helper() nodes, err := open.inventory.Nodes(t.Context()) @@ -161,122 +59,13 @@ func mustNodes(t *testing.T, open *stores) []inventory.Node { } // stdoutOf is what run printed. -func stdoutOf(t *testing.T, run func() error) string { - t.Helper() - r, w, err := os.Pipe() - if err != nil { - t.Fatal(err) - } - saved := os.Stdout - os.Stdout = w - done := make(chan string) - go func() { - all, _ := io.ReadAll(r) - done <- string(all) - }() - runErr := run() - os.Stdout = saved - w.Close() - out := <-done - if runErr != nil { - t.Fatalf("%v\n%s", runErr, out) - } - return out -} // novox/hq ADR 0100: a port a node was given for the store is where its consumers on other // machines are told to reach it, and a port given for the whole mesh is refused. -func TestConsumersAreToldTheGivenPort(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "store", Version: "1", - Provides: []catalogue.Offer{{Name: "database", Scope: catalogue.ScopeMesh}}, - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Guards: []int{5432}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, - "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) - register(t, open, catalogue.Manifest{Module: "app", Version: "1", Requires: []string{"database"}}) - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "store", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5433}}); err != nil { - t.Fatal(err) - } - - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var told any - for _, n := range plan.Needs { - if n.Name == "database" { - told = n.Serves["port"] - } - } - if told != 5433 { - t.Fatalf("the consumer is told the store is on %v", told) - } - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == "store.server" && !reflect.DeepEqual(r["ports"], []any{"5433:5432"}) { - t.Fatalf("the store publishes %v", r["ports"]) - } - } - - // A port for the whole mesh is refused where it is set, not stored to refuse every node's - // declaration afterwards. - if err := open.inventory.SetSettings(ctx, "", "store", - map[string]any{catalogue.PortsSetting: map[string]any{"5432": 5434}}); err == nil || - !strings.Contains(err.Error(), "per node") { - t.Fatalf("a port given for the whole mesh was not refused: %v", err) - } - plan, settings, err := planFor(ctx, open, "anchor") - if err != nil { - t.Fatal(err) - } - if _, err := declarationFor(ctx, open, "anchor", plan, settings); err != nil { - t.Fatalf("the refused mesh-wide layer was stored anyway: %v", err) - } -} // novox/hq ADR 0103: the guard an adopted node is sent follows what was taken there. A store // assigned but not taken is not guarded — its port may still be the predecessor's — and taking it // guards it from the next declaration. -func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "store", Version: "1", - Listens: []catalogue.Listening{{Port: 5432, From: catalogue.FromMesh}}, - Guards: []int{5432}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "mesh-store", - "ports": []any{"5432:5432"}, - "image": "registry.example/pg@sha256:" + strings.Repeat("b", 64)}}}) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "store"); err != nil { - t.Fatal(err) - } - if hasID(composed(t, open, "anchor").Resources, catalogue.GuardID()) { - t.Fatal("an untaken store is guarded") - } - if err := open.inventory.Take(ctx, "anchor", "store"); err != nil { - t.Fatal(err) - } - for _, r := range composed(t, open, "anchor").Resources { - if r["id"] == catalogue.GuardID() { - if r["content"] != catalogue.AsGuard([]int{5432}) { - t.Fatalf("the taken store's guard is:\n%s", r["content"]) - } - return - } - } - t.Fatal("a taken store is not guarded") -} // novox/hq ADR 0038 and 0100: a module may publish a port the long way — `2222:22`, because the // machine's own ssh daemon holds 22 — and say it listens on the machine side of that mapping, @@ -287,79 +76,6 @@ func TestTheGuardIsSentForTakenModulesOnly(t *testing.T) { // It reached none of them. The setting was refused outright for naming the machine side — so a // module's port could not be put back where the machine it replaces had it, and, worse, the // node's every push failed for as long as the setting existed. -func TestTheMachineSideOfAMappingIsMovedEverywhereTheNumberIsUsed(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "forge", Version: "1", - Provides: []catalogue.Offer{{Name: "git-over-ssh", Scope: catalogue.ScopeMesh}}, - Serves: map[string]map[string]any{"git-over-ssh": {"port": 2222}}, - Listens: []catalogue.Listening{{Port: 2222, From: catalogue.FromMesh}}, - Resources: []map[string]any{{"id": "server", "type": "container", "name": "forge", - "ports": []any{"2222:22"}, - "image": "registry.example/forge@sha256:" + strings.Repeat("c", 64)}}}) - register(t, open, catalogue.Manifest{Module: "app", Version: "1", - Requires: []string{"git-over-ssh"}}) - if err := open.inventory.SetAdopted(ctx, "anchor", true); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "anchor", "forge"); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", "app"); err != nil { - t.Fatal(err) - } - if err := open.inventory.Take(ctx, "anchor", "forge"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "anchor", "forge", - map[string]any{catalogue.PortsSetting: map[string]any{"2222": 222}}); err != nil { - t.Fatal(err) - } - - resources := composed(t, open, "anchor").Resources - var container, opening map[string]any - for _, r := range resources { - switch r["id"] { - case "forge.server": - container = r - case catalogue.OpeningID("tcp", 222, catalogue.PathForwarded): - opening = r - } - if id, _ := r["id"].(string); strings.HasPrefix(id, "adoption.opening-tcp-2222-") { - t.Errorf("the adopted node is told to open the port the forge was moved off: %s", id) - } - } - if container == nil || !reflect.DeepEqual(container["ports"], []any{"222:22"}) { - t.Fatalf("the forge's container publishes %v", container["ports"]) - } - if opening == nil || opening["to"] != 22 || opening["from"] != catalogue.OpeningFromMesh { - t.Fatalf("no opening for the port this node gave the forge: %v", opening) - } - var guard map[string]any - for _, r := range resources { - if r["id"] == catalogue.GuardID() { - guard = r - } - } - if guard == nil || guard["content"] != catalogue.AsGuard([]int{222}) { - t.Fatalf("the guard does not refuse the port the forge is on:\n%v", guard["content"]) - } - - // And the consumer on the other machine dials the same number. - plan, _, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - var told any - for _, n := range plan.Needs { - if n.Name == "git-over-ssh" { - told = n.Serves["port"] - } - } - if told != 222 { - t.Fatalf("the consumer is told the forge answers on %v", told) - } -} func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) { // The host refuses an empty body unless told the emptiness is meant (novox/hq issue 127). @@ -387,57 +103,3 @@ func TestAnEmptyDeclarationSaysOwnsNothing(t *testing.T) { // (novox/hq ADR 0163, rule 6): stored while it composed, a setting whose definition then moved from // under it leaves that module out of the declaration — said in the envelope, so the host keeps the // module's things — and the machine is told everything else. -func TestADefinitionMovingUnderAStoredSettingLeavesThatModuleOutNotTheMachine(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - web := helloWeb() - web.Resources[1]["ports"] = []any{"8080"} - register(t, open, web) - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "conf", "type": "file", "path": "/etc/notes.conf", "content": "x"}}}) - for _, m := range []string{"hello-web", "notes"} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - // Judged where it is stored: a port the module does not publish is refused by name. - err := open.inventory.SetSettings(ctx, "laptop", "hello-web", - map[string]any{catalogue.PortsSetting: map[string]any{"9999": 10000}}) - if err == nil || !strings.Contains(err.Error(), "hello-web on laptop") || !strings.Contains(err.Error(), "9999") { - t.Fatalf("an impossible setting was stored: %v", err) - } - if err := open.inventory.SetSettings(ctx, "laptop", "hello-web", - map[string]any{catalogue.PortsSetting: map[string]any{"8080": 10000}}); err != nil { - t.Fatal(err) - } - if declared := composed(t, open, "laptop"); len(declared.LeftOut) != 0 { - t.Fatalf("a setting that composes left a module out: %v", declared.LeftOut) - } - - // The definition moves: the container publishes another port now. - web.Version = "2" - web.Resources[1]["ports"] = []any{"9090"} - register(t, open, web) - declared := composed(t, open, "laptop") - if len(declared.LeftOut) != 1 || declared.LeftOut[0] != "hello-web" { - t.Fatalf("hello-web is not left out: %v", declared.LeftOut) - } - if !strings.Contains(declared.leftOutWhy["hello-web"], "no container of its publishes 8080") { - t.Fatalf("why it was left out is not said: %v", declared.leftOutWhy) - } - if hasID(declared.Resources, "hello-web.server") || !hasID(declared.Resources, "notes.conf") { - t.Fatalf("the machine was not told everything else: %v", declared.Resources) - } - body, err := declared.Body() - if err != nil { - t.Fatal(err) - } - var env map[string]any - if err := json.Unmarshal(body, &env); err != nil { - t.Fatal(err) - } - left, _ := env["left_out"].([]any) - if len(left) != 1 || left[0] != "hello-web" { - t.Fatalf("the envelope does not say what was left out: %v", env) - } -} diff --git a/cmd/mesh-controller/sent_required_by_integration_test.go b/cmd/mesh-controller/sent_required_by_integration_test.go new file mode 100644 index 00000000..5cef69b8 --- /dev/null +++ b/cmd/mesh-controller/sent_required_by_integration_test.go @@ -0,0 +1,104 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + client := aClientModule() + moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) + moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} + kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} + desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} + for _, m := range []catalogue.Manifest{client, moved, kept, desk} { + register(t, open, m) + } + for _, m := range []string{client.Module, desk.Module} { + if _, err := assign(ctx, open, "laptop", m); err != nil { + t.Fatal(err) + } + } + if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, + SaidAt: time.Now()}); err != nil { + t.Fatal(err) + } + + plan, settings, err := planFor(ctx, open, "laptop") + if err != nil { + t.Fatal(err) + } + if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { + t.Fatalf("the bots are on laptop because %v", plan.Because) + } + declared, err := declarationFor(ctx, open, "laptop", plan, settings) + if err != nil { + t.Fatal(err) + } + var movedToken, keptToken, movedConf map[string]any + for _, r := range declared.Resources { + switch r["id"] { + case "moved-bot.needs-token": + movedToken = r + case "kept-bot.needs-token": + keptToken = r + case "moved-bot.conf": + movedConf = r + } + } + if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) + } + if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { + t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) + } + if keptToken == nil || keptToken["sealed"] == nil { + t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) + } + if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { + t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) + } + if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { + t.Fatalf("the send hands %v", declared.plans) + } + + record, err := inv.NodeByName(ctx, "laptop") + if err != nil { + t.Fatal(err) + } + if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", + Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { + t.Fatal(err) + } + inv.SetProvisioningPlans(inv.SentProvisioningPlans) + mirror, err := inv.AssignmentsMirror(ctx) + if err != nil { + t.Fatal(err) + } + var value inventory.AssignmentMirrored + if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { + t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) + } + if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || + len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || + len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { + t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) + } + if _, has := mirror["laptop.kept-bot"]; has { + t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) + } +} diff --git a/cmd/mesh-controller/sent_required_by_test.go b/cmd/mesh-controller/sent_required_by_test.go index 68553609..7f611ca2 100644 --- a/cmd/mesh-controller/sent_required_by_test.go +++ b/cmd/mesh-controller/sent_required_by_test.go @@ -1,15 +1,10 @@ package main import ( - "encoding/json" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // A send keeps, beside each handed module's plan, what requires a module only pulled in (issue 765), so the @@ -34,92 +29,3 @@ func TestASendKeepsWhatRequiresAPulledInModule(t *testing.T) { // node running a client, a moved module that nothing assigned there, required by one that is, has its files and its // own secret handed like an assigned one's, no secret of it made or sealed by the controller, and its plan mirrored // under its own key, saying what requires it. A module not moved, pulled in the same way, is sent as before. -func TestAModulePulledInIsHandedOnAMachineThatCanTakeItAndMirrored(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - client := aClientModule() - moved, kept := aBot("moved-bot", true), aBot("kept-bot", false) - moved.Provides = []catalogue.Offer{{Name: "moved-bot-api"}} - kept.Provides = []catalogue.Offer{{Name: "kept-bot-api"}} - desk := catalogue.Manifest{Module: "desk", Version: "1", Requires: []string{"moved-bot-api", "kept-bot-api"}} - for _, m := range []catalogue.Manifest{client, moved, kept, desk} { - register(t, open, m) - } - for _, m := range []string{client.Module, desk.Module} { - if _, err := assign(ctx, open, "laptop", m); err != nil { - t.Fatal(err) - } - } - if err := issueProvisionerClient(ctx, inv, client, "laptop", broker.Broker{Fingerprint: "ab"}, "10.0.0.1:4222"); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordHealth(ctx, inventory.NodeHealth{Node: "laptop", Contract: link.HandedContract, - SaidAt: time.Now()}); err != nil { - t.Fatal(err) - } - - plan, settings, err := planFor(ctx, open, "laptop") - if err != nil { - t.Fatal(err) - } - if plan.RequiredBy("moved-bot") != "desk" || plan.RequiredBy("kept-bot") != "desk" { - t.Fatalf("the bots are on laptop because %v", plan.Because) - } - declared, err := declarationFor(ctx, open, "laptop", plan, settings) - if err != nil { - t.Fatal(err) - } - var movedToken, keptToken, movedConf map[string]any - for _, r := range declared.Resources { - switch r["id"] { - case "moved-bot.needs-token": - movedToken = r - case "kept-bot.needs-token": - keptToken = r - case "moved-bot.conf": - movedConf = r - } - } - if movedToken == nil || movedToken["sealed"] != nil || movedToken["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's secret file is sent as %v: placed, with no value", movedToken) - } - if movedConf == nil || movedConf["content"] != nil || movedConf["owner-at-runtime"] != "owned" { - t.Fatalf("the pulled-in moved module's templated file is sent as %v: placed, with no text", movedConf) - } - if keptToken == nil || keptToken["sealed"] == nil { - t.Fatalf("the pulled-in module not moved has its secret sent as %v: sealed, as before", keptToken) - } - if _, held, err := inv.ModuleSecretIfIssued(ctx, "laptop", "moved-bot", "token"); err != nil || held { - t.Fatalf("the send made the pulled-in handed module's secret in the controller's store (held %v, %v)", held, err) - } - if _, has := declared.plans["kept-bot"]; has || len(declared.plans) != 1 { - t.Fatalf("the send hands %v", declared.plans) - } - - record, err := inv.NodeByName(ctx, "laptop") - if err != nil { - t.Fatal(err) - } - if err := inv.RecordSentWith(ctx, record.ID, "first", nil, 0, 0, inventory.Send{Digest: "first", - Provisioning: sentRecordOf(ctx, declared).plans}); err != nil { - t.Fatal(err) - } - inv.SetProvisioningPlans(inv.SentProvisioningPlans) - mirror, err := inv.AssignmentsMirror(ctx) - if err != nil { - t.Fatal(err) - } - var value inventory.AssignmentMirrored - if err := json.Unmarshal(mirror["laptop.moved-bot"], &value); err != nil { - t.Fatalf("no value for the pulled-in module: %s (%v)", mirror["laptop.moved-bot"], err) - } - if value.RequiredBy != "desk" || !value.Assigned.IsZero() || value.Provisioning == nil || - len(value.Provisioning.Files) != 1 || value.Provisioning.Files[0].Path != "/etc/moved-bot/bot.conf" || - len(value.Provisioning.Secrets) != 1 || value.Provisioning.Secrets[0].Name != "token" { - t.Fatalf("the pulled-in module is mirrored as %s", mirror["laptop.moved-bot"]) - } - if _, has := mirror["laptop.kept-bot"]; has { - t.Fatalf("a pulled-in module not handed is mirrored: %s", mirror["laptop.kept-bot"]) - } -} diff --git a/cmd/mesh-controller/sequence_integration_test.go b/cmd/mesh-controller/sequence_integration_test.go new file mode 100644 index 00000000..85c5475f --- /dev/null +++ b/cmd/mesh-controller/sequence_integration_test.go @@ -0,0 +1,41 @@ +//go:build integration + +package main + +import "testing" + +func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) { + open := aMesh(t) + record, err := open.inventory.NodeByName(t.Context(), "anchor") + if err != nil { + t.Fatal(err) + } + // Sent nothing since numbering existed: what it would be sent is composed with zero, which is + // not on the wire, which is what it was actually sent. + if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 { + t.Fatalf("a fresh node reads sequence %d, %v", n, err) + } + first, err := open.inventory.NextSequence(t.Context(), record.ID) + if err != nil { + t.Fatal(err) + } + second, err := open.inventory.NextSequence(t.Context(), record.ID) + if err != nil { + t.Fatal(err) + } + if first != 1 || second != 2 { + t.Fatalf("two sends were numbered %d and %d", first, second) + } + // And the read path sees the last one taken, so the comparison composes what was sent. + if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 { + t.Fatalf("after two sends the node reads sequence %d, %v", n, err) + } + // Another node counts on its own. + other, err := open.inventory.NodeByName(t.Context(), "laptop") + if err != nil { + t.Fatal(err) + } + if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 { + t.Fatalf("a second node's first send was numbered %d, %v", n, err) + } +} diff --git a/cmd/mesh-controller/sequence_test.go b/cmd/mesh-controller/sequence_test.go index 5d2dbd67..c86f9140 100644 --- a/cmd/mesh-controller/sequence_test.go +++ b/cmd/mesh-controller/sequence_test.go @@ -39,39 +39,3 @@ func TestAnUnnumberedSendIsByteForByteWhatItWasBefore(t *testing.T) { t.Fatalf("a send numbered zero put a sequence on the wire: %s", body) } } - -func TestEachSendToANodeIsOneHigherAndReadable(t *testing.T) { - open := aMesh(t) - record, err := open.inventory.NodeByName(t.Context(), "anchor") - if err != nil { - t.Fatal(err) - } - // Sent nothing since numbering existed: what it would be sent is composed with zero, which is - // not on the wire, which is what it was actually sent. - if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 0 { - t.Fatalf("a fresh node reads sequence %d, %v", n, err) - } - first, err := open.inventory.NextSequence(t.Context(), record.ID) - if err != nil { - t.Fatal(err) - } - second, err := open.inventory.NextSequence(t.Context(), record.ID) - if err != nil { - t.Fatal(err) - } - if first != 1 || second != 2 { - t.Fatalf("two sends were numbered %d and %d", first, second) - } - // And the read path sees the last one taken, so the comparison composes what was sent. - if n, err := open.inventory.Sequence(t.Context(), record.ID); err != nil || n != 2 { - t.Fatalf("after two sends the node reads sequence %d, %v", n, err) - } - // Another node counts on its own. - other, err := open.inventory.NodeByName(t.Context(), "laptop") - if err != nil { - t.Fatal(err) - } - if n, err := open.inventory.NextSequence(t.Context(), other.ID); err != nil || n != 1 { - t.Fatalf("a second node's first send was numbered %d, %v", n, err) - } -} diff --git a/cmd/mesh-controller/served_own_names_test.go b/cmd/mesh-controller/served_own_names_test.go index 3b7866ab..430d9f0f 100644 --- a/cmd/mesh-controller/served_own_names_test.go +++ b/cmd/mesh-controller/served_own_names_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/settings_effective_integration_test.go b/cmd/mesh-controller/settings_effective_integration_test.go new file mode 100644 index 00000000..e23f7a4a --- /dev/null +++ b/cmd/mesh-controller/settings_effective_integration_test.go @@ -0,0 +1,38 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestPreferencesListedFromTheStores(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Settings: map[string]catalogue.SettingDeclaration{ + "font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"}, + }, + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "font = ${setting:font-size}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + if err := open.inventory.SetSettings(ctx, "laptop", "notes", map[string]any{"font-size": float64(16)}); err != nil { + t.Fatal(err) + } + all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) }) + if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") || + !strings.Contains(all, "font-size, default 10: readable at 100 DPI") { + t.Fatalf("the listing:\n%s", all) + } + if got := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences", "--node", "anchor"}) }); got != "no module on anchor declares a preference\n" { + t.Fatalf("a machine without the module:\n%s", got) + } + if err := settingsCommand(ctx, []string{"preferences", "--node", "nowhere"}); err == nil { + t.Fatal("a machine the mesh does not know was answered") + } +} diff --git a/cmd/mesh-controller/settings_effective_test.go b/cmd/mesh-controller/settings_effective_test.go index f248e2b4..23e20ab1 100644 --- a/cmd/mesh-controller/settings_effective_test.go +++ b/cmd/mesh-controller/settings_effective_test.go @@ -80,30 +80,3 @@ func TestPreferencesSayEachMachinesValueAndItsSource(t *testing.T) { // The listing over the real stores: each machine's value with its source; a machine names only the // modules on it; a machine the mesh does not know is refused (novox/hq ADR 0262). -func TestPreferencesListedFromTheStores(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Settings: map[string]catalogue.SettingDeclaration{ - "font-size": {Kind: catalogue.KindPreference, Default: float64(10), Why: "readable at 100 DPI"}, - }, - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", - "content": "font = ${setting:font-size}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - if err := open.inventory.SetSettings(ctx, "laptop", "notes", map[string]any{"font-size": float64(16)}); err != nil { - t.Fatal(err) - } - all := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences"}) }) - if !strings.Contains(all, "notes (on laptop)") || !strings.Contains(all, "laptop: 16 (the node)") || - !strings.Contains(all, "font-size, default 10: readable at 100 DPI") { - t.Fatalf("the listing:\n%s", all) - } - if got := stdoutOf(t, func() error { return settingsCommand(ctx, []string{"preferences", "--node", "anchor"}) }); got != "no module on anchor declares a preference\n" { - t.Fatalf("a machine without the module:\n%s", got) - } - if err := settingsCommand(ctx, []string{"preferences", "--node", "nowhere"}); err == nil { - t.Fatal("a machine the mesh does not know was answered") - } -} diff --git a/cmd/mesh-controller/settings_walk_integration_test.go b/cmd/mesh-controller/settings_walk_integration_test.go new file mode 100644 index 00000000..90bfe9ef --- /dev/null +++ b/cmd/mesh-controller/settings_walk_integration_test.go @@ -0,0 +1,439 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "notes", Version: "1", + Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", + "content": "x = ${setting:x}\n"}}}) + if _, err := assign(ctx, open, "laptop", "notes"); err != nil { + t.Fatal(err) + } + // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). + if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { + t.Fatal(err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { + t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) + } + set := setLayerIn(open) + now := time.Now() + values := map[string]any{"x": "1"} + p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { + t.Errorf("the warrant's words: %q", words) + } + plans, err := open.inventory.OpenPlans(ctx) + if err != nil || len(plans) != 1 { + t.Fatalf("open walks: %+v %v", plans, err) + } + w := plans[0] + if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || + !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || + !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { + t.Fatalf("the settings walk: %+v %+v", w, w.Release) + } + + // A machine's layer where the module does not run: set, and nothing to send. + q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, + Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} + words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") + if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { + t.Fatalf("a layer nobody runs: %q %v", words, err) + } + if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { + t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) + } + + // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. + was := openSettingsWalk + openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { + return "", fmt.Errorf("the plans are not to be had") + } + t.Cleanup(func() { openSettingsWalk = was }) + again := map[string]any{"x": "2"} + byHand := map[string]any{"x": "0"} + r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, + From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} + if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ + "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { + t.Errorf("an unopened walk: %v", err) + } +} + +func aSettledMesh(t *testing.T) *walkMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, + heard: map[string]bool{"anchor": true, "laptop": true}} + withConditionsInMemory(t) + was := doctorFrom + doctorFrom = nil + t.Cleanup(func() { doctorFrom = was }) + old := time.Now().Add(-2 * time.Hour) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { + t.Fatal(err) + } + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, "app"); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { + t.Fatal(err) + } + } + wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves + settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } + machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { + modules, err := open.inventory.Assigned(ctx, node) + if err != nil { + return nil, err + } + sent, known, err := open.inventory.SentBuilds(ctx, node) + if err != nil { + return nil, err + } + return f.moves(node, modules, sent, known, all), nil + } + releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } + n := 0 + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + w.sent = append(w.sent, append([]string(nil), names...)) + w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + for _, node := range names { + modules, _ := open.inventory.Assigned(ctx, node) + carried := map[string]string{} + for _, m := range modules { + carried[m] = current[m].Commit + } + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + if w.broken[r.Node] { + r.Outcome = inventory.OutcomeFailed + } + f.reports[r.Node] = r + } + return f, nil + } + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { + machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather + settingsDataMoves = wasData + gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound + }) + return w +} + +func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) + if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { + t.Fatalf("%q %v", words, err) + } + advancePlans(ctx, w.open) + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) + } + p := w.walk(t) + if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { + t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) + } + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v", w.sent) + } + p = w.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || + p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { + t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) + } + // It said its start and its end, so the operator is told once (ADR 0308). + if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { + t.Fatalf("the walk's start and end are not marked: %+v", p.Times) + } + // Nothing waits, and no walk of waiting builds opened beside it. + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { + t.Fatalf("%+v %v", r, err) + } +} + +func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + w.broken["anchor"] = true + if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { + t.Fatal(err) + } + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || + !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) + } + if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { + t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) + } + // A build that waits for a gate is still walked: the failed settings walk holds nothing back. + w.broken["anchor"] = false + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) + } +} + +func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + gateEvery = 0 + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { + t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) + } + // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") + if err != nil || kept["app"] != "c1" { + t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) + } + if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { + t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) + } + // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. + f, err := readMoveFacts(ctx, inv) + if err != nil { + t.Fatal(err) + } + waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) + if len(waiting) != 1 || waiting[0].Module != "app" { + t.Fatalf("anchor waits for %+v, want app's c2", waiting) + } + if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { + t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) + } +} + +func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + gateEvery = 0 + w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, w.open) + } + p := w.walk(t) + if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || + !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || + !strings.Contains(p.Note, "push laptop --move app") { + t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) + } + + v := aSettledMesh(t) + v.heard = map[string]bool{"laptop": true} + gateEvery = 0 + if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { + t.Fatal(err) + } + for i := 0; i < 4; i++ { + advancePlans(ctx, v.open) + } + p = v.walk(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || + !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { + t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) + } + + u := aSettledMesh(t) + u.heard = map[string]bool{} + if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, u.open) + if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || + !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { + t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) + } +} + +func TestACrossingSendSupersedesASettingsGate(t *testing.T) { + judgeLeavesAsTheCore(t) + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, w.open) + p := w.walk(t) + if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { + t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) + } + // Another walk sends anchor app's c2 while the settings gate judges there. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { + t.Fatal(err) + } + if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", + Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { + t.Fatal(err) + } + gateEvery = 0 + advancePlans(ctx, w.open) + p = w.walk(t) + if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || + !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { + t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) + } + if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) + } +} + +func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { + w := aSettledMesh(t) + ctx := t.Context() + inv := w.open.inventory + // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. + if _, err := inv.AddNode(ctx, "fresh"); err != nil { + t.Fatal(err) + } + if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { + t.Fatalf("a settings send to a machine never sent: %v", err) + } + w.heard = map[string]bool{"laptop": true} + settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { + return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) + } + if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, w.open) + p := w.walk(t) + if p.State != inventory.PlanFailed || len(w.sent) != 0 || + !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || + !strings.Contains(p.Note, "not heard from, left as they were: anchor") { + t.Fatalf("the walk is %s: %s", p.State, p.Note) + } + // It is not open, so a build waiting for a gate is walked. + now := time.Now().Add(-time.Minute) + manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) + if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", + Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, + Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ + Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { + t.Fatal(err) + } + if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { + t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) + } +} diff --git a/cmd/mesh-controller/settings_walk_test.go b/cmd/mesh-controller/settings_walk_test.go index 45df338a..2d9e0c18 100644 --- a/cmd/mesh-controller/settings_walk_test.go +++ b/cmd/mesh-controller/settings_walk_test.go @@ -2,17 +2,11 @@ package main import ( "context" - "encoding/json" - "errors" - "fmt" "reflect" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" ) // novox/hq ADR 0344, issue 641: the operator's one Approve on a settings proposal is the act that makes it take @@ -108,71 +102,6 @@ func TestTheAskSaysAnApprovedLayerIsSentAtOnce(t *testing.T) { // The warrant sets the layer and opens the walk that sends it, naming the machines it reaches; a layer for a // machine that does not run the module opens none and says so. -func TestAnApprovedLayerOpensTheWalkThatSendsIt(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "notes", Version: "1", - Resources: []map[string]any{{"id": "rc", "type": "file", "path": "/etc/notes.conf", "mode": "0644", - "content": "x = ${setting:x}\n"}}}) - if _, err := assign(ctx, open, "laptop", "notes"); err != nil { - t.Fatal(err) - } - // A layer set by hand opens no walk: it still waits for a push (ADR 0344 §7). - if err := atTheTerminal(t, "settings", "set", "notes", `{"x":"0"}`, "--node", "laptop"); err != nil { - t.Fatal(err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 0 { - t.Fatalf("a layer set at the terminal opened %d walk(s)", len(plans)) - } - set := setLayerIn(open) - now := time.Now() - values := map[string]any{"x": "1"} - p := settingsProposal{Module: "notes", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err := set(ctx, p, "s9", "approved by the operator (ask s9)") - if err != nil { - t.Fatal(err) - } - if !strings.Contains(words, "+ x (ask s9); settings-") || !strings.Contains(words, "sends it to laptop, one machine at a time") { - t.Errorf("the warrant's words: %q", words) - } - plans, err := open.inventory.OpenPlans(ctx) - if err != nil || len(plans) != 1 { - t.Fatalf("open walks: %+v %v", plans, err) - } - w := plans[0] - if w.Release == nil || w.Release.Settings == nil || w.Release.Settings.Module != "notes" || w.Release.Settings.Ask != "s9" || - !reflect.DeepEqual(w.Release.Order, []string{"laptop"}) || w.Repository != "the settings of notes on the whole mesh" || - !reflect.DeepEqual(w.Tiers, [][]string{{"notes"}}) { - t.Fatalf("the settings walk: %+v %+v", w, w.Release) - } - - // A machine's layer where the module does not run: set, and nothing to send. - q := settingsProposal{Module: "notes", Node: "anchor", Values: values, From: "g14/claude-code", At: now, - Digest: layerDigest(values), BeforeDigest: layerDigest(nil)} - words, err = set(ctx, q, "s10", "approved by the operator (ask s10)") - if err != nil || !strings.Contains(words, "no machine runs notes there, so there is nothing to send") { - t.Fatalf("a layer nobody runs: %q %v", words, err) - } - if plans, _ := open.inventory.OpenPlans(ctx); len(plans) != 1 { - t.Fatalf("a walk was opened for a machine that does not run the module: %d open", len(plans)) - } - - // A walk that cannot be opened is the act's failure, said with the push that sends it by hand. - was := openSettingsWalk - openSettingsWalk = func(context.Context, *stores, inventory.PlanSettings) (string, error) { - return "", fmt.Errorf("the plans are not to be had") - } - t.Cleanup(func() { openSettingsWalk = was }) - again := map[string]any{"x": "2"} - byHand := map[string]any{"x": "0"} - r := settingsProposal{Module: "notes", Node: "laptop", Values: again, Before: byHand, HadLayer: true, - From: "g14/claude-code", At: now, Digest: layerDigest(again), BeforeDigest: layerDigest(byHand)} - if _, err := set(ctx, r, "s11", "approved"); err == nil || !strings.Contains(err.Error(), "is set, and the walk that "+ - "sends it could not be opened: the plans are not to be had — `push laptop` sends it") { - t.Errorf("an unopened walk: %v", err) - } -} // walkMesh is a mesh where `app` runs c1 on anchor and laptop and nothing waits for a gate: every send applied at // once, recorded, and judged from the reports, with a machine named broken failing what it was sent. @@ -186,105 +115,6 @@ type walkMesh struct { heard map[string]bool } -func aSettledMesh(t *testing.T) *walkMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - w := &walkMesh{open: open, broken: map[string]bool{}, moves: map[string][]dataMove{}, - heard: map[string]bool{"anchor": true, "laptop": true}} - withConditionsInMemory(t) - was := doctorFrom - doctorFrom = nil - t.Cleanup(func() { doctorFrom = was }) - old := time.Now().Add(-2 * time.Hour) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c1", Module: "app", Commit: "c1", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: old, At: old, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-1"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c1", Head: "c1", Asked: old}); err != nil { - t.Fatal(err) - } - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, "app"); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n, map[string]string{"app": "c1"}); err != nil { - t.Fatal(err) - } - } - wasMoves, wasHeard, wasSend, wasGather, wasData := machineMoves, releaseHeard, sendRollout, gatherGateFacts, settingsDataMoves - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { return w.moves[node], nil } - machineMoves = func(ctx context.Context, open *stores, f moveFacts, node string, all bool) ([]inventory.CarriedMove, error) { - modules, err := open.inventory.Assigned(ctx, node) - if err != nil { - return nil, err - } - sent, known, err := open.inventory.SentBuilds(ctx, node) - if err != nil { - return nil, err - } - return f.moves(node, modules, sent, known, all), nil - } - releaseHeard = func(context.Context, *stores) (map[string]bool, error) { return w.heard, nil } - n := 0 - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - w.sent = append(w.sent, append([]string(nil), names...)) - w.kept = append(w.kept, len(names) == 1 && settingsKeptOn(ctx, names[0])) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - for _, node := range names { - modules, _ := open.inventory.Assigned(ctx, node) - carried := map[string]string{} - for _, m := range modules { - carried[m] = current[m].Commit - } - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, carried); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - if w.broken[r.Node] { - r.Outcome = inventory.OutcomeFailed - } - f.reports[r.Node] = r - } - return f, nil - } - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { - machineMoves, releaseHeard, sendRollout, gatherGateFacts = wasMoves, wasHeard, wasSend, wasGather - settingsDataMoves = wasData - gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound - }) - return w -} - func (w *walkMesh) walk(t *testing.T) inventory.Plan { t.Helper() plans, err := w.open.inventory.RecentPlans(t.Context(), 10) @@ -302,271 +132,21 @@ func (w *walkMesh) walk(t *testing.T) inventory.Plan { // The walk sends the first machine alone and judges the module there before it sends the next, and ends // delivered on both; nothing waited for a gate, so each send carried the layer alone. -func TestASettingsWalkSendsOneMachineAtATimeEachJudged(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - words, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s1", Change: "~ x"}) - if err != nil || !strings.Contains(words, "sends it to anchor, laptop") { - t.Fatalf("%q %v", words, err) - } - advancePlans(ctx, w.open) - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine alone, before it is judged", w.sent) - } - p := w.walk(t) - if p.Release.Gate == nil || p.Release.Next != 0 || !strings.Contains(p.Note, "judging app on anchor") { - t.Fatalf("the walk after its first send: %s %+v", p.Note, p.Release) - } - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v", w.sent) - } - p = w.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Done, []string{"anchor", "laptop"}) || - p.Outcome() != inventory.WalkDelivered || !strings.Contains(p.Note, "delivered on anchor, laptop") { - t.Fatalf("the walk is %s: %s %+v", p.State, p.Note, p.Release) - } - // It said its start and its end, so the operator is told once (ADR 0308). - if p.Times == nil || p.Times.Started == nil || p.Times.Ended == nil { - t.Fatalf("the walk's start and end are not marked: %+v", p.Times) - } - // Nothing waits, and no walk of waiting builds opened beside it. - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r != nil { - t.Fatalf("%+v %v", r, err) - } -} // A machine that fails what it was sent stops the walk there: the next is not sent, the layer stays, and the // failure holds back no walk of waiting builds. -func TestAFailedSettingsWalkStopsAndHoldsNothingBack(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - w.broken["anchor"] = true - if err := inv.KeepSettings(ctx, "", "app", map[string]any{"x": "1"}); err != nil { - t.Fatal(err) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s2"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "the settings of app on the whole mesh failed its gate on anchor") || - !strings.Contains(p.Note, "the layer stays as the operator approved it") || !strings.Contains(p.Note, "not sent: laptop") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: anchor alone, nothing to laptop", w.sent) - } - if layer, has, err := inv.Layer(ctx, "", "app"); err != nil || !has || layer["x"] != "1" { - t.Fatalf("the layer after the failure: %v %v %v", layer, has, err) - } - // A build that waits for a gate is still walked: the failed settings walk holds nothing back. - w.broken["anchor"] = false - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held back by a failed settings walk: %+v %v", r, err) - } -} // Each send of a settings walk keeps every build of its machine: the layer goes, and a build held for a person or // waiting for a gate stays where it is (ADR 0344 §3). -func TestASettingsWalkSendsTheLayerAloneEveryBuildKept(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - gateEvery = 0 - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s3"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - if !reflect.DeepEqual(w.kept, []bool{true, true}) || w.walk(t).State != inventory.PlanDone { - t.Fatalf("the sends kept every build: %v; the walk: %s", w.kept, w.walk(t).Note) - } - // What such a send keeps, on the store: a rolled-out build waiting there is kept at the one last sent. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - kept, err := recordedKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor") - if err != nil || kept["app"] != "c1" { - t.Fatalf("a settings send to anchor keeps %v (%v), want app at c1", kept, err) - } - if kept, _ := recordedKept(ctx, w.open, "anchor"); kept["app"] != "" { - t.Fatalf("an ordinary send keeps a rolled-out build: %v", kept) - } - // And so the ungated build waiting on anchor is no move of the settings send's: nothing for a gate to see. - f, err := readMoveFacts(ctx, inv) - if err != nil { - t.Fatal(err) - } - waiting, _ := machineMoves(ctx, w.open, f, "anchor", false) - if len(waiting) != 1 || waiting[0].Module != "app" { - t.Fatalf("anchor waits for %+v, want app's c2", waiting) - } - if moved, err := withoutKept(keepingBuildsForSettings(ctx, "anchor"), w.open, "anchor", waiting); err != nil || len(moved) != 0 { - t.Fatalf("a settings send to anchor would move %+v (%v)", moved, err) - } -} // A machine whose send would move a running module's data stops the walk there, failed and said with the push that // sends it on a person's word (ADR 0217 §3); one not heard from is passed over and named; and a walk that reached // no machine is not said delivered. -func TestASettingsWalkStopsBeforeADataMoveAndPassesOverAMachineAway(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - gateEvery = 0 - w.moves["laptop"] = []dataMove{{Container: "app-server", Inside: "/data", From: "/srv/a", To: "/srv/b"}} - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s4"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, w.open) - } - p := w.walk(t) - if p.State != inventory.PlanFailed || !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) || - !strings.Contains(p.Note, "not sent to laptop: a running module's data would move there") || - !strings.Contains(p.Note, "push laptop --move app") { - t.Fatalf("sent %v; the walk is %s: %s", w.sent, p.State, p.Note) - } - - v := aSettledMesh(t) - v.heard = map[string]bool{"laptop": true} - gateEvery = 0 - if _, err := openSettingsWalk(ctx, v.open, inventory.PlanSettings{Module: "app", Ask: "s5"}); err != nil { - t.Fatal(err) - } - for i := 0; i < 4; i++ { - advancePlans(ctx, v.open) - } - p = v.walk(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(p.Release.Skipped, []string{"anchor"}) || - !strings.Contains(p.Note, "delivered on laptop; not heard from, left as they were, their next send carries it: anchor") { - t.Fatalf("the walk with anchor away is %s: %s", p.State, p.Note) - } - - u := aSettledMesh(t) - u.heard = map[string]bool{} - if _, err := openSettingsWalk(ctx, u.open, inventory.PlanSettings{Module: "app", Ask: "s6"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, u.open) - if p := u.walk(t); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed || - !strings.Contains(p.Note, "was sent to no machine") || len(u.sent) != 0 { - t.Fatalf("a walk that reached nobody is %s: %s", p.State, p.Note) - } -} // The gate keeps the module's build its send kept, so a crossing send — a merge's walk sending the machine a new // build of the module while the settings gate judges there — supersedes it, and its report is never read as the // settings send's (issue 641 review, finding 1). -func TestACrossingSendSupersedesASettingsGate(t *testing.T) { - judgeLeavesAsTheCore(t) - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s7"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if g := p.Release.Gate; g == nil || g.To != "c1" || g.From != "c1" { - t.Fatalf("the settings gate judges no build: %+v", p.Release.Gate) - } - // Another walk sends anchor app's c2 while the settings gate judges there. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, nodeID(t, w.open, "anchor"), "d-merge", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - if _, err := inv.RecordDoing(ctx, nodeID(t, w.open, "anchor"), inventory.Doing{Node: "anchor", - Outcome: inventory.OutcomeApplied, Declared: "d-merge", Applied: 1, At: time.Now()}); err != nil { - t.Fatal(err) - } - gateEvery = 0 - advancePlans(ctx, w.open) - p = w.walk(t) - if p.State != inventory.PlanSuperseded || p.Release.Gate == nil || p.Release.Gate.Verdict != inventory.GateSuperseded || - !strings.Contains(p.Note, "superseded on anchor") || len(p.Release.Done) != 0 { - t.Fatalf("the settings walk after a crossing send is %s: %s %+v", p.State, p.Note, p.Release) - } - if !reflect.DeepEqual(w.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: laptop was sent after a superseded judging", w.sent) - } -} // A send refusal that cannot clear by itself — what the machine was last sent is not known, or a build it runs is // gone — fails the walk with why, rather than keeping it open and holding the walk of waiting builds; and a // failure names the machines passed over as away (issue 641 review, findings 2 and 3). -func TestASettingsWalkFailsOnARefusalThatCannotClear(t *testing.T) { - w := aSettledMesh(t) - ctx := t.Context() - inv := w.open.inventory - // What a send to a machine never sent can keep: nothing, and said as the refusal the walk fails on. - if _, err := inv.AddNode(ctx, "fresh"); err != nil { - t.Fatal(err) - } - if _, err := recordedKept(keepingBuildsForSettings(ctx, "fresh"), w.open, "fresh"); !errors.Is(err, errSettingsCannotKeep) { - t.Fatalf("a settings send to a machine never sent: %v", err) - } - w.heard = map[string]bool{"laptop": true} - settingsDataMoves = func(_ context.Context, _ *stores, node string) ([]dataMove, error) { - return nil, fmt.Errorf("%w: what %s was last sent is not known", errSettingsCannotKeep, node) - } - if _, err := openSettingsWalk(ctx, w.open, inventory.PlanSettings{Module: "app", Ask: "s8"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, w.open) - p := w.walk(t) - if p.State != inventory.PlanFailed || len(w.sent) != 0 || - !strings.Contains(p.Note, "not sent to laptop: the settings cannot be sent with every build kept: what laptop was last sent is not known") || - !strings.Contains(p.Note, "not heard from, left as they were: anchor") { - t.Fatalf("the walk is %s: %s", p.State, p.Note) - } - // It is not open, so a build waiting for a gate is walked. - now := time.Now().Add(-time.Minute) - manifest, _ := json.Marshal(catalogue.Manifest{Module: "app", Version: "1"}) - if err := inv.RecordBuild(ctx, inventory.Build{ID: "build-app-c2", Module: "app", Commit: "c2", - Repository: "novox/mesh-catalog", Path: "modules/app", Manifest: manifest, Asked: now, At: now, - Made: []inventory.Artifact{{Name: "x", Kind: "bundle", Reference: "sha256:app-2"}}}); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: "app", Version: "1"}, inventory.Source{ - Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/app", BuiltFrom: "c2", Head: "c2", Asked: now}); err != nil { - t.Fatal(err) - } - if r, err := releaseBacklog(ctx, w.open, ""); err != nil || r == nil { - t.Fatalf("the waiting build is held by the failed settings walk: %+v %v", r, err) - } -} diff --git a/cmd/mesh-controller/signals.go b/cmd/mesh-controller/signals.go index a0a5b560..e086e5d9 100644 --- a/cmd/mesh-controller/signals.go +++ b/cmd/mesh-controller/signals.go @@ -225,6 +225,15 @@ var signalsTable = []signalRow{ newest: func(f *signalFacts) time.Time { return newestOf(f.batches, func(b batchFacts) time.Time { return b.closed }) }}, + {Row: "S22", Signal: "the last member of a delivery group merges", Emitter: "mesh-delivery's group, held by the controller's merge window", + Trigger: "each batch held for a group with a member still to merge (novox/hq ADR 0418)", + Bound: "merge-window-at-most from the group's first merge: naming the member and why it did not merge; " + + "nothing of the group is delivered while it waits", + Kind: kindGroupNotMerged, Severity: conditions.Warning, Phase: 3, + needs: func(f *signalFacts) error { return f.plansErr }, watch: watchGroupsNotMerged, + newest: func(f *signalFacts) time.Time { + return newestOf(f.batches, func(b batchFacts) time.Time { return b.created }) + }}, {Row: "S20", Signal: "a send refused for the assignment generation it was composed from", Emitter: "node-engine", Trigger: "each refusal (novox/hq issue 234)", Bound: "none: raised at the first refusal, naming its sender, the generation it came from and the one the " + diff --git a/cmd/mesh-controller/signals_test.go b/cmd/mesh-controller/signals_test.go index cde92547..efa2ae7e 100644 --- a/cmd/mesh-controller/signals_test.go +++ b/cmd/mesh-controller/signals_test.go @@ -147,6 +147,19 @@ var suppressions = map[string]suppression{ atMost: f.now.Add(-61 * time.Second), closed: f.now.Add(-61 * time.Second)}} }, }, + // A batch held for a group whose last member has not merged by the window's maximum after the first (ADR 0418). + "S22": { + inside: func(f *signalFacts) { + f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", + holds: map[string]string{"feat/g": "group feat/g has a member still to merge: novox/two (ready)"}, + created: f.now.Add(-9 * time.Minute), atMostWindow: 10 * time.Minute}} + }, + past: func(f *signalFacts) { + f.batches = []batchFacts{{id: "plan-3", state: inventory.PlanAssembling, grouped: "novox/app@c0ffee11", + holds: map[string]string{"feat/g": "group feat/g has a member still to merge: novox/two (ready)"}, + created: f.now.Add(-11 * time.Minute), atMostWindow: 10 * time.Minute}} + }, + }, // A batch queued behind an open walk past that walk's bound, named. "S19": { inside: func(f *signalFacts) { diff --git a/cmd/mesh-controller/standing_integration_test.go b/cmd/mesh-controller/standing_integration_test.go new file mode 100644 index 00000000..625f0f15 --- /dev/null +++ b/cmd/mesh-controller/standing_integration_test.go @@ -0,0 +1,105 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/link" +) + +func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, catalogue.Manifest{Module: "idp", Version: "1", + Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}}) + if _, err := assign(ctx, open, "anchor", "idp"); err != nil { + t.Fatal(err) + } + kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} + since := time.Now().Add(-23 * time.Hour) + failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor", + Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected", + Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000} + if _, err := kept.Stood(ctx, failing); err != nil { + t.Fatal(err) + } + + asked, err := theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if asked.well() { + t.Fatal("a mesh whose identity provider fails a consumer reads as well") + } + said := printed(t, func() error { return printStatus(asked) }) + for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing", + "idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} { + if !strings.Contains(said, want) { + t.Fatalf("status does not say %q:\n%s", want, said) + } + } + if strings.Contains(said, "all doing what they were told") { + t.Fatalf("status said all well beside a failing provider:\n%s", said) + } + if !strings.HasPrefix(said, "1 open condition(s)") { + t.Fatalf("status does not lead with what is open:\n%s", said) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Conditions []conditions.Condition `json:"conditions"` + Failing []conditions.Condition `json:"failing"` + } + if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 || + !strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") { + t.Fatalf("the document does not carry it: %v\n%s", err, body) + } + // Both machines' `node show` name it: where the provider runs, and where the consumer is. + for _, node := range []string{"anchor", "laptop"} { + shown := printed(t, func() error { return showNode(ctx, open, node) }) + if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { + t.Fatalf("node show %s does not name it:\n%s", node, shown) + } + } + + // Recovered: gone, and the mesh may be well again as far as this is concerned. + failing.Failing = false + if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared { + t.Fatalf("%v %v", cleared, err) + } + asked, err = theThreeQuestions(ctx, open) + if err != nil { + t.Fatal(err) + } + if len(asked.conditions) != 0 { + t.Fatalf("a recovered consumer is still named: %+v", asked.conditions) + } +} + +func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} + if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true, + ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil { + t.Fatal(err) + } + all, err := conditionsFrom.Open(ctx) + if err != nil || len(all) != 1 { + t.Fatalf("%+v %v", all, err) + } + if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil { + t.Fatal(err) + } + if all, _ := conditionsFrom.Open(ctx); len(all) != 0 { + t.Fatalf("%+v", all) + } +} diff --git a/cmd/mesh-controller/standing_test.go b/cmd/mesh-controller/standing_test.go index d174dd0c..88d69600 100644 --- a/cmd/mesh-controller/standing_test.go +++ b/cmd/mesh-controller/standing_test.go @@ -1,12 +1,9 @@ package main import ( - "encoding/json" "strings" "testing" - "time" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/link" ) @@ -15,98 +12,9 @@ import ( // the condition store's first kind (to-be 45 §2). On 2026-10-05 the identity provider refused every // consumer for a day and status called the mesh well (04-ISSUES/179): this is that day, told to the // controller the way the provider now tells it. -func TestAProviderFailingAConsumerBreaksAllWellUntilItRecovers(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, catalogue.Manifest{Module: "idp", Version: "1", - Receives: map[string]string{"oidc-client": "/var/lib/mesh/idp/mesh.json"}}) - if _, err := assign(ctx, open, "anchor", "idp"); err != nil { - t.Fatal(err) - } - kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} - since := time.Now().Add(-23 * time.Hour) - failing := link.Standing{Module: "idp", Failing: true, Provider: "oidc-client", ProviderNode: "anchor", - Consumer: "mesh_laptop_dashboard", Node: "laptop", Class: "credentials-rejected", - Error: `Keycloak token request failed: 401 {"error":"invalid_grant"}`, Since: since, Attempts: 31000} - if _, err := kept.Stood(ctx, failing); err != nil { - t.Fatal(err) - } - - asked, err := theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if asked.well() { - t.Fatal("a mesh whose identity provider fails a consumer reads as well") - } - said := printed(t, func() error { return printStatus(asked) }) - for _, want := range []string{"1 open condition(s)", "provider.idp.anchor.mesh_laptop_dashboard.failing", - "idp on anchor keeps failing mesh_laptop_dashboard on laptop", "credentials-rejected", "31000 attempt(s)"} { - if !strings.Contains(said, want) { - t.Fatalf("status does not say %q:\n%s", want, said) - } - } - if strings.Contains(said, "all doing what they were told") { - t.Fatalf("status said all well beside a failing provider:\n%s", said) - } - if !strings.HasPrefix(said, "1 open condition(s)") { - t.Fatalf("status does not lead with what is open:\n%s", said) - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Conditions []conditions.Condition `json:"conditions"` - Failing []conditions.Condition `json:"failing"` - } - if err := json.Unmarshal(body, &doc); err != nil || len(doc.Failing) != 1 || len(doc.Conditions) != 1 || - !strings.Contains(doc.Failing[0].Evidence[0].Said, "invalid_grant") { - t.Fatalf("the document does not carry it: %v\n%s", err, body) - } - // Both machines' `node show` name it: where the provider runs, and where the consumer is. - for _, node := range []string{"anchor", "laptop"} { - shown := printed(t, func() error { return showNode(ctx, open, node) }) - if !strings.Contains(shown, "open condition(s) about this machine") || !strings.Contains(shown, "mesh_laptop_dashboard") { - t.Fatalf("node show %s does not name it:\n%s", node, shown) - } - } - - // Recovered: gone, and the mesh may be well again as far as this is concerned. - failing.Failing = false - if cleared, err := kept.Stood(ctx, failing); err != nil || !cleared { - t.Fatalf("%v %v", cleared, err) - } - asked, err = theThreeQuestions(ctx, open) - if err != nil { - t.Fatal(err) - } - if len(asked.conditions) != 0 { - t.Fatalf("a recovered consumer is still named: %+v", asked.conditions) - } -} // A provider no longer assigned where it ran has nothing running to fail anybody: its last word is // cleared on the next look, said as resolved by the assignment. -func TestAnUnassignedProvidersLastWordIsCleared(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - kept := standings{keeper: func() *conditions.Keeper { return conditionsFrom }} - if _, err := kept.Stood(ctx, link.Standing{Module: "gone", Failing: true, - ProviderNode: "anchor", Consumer: "x", Since: time.Now()}); err != nil { - t.Fatal(err) - } - all, err := conditionsFrom.Open(ctx) - if err != nil || len(all) != 1 { - t.Fatalf("%+v %v", all, err) - } - if err := unassignedProviders(ctx, open.inventory, conditionsFrom, all); err != nil { - t.Fatal(err) - } - if all, _ := conditionsFrom.Open(ctx); len(all) != 0 { - t.Fatalf("%+v", all) - } -} // **The store away holds a recovery** (ADR 0224 §3): a standing that cannot be kept is asked again. func TestAStandingTheStoreCannotKeepIsAskedAgain(t *testing.T) { diff --git a/cmd/mesh-controller/status_summary_test.go b/cmd/mesh-controller/status_summary_test.go index 6f8a6984..8b342cf3 100644 --- a/cmd/mesh-controller/status_summary_test.go +++ b/cmd/mesh-controller/status_summary_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/status_test.go b/cmd/mesh-controller/status_test.go index db5102ce..2272a6d7 100644 --- a/cmd/mesh-controller/status_test.go +++ b/cmd/mesh-controller/status_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/template_integration_test.go b/cmd/mesh-controller/template_integration_test.go new file mode 100644 index 00000000..89338cd5 --- /dev/null +++ b/cmd/mesh-controller/template_integration_test.go @@ -0,0 +1,148 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" + snapshot "github.com/novox/mesh-controller/internal/facts" + "github.com/novox/mesh-controller/internal/inventory" +) + +func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { + t.Helper() + open := aMesh(t) + ctx := t.Context() + manifests := map[string]string{ + "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], + "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, + "notifier": inlineNotifier, + } + for name, raw := range manifests { + m, err := catalogue.ParseManifest([]byte(raw)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", + Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { + t.Fatal(err) + } + if _, err := assign(ctx, open, "laptop", name); err != nil { + t.Fatalf("assign %s: %v", name, err) + } + } + f, err := gatherFacts(ctx, open, "2.11.17") + if err != nil { + t.Fatal(err) + } + return f, manifests +} + +func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { + f, manifests := templatedMesh(t) + + // The module moved to its template, the same text: nothing changes on any machine. + v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", + notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "" { + t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) + } + + // The template alone changed: its module changed, and composes. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) + if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { + t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) + } + if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { + t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) + } + + // The template alone changed to text the controller refuses: the pull request fails, naming the module. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { + t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) + } + + // A symlinked files/ fails: nothing is read from outside the tree. + tree := aTree(t, withEdit(manifests, "notifier", "")) + elsewhere := t.TempDir() + if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { + t.Fatal(err) + } + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) + if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { + t.Fatal(err) + } + v = gateJudged(t, f, tree, notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { + t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) + } + + // And the template removed from the tree fails, naming it. + v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) + if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { + t.Fatalf("a missing template passed the gate:\n%s", v.Report()) + } +} + +func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { + dir := t.TempDir() + path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + m, err := catalogue.ParseManifest(raw) + if err != nil { + t.Fatal(err) + } + texts, err := catalogue.ReadTemplates(m, dir) + if err != nil { + t.Fatal(err) + } + resolved, err := standInBuild(m, texts) + if err != nil { + t.Fatal(err) + } + inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) + if err != nil { + t.Fatal(err) + } + if builtAlike(resolved) != builtAlike(inline) { + got, _ := json.Marshal(resolved) + t.Fatalf("the templated module resolves to %s, not its inline twin", got) + } +} + +func TestReplay876(t *testing.T) { + moved := strings.Replace(templatedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1) + judge := func(manifest string, files map[string]string) (map[string]catalogue.Manifest, []string) { + tree := t.TempDir() + aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), manifest, files) + out, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, nil, snapshot.Facts{}, + "novox/mesh-catalog", tree, map[string]string{}, nil) + if err != nil { + t.Fatal(err) + } + return out, failures + } + out, failures := judge(moved, map[string]string{"files/notifier.conf": "volume = 3\n"}) + if len(failures) > 0 { + t.Fatalf("a moved module naming its templates was refused: %v", failures) + } + if h := out["notifier"].Handed; h == nil || h.Templates["conf"] != "notifier.conf" { + t.Fatalf("the gate did not hand the moved module its templates: %+v", h) + } + + // A moved module naming no template is still refused. + none := strings.Replace(strings.Replace(moved, `"template":"files/notifier.conf"`, + `"content":"volume = 3\n"`, 1), `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1) + if _, failures := judge(none, nil); len(failures) != 1 || !strings.Contains(failures[0], "names no template") { + t.Fatalf("a moved module naming no template was not refused: %v", failures) + } +} diff --git a/cmd/mesh-controller/template_test.go b/cmd/mesh-controller/template_test.go index 2a14e3e4..2ece11f5 100644 --- a/cmd/mesh-controller/template_test.go +++ b/cmd/mesh-controller/template_test.go @@ -2,15 +2,10 @@ package main import ( "bytes" - "encoding/json" "os" "path/filepath" "strings" "testing" - - "github.com/novox/mesh-controller/internal/catalogue" - snapshot "github.com/novox/mesh-controller/internal/facts" - "github.com/novox/mesh-controller/internal/inventory" ) // A file's text kept as a template file in its module's repository (novox/hq ADR 0349 §2, design 57 package T1, @@ -83,34 +78,6 @@ func TestModuleCheckReadsTemplatesFromTheTreeBesideTheManifest(t *testing.T) { } // templatedMesh is catalogueMesh with the notifier on the laptop, registered as built: its text inline. -func templatedMesh(t *testing.T) (snapshot.Facts, map[string]string) { - t.Helper() - open := aMesh(t) - ctx := t.Context() - manifests := map[string]string{ - "lemurs": `{"module":"lemurs","version":"1","capabilities":["systemd"], - "resources":[{"id":"service","type":"service","unit":"lemurs.service","state":"running","boot":"enabled"}]}`, - "notifier": inlineNotifier, - } - for name, raw := range manifests { - m, err := catalogue.ParseManifest([]byte(raw)) - if err != nil { - t.Fatalf("%s: %v", name, err) - } - if err := open.inventory.RegisterModule(ctx, m, inventory.Source{Repository: "novox/mesh-catalog", - Path: "modules/" + name, BuiltFrom: "c0ffee"}); err != nil { - t.Fatal(err) - } - if _, err := assign(ctx, open, "laptop", name); err != nil { - t.Fatalf("assign %s: %v", name, err) - } - } - f, err := gatherFacts(ctx, open, "2.11.17") - if err != nil { - t.Fatal(err) - } - return f, manifests -} // aTemplatedTree is the catalogue's checkout with the notifier's text in files/. func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has bool) string { @@ -126,82 +93,7 @@ func aTemplatedTree(t *testing.T, manifests map[string]string, text string, has const notifierTemplate = "modules/notifier/files/notifier.conf" -func TestTheMergeGateJudgesAChangeToATemplateAlone(t *testing.T) { - f, manifests := templatedMesh(t) - - // The module moved to its template, the same text: nothing changes on any machine. - v := gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 3\n", true), "modules/notifier/module.json", - notifierTemplate) - if v.Verdict == "fail" || v.Modules["notifier"] != "" { - t.Fatalf("the module moved to its template with the same text reads as a change (%v):\n%s", v.Modules, v.Report()) - } - - // The template alone changed: its module changed, and composes. - v = gateJudged(t, f, aTemplatedTree(t, manifests, "volume = 4\n", true), notifierTemplate) - if v.Verdict == "fail" || v.Modules["notifier"] != "changed" { - t.Fatalf("a change to the template alone is not a change to its module (%v):\n%s", v.Modules, v.Report()) - } - if v.Width == nil || strings.Join(v.Width.Modules, ",") != "notifier" { - t.Fatalf("a change to the template rebuilds %+v, not its module", v.Width) - } - - // The template alone changed to text the controller refuses: the pull request fails, naming the module. - v = gateJudged(t, f, aTemplatedTree(t, manifests, "store = ${dir:nowhere}\n", true), notifierTemplate) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "notifier") { - t.Fatalf("a template the controller refuses passed the gate:\n%s", v.Report()) - } - - // A symlinked files/ fails: nothing is read from outside the tree. - tree := aTree(t, withEdit(manifests, "notifier", "")) - elsewhere := t.TempDir() - if err := os.WriteFile(filepath.Join(elsewhere, "notifier.conf"), []byte("volume = 3\n"), 0o644); err != nil { - t.Fatal(err) - } - aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), templatedNotifier, nil) - if err := os.Symlink(elsewhere, filepath.Join(tree, "modules", "notifier", "files")); err != nil { - t.Fatal(err) - } - v = gateJudged(t, f, tree, notifierTemplate) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/ is a symlink") { - t.Fatalf("a symlinked files/ passed the gate:\n%s", v.Report()) - } - - // And the template removed from the tree fails, naming it. - v = gateJudged(t, f, aTemplatedTree(t, manifests, "", false), notifierTemplate) - if v.Verdict != "fail" || !strings.Contains(v.Report(), "files/notifier.conf") { - t.Fatalf("a missing template passed the gate:\n%s", v.Report()) - } -} - // What the gate composes for the module in its template form is what the mesh composes for it inline. -func TestTheMergeGateResolvesATemplatedModuleToItsInlineTwin(t *testing.T) { - dir := t.TempDir() - path := aTemplatedModule(t, dir, templatedNotifier, map[string]string{"files/notifier.conf": "volume = 3\n"}) - raw, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - m, err := catalogue.ParseManifest(raw) - if err != nil { - t.Fatal(err) - } - texts, err := catalogue.ReadTemplates(m, dir) - if err != nil { - t.Fatal(err) - } - resolved, err := standInBuild(m, texts) - if err != nil { - t.Fatal(err) - } - inline, err := catalogue.ParseManifest([]byte(inlineNotifier)) - if err != nil { - t.Fatal(err) - } - if builtAlike(resolved) != builtAlike(inline) { - got, _ := json.Marshal(resolved) - t.Fatalf("the templated module resolves to %s, not its inline twin", got) - } -} // A module whose files/ is unpacked onto the machine names its own templates directory (issue 731): `module check` // reads its templates from there and says of it exactly what it says of the same module with its text inline. @@ -243,30 +135,3 @@ func TestModuleCheckReadsTemplatesFromTheTemplatesDirectoryTheManifestNames(t *t // Issue 876: the gate judges a moved module that names its templates as registration reads it after a real build, // handed its templates; it refused every such module as "marked moved and names no template". This calls the gate's // own shelf (shelfWithChange), which needs no database. -func TestReplay876(t *testing.T) { - moved := strings.Replace(templatedNotifier, `"version":"1",`, `"version":"1","moved":true,`, 1) - judge := func(manifest string, files map[string]string) (map[string]catalogue.Manifest, []string) { - tree := t.TempDir() - aTemplatedModule(t, filepath.Join(tree, "modules", "notifier"), manifest, files) - out, failures, err := shelfWithChange(map[string]catalogue.Manifest{}, nil, snapshot.Facts{}, - "novox/mesh-catalog", tree, map[string]string{}, nil) - if err != nil { - t.Fatal(err) - } - return out, failures - } - out, failures := judge(moved, map[string]string{"files/notifier.conf": "volume = 3\n"}) - if len(failures) > 0 { - t.Fatalf("a moved module naming its templates was refused: %v", failures) - } - if h := out["notifier"].Handed; h == nil || h.Templates["conf"] != "notifier.conf" { - t.Fatalf("the gate did not hand the moved module its templates: %+v", h) - } - - // A moved module naming no template is still refused. - none := strings.Replace(strings.Replace(moved, `"template":"files/notifier.conf"`, - `"content":"volume = 3\n"`, 1), `"build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]},`, "", 1) - if _, failures := judge(none, nil); len(failures) != 1 || !strings.Contains(failures[0], "names no template") { - t.Fatalf("a moved module naming no template was not refused: %v", failures) - } -} diff --git a/cmd/mesh-controller/terminal_settings_test.go b/cmd/mesh-controller/terminal_settings_test.go index 783ca251..62dd3954 100644 --- a/cmd/mesh-controller/terminal_settings_test.go +++ b/cmd/mesh-controller/terminal_settings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/tier_send_integration_test.go b/cmd/mesh-controller/tier_send_integration_test.go new file mode 100644 index 00000000..d05dc9ca --- /dev/null +++ b/cmd/mesh-controller/tier_send_integration_test.go @@ -0,0 +1,410 @@ +//go:build integration + +package main + +import ( + "context" + "encoding/json" + "fmt" + "reflect" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/conditions" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/lease" +) + +func aTierMesh(t *testing.T, modules ...string) *tierMesh { + t.Helper() + open := aMesh(t) + ctx := t.Context() + inv := open.inventory + tm := &tierMesh{open: open, modules: modules} + tm.keeper, _ = withConditionsInMemory(t) + was := doctorFrom + doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}} + t.Cleanup(func() { doctorFrom = was }) + + register := func(m, commit string, asked time.Time) { + if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, + inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit, + Head: commit, Asked: asked}); err != nil { + t.Fatal(err) + } + } + first := map[string]string{} + for _, m := range modules { + for i, commit := range []string{"c1", "c2"} { + at := time.Now().Add(-2 * time.Hour) + if commit == "c2" { + at = time.Now().Add(-time.Minute) + } + b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit, + Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at} + b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + } + register(m, "c1", time.Now().Add(-2*time.Hour)) + first[m] = "c1" + for _, n := range []string{"anchor", "laptop"} { + if _, err := inv.Assign(ctx, n, m); err != nil { + t.Fatal(err) + } + } + } + for _, n := range []string{"anchor", "laptop"} { + if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil { + t.Fatal(err) + } + } + for _, m := range modules { + register(m, "c2", time.Now().Add(-time.Minute)) + } + + n := 0 + wasSend := sendRollout + sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { + tm.sent = append(tm.sent, append([]string(nil), names...)) + current, err := open.inventory.CurrentBuilds(ctx) + if err != nil { + return nil, err + } + builds := map[string]string{} + for _, m := range tm.modules { + builds[m] = current[m].Commit + } + for _, node := range names { + n++ + digest := fmt.Sprintf("d-%s-%d", node, n) + if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { + return nil, err + } + if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, + Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { + return nil, err + } + } + return names, nil + } + t.Cleanup(func() { sendRollout = wasSend }) + + wasGather := gatherGateFacts + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, + rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true, + open: append([]conditions.Condition(nil), tm.conds...)} + reports, err := open.inventory.LastReports(ctx) + if err != nil { + return f, err + } + for _, r := range reports { + f.reports[r.Node] = r + f.served[r.Node] = served{runtime: true, tools: map[string]bool{}} + } + return f, nil + } + t.Cleanup(func() { gatherGateFacts = wasGather }) + + wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound + // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). + wasWindow := gateWitnessWindow + gateWitnessWindow = 0 + t.Cleanup(func() { gateWitnessWindow = wasWindow }) + // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. + gateWithin, gateEvery = time.Hour, time.Hour + t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) + + built := time.Now().UTC() + states := map[string]*inventory.PlanModule{} + for _, m := range modules { + states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"} + } + plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", + Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states} + if err := inv.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + return tm +} + +func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + var modules []string + for i := 1; i <= 12; i++ { + modules = append(modules, fmt.Sprintf("m%02d", i)) + } + tm := aTierMesh(t, modules...) + ctx := t.Context() + + advancePlans(ctx, tm.open) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { + t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent) + } + p := tm.plan(t) + lead := p.Modules["m01"] + if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" { + t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate) + } + for _, m := range modules { + s := p.Modules[m] + if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" { + t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s) + } + if m != "m01" && s.GatedBy != "m01" { + t.Fatalf("%s is not gated by the send's gate: %+v", m, s) + } + } + + gateEvery = 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("sent %v: one send per machine for the tier", tm.sent) + } + if p.State != inventory.PlanDone { + t.Fatalf("the plan is %s: %s", p.State, p.Note) + } + for _, m := range modules { + if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed { + t.Fatalf("%s's gate: %+v", m, g) + } + if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed { + t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err) + } + } +} + +func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) + ctx := t.Context() + inv := tm.open.inventory + // The node tools are held inside their witness window, which the node-engine keeps while they settle: their + // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). + gateWitnessWindow = time.Hour + + advancePlans(ctx, tm.open) + if len(tm.sent) != 1 { + t.Fatalf("sent %v", tm.sent) + } + tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+ + "and no plan is rolling them out") + gateEvery = 0 + advancePlans(ctx, tm.open) + g := tm.plan(t).Modules["app1"].Gate + if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { + t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g) + } + + gateBound = -time.Second + advancePlans(ctx, tm.open) + p := tm.plan(t) + g = p.Modules["app1"].Gate + if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed || + !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { + t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) + } + current, _ := inv.CurrentBuilds(ctx) + if current[broker.RuntimeModule].Commit != "c1" { + t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit) + } + for _, m := range []string{"app1", "app2"} { + if current[m].Commit != "c2" { + t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit) + } + if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { + t.Fatalf("%s's build was marked failed for the node tools' condition", m) + } + } + // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 + // review): it is not blamed, and not left without a verdict. + if !strings.Contains(p.Note, "app1 passed on its own and is kept") { + t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) + } + if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app1's own pass was not kept: %+v", v) + } +} + +func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + tm := aTierMesh(t, "app1", "app2", "app3") + ctx := t.Context() + inv := tm.open.inventory + advancePlans(ctx, tm.open) + tm.raise("anchor", "disk-full", "anchor's root file system is full") + gateEvery, gateBound = 0, -time.Second + advancePlans(ctx, tm.open) + p := tm.plan(t) + g := p.Modules["app1"].Gate + if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") || + !reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) { + t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) + } + current, _ := inv.CurrentBuilds(ctx) + for _, m := range []string{"app1", "app2", "app3"} { + if current[m].Commit != "c1" { + t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit) + } + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) { + t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent) + } + if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") { + t.Fatalf("the plan does not say what was put back: %s", p.Note) + } +} + +func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + g := aGateMesh(t) + ctx := t.Context() + inv := g.open.inventory + // An earlier send carried c2 to anchor before the plan's own. + if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil { + t.Fatal(err) + } + advancePlans(ctx, g.open) + if p := g.plan(t); p.Modules["app"].Previous != "c2" { + t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"]) + } + g.health["anchor"] = healthNotYet + gateEvery, gateBound = 0, -time.Second + advancePlans(ctx, g.open) + + p := g.plan(t) + gate := p.Modules["app"].Gate + if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged || + !strings.Contains(p.Note, "app was left as it was") { + t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate) + } + if failed, _ := inv.GateFailed(ctx, "build-2"); failed { + t.Fatal("a build its send changed nothing of was marked failed") + } + if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { + t.Fatalf("the module was moved: %s", current["app"].Commit) + } + open, _ := g.keeper.Open(ctx) + for _, c := range open { + if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack { + t.Fatalf("said as a rollback: %+v", c) + } + } + + was := askABuild + askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil } + t.Cleanup(func() { askABuild = was }) + g.health["anchor"] = healthGood + said, err := retryPlan(ctx, g.open, "plan-gate") + if err != nil { + t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err) + } + p = g.plan(t) + s := p.Modules["app"] + if p.State != inventory.PlanBuilding || s.State != "asked" || s.Build != "build-3" || s.FirstAt != nil || s.Gate != nil { + t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s) + } +} + +func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + judgeLeavesAsTheCore(t) + for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { + t.Run("gate kept on "+order[0], func(t *testing.T) { + tm := aTierMesh(t, order...) + ctx := t.Context() + inv := tm.open.inventory + tierFacts := gatherGateFacts + var seen []string + gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { + f, err := tierFacts(ctx, open, component) + current, _ := open.inventory.CurrentBuilds(ctx) + seen = append(seen, current[broker.RuntimeModule].Commit) + if current[broker.RuntimeModule].Commit == "c2" { + f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, + Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: time.Now(), Why: "the node tools did not answer"}} + } + return f, err + } + // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the + // judgings that find the node tools broken, and is judged to its own pass after them (the hold the + // settle time gave before novox/hq ADR 0352 counted readings from the send). + gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second + for i := 0; i < 20 && len(seen) < 2; i++ { + advancePlans(ctx, tm.open) + } + if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { + t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at once", seen) + } + if failed, _ := inv.GateFailed(ctx, "build-"+broker.RuntimeModule+"-2"); !failed { + t.Fatal("the node tools' build is not marked failed at once") + } + if p := tm.plan(t); p.State == inventory.PlanFailed { + t.Fatalf("the plan failed at once: %s; want it judging app1 first", p.Note) + } + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { + advancePlans(ctx, tm.open) + if tm.plan(t).State == inventory.PlanFailed { + break + } + time.Sleep(20 * time.Millisecond) + } + p := tm.plan(t) + if p.State != inventory.PlanFailed { + t.Fatalf("the plan is %s: %s; want failed, for the node tools", p.State, p.Note) + } + if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { + t.Fatalf("app1's verdict: %+v; want its own pass (plan: %s)", v, p.Note) + } + if current, _ := inv.CurrentBuilds(ctx); current["app1"].Commit != "c2" || current[broker.RuntimeModule].Commit != "c1" { + t.Fatalf("registered app1 %s, node tools %s; want c2 and c1", current["app1"].Commit, + current[broker.RuntimeModule].Commit) + } + }) + } +} + +func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { + withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) + tm := aTierMesh(t, broker.RuntimeModule, "app1") + ctx := t.Context() + // One reading counted per advance, as every test with a gate left open does: a gate judged with no time + // between readings would count a pass at every look and never let an advance rest. + gateEvery, gateWitnessWindow = time.Hour, time.Hour + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p := tm.plan(t) + lead := p.Modules[broker.RuntimeModule].Gate + if lead == nil { + lead = p.Modules["app1"].Gate + } + if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { + t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) + } + gateEvery, gateWitnessWindow = 0, 0 + for i := 0; i < 6; i++ { + advancePlans(ctx, tm.open) + } + p = tm.plan(t) + if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { + t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) + } + for _, m := range []string{broker.RuntimeModule, "app1"} { + if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || + !strings.Contains(g.Why, "the tool runner's rule")) { + t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) + } + } +} diff --git a/cmd/mesh-controller/tier_send_test.go b/cmd/mesh-controller/tier_send_test.go index 590900ad..188ec090 100644 --- a/cmd/mesh-controller/tier_send_test.go +++ b/cmd/mesh-controller/tier_send_test.go @@ -1,19 +1,13 @@ package main import ( - "context" - "encoding/json" - "fmt" - "reflect" "strings" "testing" "time" "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/conditions" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/lease" ) // A plan's tier is sent to each machine once, and judged there by one gate (novox/hq issue 281). @@ -37,121 +31,6 @@ type tierMesh struct { keeper *conditions.Keeper } -func aTierMesh(t *testing.T, modules ...string) *tierMesh { - t.Helper() - open := aMesh(t) - ctx := t.Context() - inv := open.inventory - tm := &tierMesh{open: open, modules: modules} - tm.keeper, _ = withConditionsInMemory(t) - was := doctorFrom - doctorFrom = &doctor{open: open, keeper: tm.keeper, teller: &conditions.Told{}} - t.Cleanup(func() { doctorFrom = was }) - - register := func(m, commit string, asked time.Time) { - if err := inv.RegisterModule(ctx, catalogue.Manifest{Module: m, Version: commit}, - inventory.Source{Repository: "novox/mesh-catalog", Seat: "git", Path: "modules/" + m, BuiltFrom: commit, - Head: commit, Asked: asked}); err != nil { - t.Fatal(err) - } - } - first := map[string]string{} - for _, m := range modules { - for i, commit := range []string{"c1", "c2"} { - at := time.Now().Add(-2 * time.Hour) - if commit == "c2" { - at = time.Now().Add(-time.Minute) - } - b := inventory.Build{ID: fmt.Sprintf("build-%s-%d", m, i+1), Module: m, Commit: commit, - Repository: "novox/mesh-catalog", Path: "modules/" + m, Asked: at, At: at} - b.Manifest, _ = json.Marshal(catalogue.Manifest{Module: m, Version: commit}) - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - } - register(m, "c1", time.Now().Add(-2*time.Hour)) - first[m] = "c1" - for _, n := range []string{"anchor", "laptop"} { - if _, err := inv.Assign(ctx, n, m); err != nil { - t.Fatal(err) - } - } - } - for _, n := range []string{"anchor", "laptop"} { - if err := inv.RecordSent(ctx, nodeID(t, open, n), "d-"+n+"-c1", first); err != nil { - t.Fatal(err) - } - } - for _, m := range modules { - register(m, "c2", time.Now().Add(-time.Minute)) - } - - n := 0 - wasSend := sendRollout - sendRollout = func(ctx context.Context, open *stores, names []string) ([]string, error) { - tm.sent = append(tm.sent, append([]string(nil), names...)) - current, err := open.inventory.CurrentBuilds(ctx) - if err != nil { - return nil, err - } - builds := map[string]string{} - for _, m := range tm.modules { - builds[m] = current[m].Commit - } - for _, node := range names { - n++ - digest := fmt.Sprintf("d-%s-%d", node, n) - if err := open.inventory.RecordSent(ctx, nodeID(t, open, node), digest, builds); err != nil { - return nil, err - } - if _, err := open.inventory.RecordDoing(ctx, nodeID(t, open, node), inventory.Doing{Node: node, - Outcome: inventory.OutcomeApplied, Declared: digest, Applied: 1, At: time.Now()}); err != nil { - return nil, err - } - } - return names, nil - } - t.Cleanup(func() { sendRollout = wasSend }) - - wasGather := gatherGateFacts - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f := gateFacts{now: time.Now(), reports: map[string]inventory.Reported{}, engines: map[string]string{}, - rolledBack: map[string][]lease.Rollback{}, served: map[string]served{}, judged: true, - open: append([]conditions.Condition(nil), tm.conds...)} - reports, err := open.inventory.LastReports(ctx) - if err != nil { - return f, err - } - for _, r := range reports { - f.reports[r.Node] = r - f.served[r.Node] = served{runtime: true, tools: map[string]bool{}} - } - return f, nil - } - t.Cleanup(func() { gatherGateFacts = wasGather }) - - wasWithin, wasEvery, wasBound := gateWithin, gateEvery, gateBound - // The witness window is the node-engine's, from a start no test makes (novox/hq ADR 0351). - wasWindow := gateWitnessWindow - gateWitnessWindow = 0 - t.Cleanup(func() { gateWitnessWindow = wasWindow }) - // One judging per advance until a test says otherwise: a pass waits gateEvery for the next. - gateWithin, gateEvery = time.Hour, time.Hour - t.Cleanup(func() { gateWithin, gateEvery, gateBound = wasWithin, wasEvery, wasBound }) - - built := time.Now().UTC() - states := map[string]*inventory.PlanModule{} - for _, m := range modules { - states[m] = &inventory.PlanModule{State: "built", BuiltAt: &built, Commit: "c2", Build: "build-" + m + "-2"} - } - plan := inventory.Plan{ID: "plan-tier", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c2", - Created: built, State: inventory.PlanBuilding, Tiers: [][]string{modules}, Modules: states} - if err := inv.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - return tm -} - func (tm *tierMesh) plan(t *testing.T) inventory.Plan { t.Helper() p, err := tm.open.inventory.PlanByID(t.Context(), "plan-tier") @@ -170,142 +49,13 @@ func (tm *tierMesh) raise(machine, kind, summary string) { // The incident's tier, small: a dozen modules whose first machine is the same are sent there in one // send, judged by one gate, and the rest of their machines are sent once — two sends, not two dozen. -func TestATierOfManyModulesIsSentToEachMachineOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - var modules []string - for i := 1; i <= 12; i++ { - modules = append(modules, fmt.Sprintf("m%02d", i)) - } - tm := aTierMesh(t, modules...) - ctx := t.Context() - - advancePlans(ctx, tm.open) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) { - t.Fatalf("sent %v: the first machine once, for the whole tier", tm.sent) - } - p := tm.plan(t) - lead := p.Modules["m01"] - if lead.Gate == nil || len(lead.Gate.Carried) != len(modules) || lead.GatedBy != "" { - t.Fatalf("the gate on the first module does not judge the whole send: %+v", lead.Gate) - } - for _, m := range modules { - s := p.Modules[m] - if s.FirstAt == nil || !reflect.DeepEqual(s.First, []string{"anchor"}) || s.Previous != "c1" { - t.Fatalf("%s's first send is not recorded, or what anchor ran before is not c1: %+v", m, s) - } - if m != "m01" && s.GatedBy != "m01" { - t.Fatalf("%s is not gated by the send's gate: %+v", m, s) - } - } - - gateEvery = 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("sent %v: one send per machine for the tier", tm.sent) - } - if p.State != inventory.PlanDone { - t.Fatalf("the plan is %s: %s", p.State, p.Note) - } - for _, m := range modules { - if g := p.Modules[m].Gate; g == nil || g.Verdict != inventory.GatePassed { - t.Fatalf("%s's gate: %+v", m, g) - } - if v, found, err := tm.open.inventory.GateOf(ctx, "build-"+m+"-2"); err != nil || !found || v.Verdict != inventory.GatePassed { - t.Fatalf("%s's pass was not kept: %+v %v %v", m, v, found, err) - } - } -} // The core behind about the node tools, while the node tools are themselves among what was sent and // settling, is the node tools' — inside their settle window, the gate's bound — and no failure of the // modules sent with them: at the bound only the node tools are put back. -func TestACoreBehindWhileTheNodeToolsSettleFailsNoOtherModule(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, "app1", "app2", broker.RuntimeModule) - ctx := t.Context() - inv := tm.open.inventory - // The node tools are held inside their witness window, which the node-engine keeps while they settle: their - // one reading does not count yet, so the core behind raised meanwhile is theirs (novox/hq ADR 0351 decision 4). - gateWitnessWindow = time.Hour - - advancePlans(ctx, tm.open) - if len(tm.sent) != 1 { - t.Fatalf("sent %v", tm.sent) - } - tm.raise("anchor", kindCoreBehind, "anchor runs the node tools it was last sent, not yet reported applied, "+ - "and no plan is rolling them out") - gateEvery = 0 - advancePlans(ctx, tm.open) - g := tm.plan(t).Modules["app1"].Gate - if g.Verdict != "" || !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("inside the settle window the gate is %+v: only the node tools wait on it", g) - } - - gateBound = -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g = p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || g.Verdict != inventory.GateFailed || - !reflect.DeepEqual(g.Failing, []string{broker.RuntimeModule}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - if current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("the node tools were not put back: %s", current[broker.RuntimeModule].Commit) - } - for _, m := range []string{"app1", "app2"} { - if current[m].Commit != "c2" { - t.Fatalf("%s was put back for the node tools' condition: %s", m, current[m].Commit) - } - if failed, _ := inv.GateFailed(ctx, "build-"+m+"-2"); failed { - t.Fatalf("%s's build was marked failed for the node tools' condition", m) - } - } - // Healthy for the passes asked, the module the gate is kept on keeps its own pass (novox/hq issue 318 - // review): it is not blamed, and not left without a verdict. - if !strings.Contains(p.Note, "app1 passed on its own and is kept") { - t.Fatalf("the plan blames the module its gate was kept on, or leaves it unjudged: %s", p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's own pass was not kept: %+v", v) - } -} // A machine-level condition about anything else holds the machine back as a whole: everything the send // moved there fails together, once, with that reason — not one module for it. -func TestAMachineUnhealthyAsAWholeFailsTheWholeSendTogether(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - tm := aTierMesh(t, "app1", "app2", "app3") - ctx := t.Context() - inv := tm.open.inventory - advancePlans(ctx, tm.open) - tm.raise("anchor", "disk-full", "anchor's root file system is full") - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, tm.open) - p := tm.plan(t) - g := p.Modules["app1"].Gate - if p.State != inventory.PlanFailed || !strings.Contains(g.Why, "anchor as a whole") || - !reflect.DeepEqual(g.Failing, []string{"app1", "app2", "app3"}) { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, g) - } - current, _ := inv.CurrentBuilds(ctx) - for _, m := range []string{"app1", "app2", "app3"} { - if current[m].Commit != "c1" { - t.Fatalf("%s was not put back with the send: %s", m, current[m].Commit) - } - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"anchor"}}) { - t.Fatalf("sent %v: the first send and one send putting all of it back, and nothing to laptop", tm.sent) - } - if !strings.Contains(p.Note, "put back app1 to c1, app2 to c1, app3 to c1 on anchor, in one send") { - t.Fatalf("the plan does not say what was put back: %s", p.Note) - } -} // The machine-level conditions, sorted: one naming a module is that module's; the core behind is the // core's when the send moved it and nobody's when it did not; anything else is the machine's as a whole. @@ -342,153 +92,12 @@ func TestTheConditionsAboutAMachineItself(t *testing.T) { // fails a gate as no verdict on its build: left as it was, never marked failed, nothing said as a // rollback that failed; and `plans retry` asks it again rather than refusing a build that was never // judged (the incident's "NOT put back: no build kept"). -func TestAModuleItsSendChangedNothingOfIsLeftAsItWasAndRetried(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - g := aGateMesh(t) - ctx := t.Context() - inv := g.open.inventory - // An earlier send carried c2 to anchor before the plan's own. - if err := inv.RecordSent(ctx, nodeID(t, g.open, "anchor"), "d-anchor-early", map[string]string{"app": "c2"}); err != nil { - t.Fatal(err) - } - advancePlans(ctx, g.open) - if p := g.plan(t); p.Modules["app"].Previous != "c2" { - t.Fatalf("what anchor ran before the send: %+v", p.Modules["app"]) - } - g.health["anchor"] = healthNotYet - gateEvery, gateBound = 0, -time.Second - advancePlans(ctx, g.open) - - p := g.plan(t) - gate := p.Modules["app"].Gate - if p.State != inventory.PlanFailed || gate.Verdict != inventory.GateFailed || gate.Rollback != gateUnchanged || - !strings.Contains(p.Note, "app was left as it was") { - t.Fatalf("the plan is %s (%s), the gate %+v", p.State, p.Note, gate) - } - if failed, _ := inv.GateFailed(ctx, "build-2"); failed { - t.Fatal("a build its send changed nothing of was marked failed") - } - if current, _ := inv.CurrentBuilds(ctx); current["app"].Commit != "c2" { - t.Fatalf("the module was moved: %s", current["app"].Commit) - } - open, _ := g.keeper.Open(ctx) - for _, c := range open { - if c.Kind == kindRollbackFailed || c.Kind == kindRolledBack { - t.Fatalf("said as a rollback: %+v", c) - } - } - - was := askABuild - askABuild = func(context.Context, buildSource, string, string) (string, error) { return "build-3", nil } - t.Cleanup(func() { askABuild = was }) - g.health["anchor"] = healthGood - said, err := retryPlan(ctx, g.open, "plan-gate") - if err != nil { - t.Fatalf("a plan stopped at a gate that judged no build was not retried: %v", err) - } - p = g.plan(t) - s := p.Modules["app"] - if p.State != inventory.PlanBuilding || s.State != "asked" || s.Build != "build-3" || s.FirstAt != nil || s.Gate != nil { - t.Fatalf("retried as %q: the plan is %s, app %+v", said, p.State, s) - } -} // **In a plan's tier path, a broken module is put back at once too** (novox/hq issue 318 review): the node // tools' witness reverts them on the first machine. Whether the gate is kept on them or on the module beside // them, they are registered back and marked in the judging that finds them broken, the plan goes on judging // the other module to its own pass, and only then fails. -func TestATierPutsABrokenModuleBackAtOnce(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - judgeLeavesAsTheCore(t) - for _, order := range [][]string{{broker.RuntimeModule, "app1"}, {"app1", broker.RuntimeModule}} { - t.Run("gate kept on "+order[0], func(t *testing.T) { - tm := aTierMesh(t, order...) - ctx := t.Context() - inv := tm.open.inventory - tierFacts := gatherGateFacts - var seen []string - gatherGateFacts = func(ctx context.Context, open *stores, component string) (gateFacts, error) { - f, err := tierFacts(ctx, open, component) - current, _ := open.inventory.CurrentBuilds(ctx) - seen = append(seen, current[broker.RuntimeModule].Commit) - if current[broker.RuntimeModule].Commit == "c2" { - f.rolledBack["anchor"] = []lease.Rollback{{Component: lease.ComponentNodeTools, - Outcome: lease.OutcomeRolledBack, From: "c2", To: "c1", At: time.Now(), Why: "the node tools did not answer"}} - } - return f, err - } - // app1, judged as the core, counts a reading at most every 100 ms: it cannot have its three in the - // judgings that find the node tools broken, and is judged to its own pass after them (the hold the - // settle time gave before novox/hq ADR 0352 counted readings from the send). - gateEvery, gateWithin, gateBound = 100*time.Millisecond, time.Hour, 10*time.Second - for i := 0; i < 20 && len(seen) < 2; i++ { - advancePlans(ctx, tm.open) - } - if len(seen) < 2 || seen[0] != "c2" || seen[1] != "c1" { - t.Fatalf("the node tools' registered build at each judging: %v; want c2, then c1 at once", seen) - } - if failed, _ := inv.GateFailed(ctx, "build-"+broker.RuntimeModule+"-2"); !failed { - t.Fatal("the node tools' build is not marked failed at once") - } - if p := tm.plan(t); p.State == inventory.PlanFailed { - t.Fatalf("the plan failed at once: %s; want it judging app1 first", p.Note) - } - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); { - advancePlans(ctx, tm.open) - if tm.plan(t).State == inventory.PlanFailed { - break - } - time.Sleep(20 * time.Millisecond) - } - p := tm.plan(t) - if p.State != inventory.PlanFailed { - t.Fatalf("the plan is %s: %s; want failed, for the node tools", p.State, p.Note) - } - if v, found, _ := inv.GateOf(ctx, "build-app1-2"); !found || v.Verdict != inventory.GatePassed { - t.Fatalf("app1's verdict: %+v; want its own pass (plan: %s)", v, p.Note) - } - if current, _ := inv.CurrentBuilds(ctx); current["app1"].Commit != "c2" || current[broker.RuntimeModule].Commit != "c1" { - t.Fatalf("registered app1 %s, node tools %s; want c2 and c1", current["app1"].Commit, - current[broker.RuntimeModule].Commit) - } - }) - } -} // **A send carrying the tool runner waits for its witness window** (issue 524 review, novox/hq ADR 0351 // decision 4): the leaf beside the node tools has its one reading at once, but the send is judged by the tool // runner's rule: one reading, not before the node-engine's witness window for it has ended. -func TestASendCarryingTheToolRunnerWaitsForItsWitnessWindow(t *testing.T) { - withTheGate(t) // the first-machine gate\'s own path (novox/hq ADR 0360 keeps it for earlier walks) - tm := aTierMesh(t, broker.RuntimeModule, "app1") - ctx := t.Context() - // One reading counted per advance, as every test with a gate left open does: a gate judged with no time - // between readings would count a pass at every look and never let an advance rest. - gateEvery, gateWitnessWindow = time.Hour, time.Hour - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p := tm.plan(t) - lead := p.Modules[broker.RuntimeModule].Gate - if lead == nil { - lead = p.Modules["app1"].Gate - } - if !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}}) || !p.Open() || lead == nil || lead.Verdict != "" || lead.Wants != 1 { - t.Fatalf("passed inside the witness window: sent %v, plan %s (%s), gate %+v", tm.sent, p.State, p.Note, lead) - } - gateEvery, gateWitnessWindow = 0, 0 - for i := 0; i < 6; i++ { - advancePlans(ctx, tm.open) - } - p = tm.plan(t) - if p.State != inventory.PlanDone || !reflect.DeepEqual(tm.sent, [][]string{{"anchor"}, {"laptop"}}) { - t.Fatalf("the plan is %s (%s), sent %v", p.State, p.Note, tm.sent) - } - for _, m := range []string{broker.RuntimeModule, "app1"} { - if g := p.Modules[m].Gate; g != nil && (g.Verdict != inventory.GatePassed || g.Passes < 1 || - !strings.Contains(g.Why, "the tool runner's rule")) { - t.Fatalf("%s's gate: %+v; want passed on one reading by the tool runner's rule", m, g) - } - } -} diff --git a/cmd/mesh-controller/try_integration_test.go b/cmd/mesh-controller/try_integration_test.go new file mode 100644 index 00000000..c3b930e8 --- /dev/null +++ b/cmd/mesh-controller/try_integration_test.go @@ -0,0 +1,39 @@ +//go:build integration + +package main + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/link" +) + +func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { + said := &saidOnTheBus{} + checkEvents = said + defer func() { checkEvents = nil }() + for name, result := range map[string]link.BuildResult{ + "a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}}, + "a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, + Failed: "cannot clone"}, + // A build seat from before the mark echoes the ask without it: a check of no pull request and no + // group is still nobody's status. + "a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"}, + Check: &link.CheckOutcome{Verdict: "pass"}}, + } { + if err := (builds{}).Built(context.Background(), result); err != nil { + t.Fatalf("%s: %v", name, err) + } + } + if len(said.events) != 0 { + t.Fatalf("a try's outcome was said for the forge: %v", said.events) + } + // A pull request's check is still said. + _ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{ + Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}}) + if len(said.events) != 1 { + t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) + } +} diff --git a/cmd/mesh-controller/try_test.go b/cmd/mesh-controller/try_test.go index 5f72bcda..f76953c1 100644 --- a/cmd/mesh-controller/try_test.go +++ b/cmd/mesh-controller/try_test.go @@ -40,34 +40,6 @@ func TestATryAsksACheckOfTheRepositorysOwnScriptAlone(t *testing.T) { // A try's outcome is heard and left there: no `checked` is said, so the forge's holder sets no status, and // nothing is recorded or registered (the handler holds no store here, and would fail reaching for one). -func TestATrysOutcomeSetsNoStatusAndRegistersNothing(t *testing.T) { - said := &saidOnTheBus{} - checkEvents = said - defer func() { checkEvents = nil }() - for name, result := range map[string]link.BuildResult{ - "a try": {ID: "build-1", Ref: "c0ffee", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, - Check: &link.CheckOutcome{Verdict: "pass", RepoCheck: &link.CheckLayer{Verdict: "fail", Summary: "failed"}}}, - "a try that could not run": {ID: "build-2", Checked: &link.CheckRequest{Owner: "novox", Repo: "a", Try: true}, - Failed: "cannot clone"}, - // A build seat from before the mark echoes the ask without it: a check of no pull request and no - // group is still nobody's status. - "a try echoed by an older build seat": {ID: "build-3", Checked: &link.CheckRequest{Owner: "novox", Repo: "a"}, - Check: &link.CheckOutcome{Verdict: "pass"}}, - } { - if err := (builds{}).Built(context.Background(), result); err != nil { - t.Fatalf("%s: %v", name, err) - } - } - if len(said.events) != 0 { - t.Fatalf("a try's outcome was said for the forge: %v", said.events) - } - // A pull request's check is still said. - _ = (builds{}).Built(context.Background(), link.BuildResult{ID: "build-4", Checked: &link.CheckRequest{ - Owner: "novox", Repo: "a", Number: 7}, Check: &link.CheckOutcome{Verdict: "pass"}}) - if len(said.events) != 1 { - t.Fatalf("a pull request's check was said %d time(s)", len(said.events)) - } -} // saidOnTheBus keeps the seat events said, and nothing else. type saidOnTheBus struct { diff --git a/cmd/mesh-controller/unbound_test.go b/cmd/mesh-controller/unbound_test.go index 8bbc28a7..554a744f 100644 --- a/cmd/mesh-controller/unbound_test.go +++ b/cmd/mesh-controller/unbound_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/unheld_report_integration_test.go b/cmd/mesh-controller/unheld_report_integration_test.go new file mode 100644 index 00000000..ec1bdd51 --- /dev/null +++ b/cmd/mesh-controller/unheld_report_integration_test.go @@ -0,0 +1,36 @@ +//go:build integration + +package main + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + register(t, open, aContainer("web")) + register(t, open, aContainer("db")) + // anchor already lacks a runtime for db: true, and not this act's to say. + if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil { + t.Fatal(err) + } + if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil { + t.Fatal(err) + } + said, err := assign(ctx, open, "laptop", "web") + if err != nil { + t.Fatalf("%v\n%s", err, said) + } + if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) { + t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said) + } + for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} { + if strings.Contains(said, not) { + t.Errorf("the assignment says %q, which it did not change:\n%s", not, said) + } + } +} diff --git a/cmd/mesh-controller/unheld_report_test.go b/cmd/mesh-controller/unheld_report_test.go index 9430d671..6d7d2e61 100644 --- a/cmd/mesh-controller/unheld_report_test.go +++ b/cmd/mesh-controller/unheld_report_test.go @@ -17,32 +17,6 @@ func aContainer(name string) catalogue.Manifest { Resources: []map[string]any{{"id": name, "type": "container", "image": name}}} } -func TestAnAssignmentSaysOnlyWhatItChangedOnItsOwnNode(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - register(t, open, aContainer("web")) - register(t, open, aContainer("db")) - // anchor already lacks a runtime for db: true, and not this act's to say. - if _, err := open.inventory.Assign(ctx, "anchor", "db"); err != nil { - t.Fatal(err) - } - if _, err := open.inventory.Assign(ctx, "laptop", "db"); err != nil { - t.Fatal(err) - } - said, err := assign(ctx, open, "laptop", "web") - if err != nil { - t.Fatalf("%v\n%s", err, said) - } - if !strings.Contains(said, "web on laptop depends on "+catalogue.ContainerRuntimeSeat) { - t.Errorf("the assignment does not say what the module it assigned depends on:\n%s", said) - } - for _, not := range []string{"db on laptop", "db on anchor", "anchor:"} { - if strings.Contains(said, not) { - t.Errorf("the assignment says %q, which it did not change:\n%s", not, said) - } - } -} - func TestAnAssignmentThatMeetsADependencySaysSo(t *testing.T) { shelf := map[string]catalogue.Manifest{ "web": aContainer("web"), diff --git a/cmd/mesh-controller/unseen_integration_test.go b/cmd/mesh-controller/unseen_integration_test.go new file mode 100644 index 00000000..e15a2acb --- /dev/null +++ b/cmd/mesh-controller/unseen_integration_test.go @@ -0,0 +1,19 @@ +//go:build integration + +package main + +import ( + "context" + "strings" + "testing" +) + +func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { + err := pushCommand(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "--all") { + t.Fatalf("a bare push was not refused: %v", err) + } + if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { + t.Fatal("--all with --behind was accepted") + } +} diff --git a/cmd/mesh-controller/unseen_test.go b/cmd/mesh-controller/unseen_test.go index b675766b..6d8ad915 100644 --- a/cmd/mesh-controller/unseen_test.go +++ b/cmd/mesh-controller/unseen_test.go @@ -2,7 +2,6 @@ package main import ( "bytes" - "context" "reflect" "strings" "testing" @@ -140,16 +139,6 @@ func TestARunningContainersDataMovingIsAMoveAndAnAddedMountIsNot(t *testing.T) { } } -func TestAPushNamingNoMachineIsRefusedUnlessItSaysAll(t *testing.T) { - err := pushCommand(context.Background(), nil) - if err == nil || !strings.Contains(err.Error(), "--all") { - t.Fatalf("a bare push was not refused: %v", err) - } - if err := pushCommand(context.Background(), []string{"--all", "--behind"}); err == nil { - t.Fatal("--all with --behind was accepted") - } -} - func TestTheVerbsCarryReadReplaceAndMove(t *testing.T) { for _, c := range []struct { verb string diff --git a/cmd/mesh-controller/untaken_integration_test.go b/cmd/mesh-controller/untaken_integration_test.go new file mode 100644 index 00000000..1eb62216 --- /dev/null +++ b/cmd/mesh-controller/untaken_integration_test.go @@ -0,0 +1,80 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) { + open := aMesh(t) + heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server") + + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + if got := asked.untaken["anchor"]["route-proxy"]; got != 3 { + t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got) + } +} + +func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) { + // End to end through the store, so the condition above is reached by real data and not only by + // a constructed value: the machine reports, the mesh records, status asks. + open := aMesh(t) + heldOn(t, open, "anchor", "route-proxy", "ca", "server") + + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + if len(asked.untaken) == 0 { + t.Fatal("what the machine reported holding did not reach status") + } + if asked.well() { + t.Fatal("a mesh whose machine reported holds reads as well") + } +} + +func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) { + open := aMesh(t) + heldOn(t, open, "anchor", "route-proxy", "ca", "certs") + + asked, err := theThreeQuestions(t.Context(), open) + if err != nil { + t.Fatal(err) + } + body, err := statusAsJSON(asked) + if err != nil { + t.Fatal(err) + } + var doc struct { + Untaken []struct { + Node string `json:"node"` + Module string `json:"module"` + Held int `json:"held"` + } `json:"untaken"` + } + if err := json.Unmarshal(body, &doc); err != nil { + t.Fatal(err) + } + if len(doc.Untaken) != 1 { + t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body) + } + row := doc.Untaken[0] + if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 { + t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row) + } + // Absent rather than empty when nothing is held, so a well mesh's document does not carry a + // field a reader has to interpret. + clean, err := statusAsJSON(answers{}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(clean), "untaken") { + t.Fatalf("a mesh holding nothing still names untaken: %s", clean) + } +} diff --git a/cmd/mesh-controller/untaken_test.go b/cmd/mesh-controller/untaken_test.go index cbd47674..8b768b36 100644 --- a/cmd/mesh-controller/untaken_test.go +++ b/cmd/mesh-controller/untaken_test.go @@ -1,8 +1,6 @@ package main import ( - "encoding/json" - "strings" "testing" "github.com/novox/mesh-controller/internal/inventory" @@ -30,19 +28,6 @@ func heldOn(t *testing.T, open *stores, node, module string, ids ...string) { } } -func TestStatusNamesAModuleHeldBecauseNothingTookIt(t *testing.T) { - open := aMesh(t) - heldOn(t, open, "anchor", "route-proxy", "ca", "certs", "server") - - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - if got := asked.untaken["anchor"]["route-proxy"]; got != 3 { - t.Fatalf("status counted %d resources held for route-proxy, wanted 3", got) - } -} - func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) { // The whole of the fault. "all doing what they were told" was true throughout the outage, and // true is not the same as safe to act on: the machine was doing what it was told, and what it @@ -62,61 +47,3 @@ func TestAHeldModuleStopsTheMeshReadingAsWell(t *testing.T) { t.Fatal("the well condition is not stable") } } - -func TestAHeldModuleIsFoundFromWhatTheMachineReported(t *testing.T) { - // End to end through the store, so the condition above is reached by real data and not only by - // a constructed value: the machine reports, the mesh records, status asks. - open := aMesh(t) - heldOn(t, open, "anchor", "route-proxy", "ca", "server") - - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - if len(asked.untaken) == 0 { - t.Fatal("what the machine reported holding did not reach status") - } - if asked.well() { - t.Fatal("a mesh whose machine reported holds reads as well") - } -} - -func TestTheJSONStatusCarriesWhatIsHeldAndForWhichModule(t *testing.T) { - open := aMesh(t) - heldOn(t, open, "anchor", "route-proxy", "ca", "certs") - - asked, err := theThreeQuestions(t.Context(), open) - if err != nil { - t.Fatal(err) - } - body, err := statusAsJSON(asked) - if err != nil { - t.Fatal(err) - } - var doc struct { - Untaken []struct { - Node string `json:"node"` - Module string `json:"module"` - Held int `json:"held"` - } `json:"untaken"` - } - if err := json.Unmarshal(body, &doc); err != nil { - t.Fatal(err) - } - if len(doc.Untaken) != 1 { - t.Fatalf("the document carries %d untaken rows, wanted 1: %s", len(doc.Untaken), body) - } - row := doc.Untaken[0] - if row.Node != "anchor" || row.Module != "route-proxy" || row.Held != 2 { - t.Fatalf("the row is %+v, wanted anchor/route-proxy/2", row) - } - // Absent rather than empty when nothing is held, so a well mesh's document does not carry a - // field a reader has to interpret. - clean, err := statusAsJSON(answers{}) - if err != nil { - t.Fatal(err) - } - if strings.Contains(string(clean), "untaken") { - t.Fatalf("a mesh holding nothing still names untaken: %s", clean) - } -} diff --git a/cmd/mesh-controller/upgrades.go b/cmd/mesh-controller/upgrades.go index 31626b80..ea9ec498 100644 --- a/cmd/mesh-controller/upgrades.go +++ b/cmd/mesh-controller/upgrades.go @@ -326,8 +326,7 @@ func movesOfMerge(ctx context.Context, inv *inventory.Inventory, m link.SourceMo "from it\n", m.Owner, m.Repo, m.Base, m.Commit, already) return nil, nil } - fmt.Printf("%s/%s merged into %s (%.8s); nothing the mesh holds reads it\n", - m.Owner, m.Repo, m.Base, m.Commit) + fmt.Print(nothingReadsIt(m.Owner+"/"+m.Repo, m)) return nil, nil } // Said, never silent (novox/hq 04-ISSUES/215): a module built from this repository that follows @@ -531,7 +530,7 @@ func whyMoved(e inventory.Entry, read []inventory.ReadRepository, m link.SourceM return "read whole: " + whole } for _, r := range read { - if r.Own || !sameRepository(r.Repository, m) || (r.Ref != "" && r.Ref != m.Base) { + if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) { continue } if len(r.Paths) > 0 { @@ -615,6 +614,17 @@ func retireDeleted(ctx context.Context, inv *inventory.Inventory, e inventory.En } } +// nothingReadsIt is what is said of a merge no module the mesh holds reads. **A merge into a branch other than the +// trunk is said to start no delivery** (novox/hq ADR 0401 clause 3, issue 889): every delivery is cut from a +// repository's trunk, and whoever merged into a feature branch should not look for a broken trigger. +func nothingReadsIt(repository string, m link.SourceMoved) string { + if m.Base != "" && m.Base != "main" { + return fmt.Sprintf("%s merged into %s (%.8s), not the trunk of %s; a merge into any other branch starts no "+ + "delivery\n", repository, m.Base, m.Commit, repository) + } + return fmt.Sprintf("%s merged into %s (%.8s); nothing the mesh holds reads it\n", repository, m.Base, m.Commit) +} + // sourceIs is whether a recorded source is the repository and branch a merge announced. A source on // the git seat is recorded as its path on the forge; one elsewhere as the URL it was cloned from. // An empty recorded ref is the repository's default branch, which is what a merge into the base @@ -623,8 +633,18 @@ func sourceIs(s inventory.Source, m link.SourceMoved) bool { if !sameRepository(s.Repository, m) { return false } - ref := followedBranch(s.Ref) - return ref == "" || ref == m.Base + return followsBase(s.Ref, m.Base) +} + +// followsBase is whether a recorded ref follows the branch a merge went into. **An empty ref is the +// trunk, main, and nothing else** (novox/hq ADR 0401 clause 3, issue 889): it used to match a merge +// into any branch, so a merge into a feature branch started a delivery. A ref recorded as a branch +// keeps following that branch. +func followsBase(ref, base string) bool { + if ref = strings.TrimSpace(followedBranch(ref)); ref == "" { + ref = "main" + } + return ref == base } // commitRef is a ref that names a commit rather than a branch: what `build --ref ` asks for. @@ -663,7 +683,7 @@ func sameRepository(repository string, m link.SourceMoved) bool { // read, as before. func readsFrom(read []inventory.ReadRepository, m link.SourceMoved) bool { for _, r := range read { - if r.Own || !sameRepository(r.Repository, m) || (r.Ref != "" && r.Ref != m.Base) { + if r.Own || !sameRepository(r.Repository, m) || !followsBase(r.Ref, m.Base) { continue } if len(r.Paths) == 0 || len(m.Paths) == 0 || m.PathsTruncated { @@ -757,7 +777,7 @@ func unchangedOfMerge(m link.SourceMoved, entries []inventory.Entry, // branch, as a context beside its own. func packagesSaid(read []inventory.ReadRepository, m link.SourceMoved) bool { for _, r := range read { - if !r.Own && sameRepository(r.Repository, m) && (r.Ref == "" || r.Ref == m.Base) && len(r.Paths) > 0 { + if !r.Own && sameRepository(r.Repository, m) && followsBase(r.Ref, m.Base) && len(r.Paths) > 0 { return true } } diff --git a/cmd/mesh-controller/urgent_unhealthy_test.go b/cmd/mesh-controller/urgent_unhealthy_test.go index 661ff392..3ad82512 100644 --- a/cmd/mesh-controller/urgent_unhealthy_test.go +++ b/cmd/mesh-controller/urgent_unhealthy_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/vault_handed_test.go b/cmd/mesh-controller/vault_handed_test.go index 51349b3e..8e9ca26b 100644 --- a/cmd/mesh-controller/vault_handed_test.go +++ b/cmd/mesh-controller/vault_handed_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/vault_secrets_integration_test.go b/cmd/mesh-controller/vault_secrets_integration_test.go new file mode 100644 index 00000000..989d17f1 --- /dev/null +++ b/cmd/mesh-controller/vault_secrets_integration_test.go @@ -0,0 +1,229 @@ +//go:build integration + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log" + "strings" + "sync" + "testing" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAModulesSecretsAreTheVaultsOnceItIsMarkedMoved(t *testing.T) { + open := aMesh(t) + register(t, open, aBot("moved-bot", true)) + register(t, open, aBot("kept-bot", false)) + for module, want := range map[string]bool{"moved-bot": true, "kept-bot": false, "nowhere": false} { + got, err := secretsOfTheVault(module) + if err != nil || got != want { + t.Errorf("%s: the vault's %v (%v), want %v", module, got, err, want) + } + } +} + +func TestAnAcceptedValueCrossesTheBusOnlySealedToTheVault(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + before := handActConn + handActConn = conn + t.Cleanup(func() { handActConn = before }) + + var tapped bytes.Buffer + var tapMu sync.Mutex + tap, err := conn.Subscribe(">", func(m *nats.Msg) { + tapMu.Lock() + defer tapMu.Unlock() + tapped.WriteString(m.Subject + " " + string(m.Data) + "\n") + }) + if err != nil { + t.Fatal(err) + } + defer tap.Unsubscribe() + + vault := newStandInVault(t) + var subjects []string + holder, err := conn.Subscribe(broker.CalledPattern(link.SeatToolSubject("mesh-vault", "accept")), func(m *nats.Msg) { + subjects = append(subjects, m.Subject) + var args map[string]any + _ = json.Unmarshal(m.Data, &args) + result, err := vault.ask("accept", args) + answer := link.Answer{Result: result} + if err != nil { + answer = link.Answer{Error: err.Error()} + } + raw, _ := json.Marshal(answer) + _ = m.Respond(raw) + }) + if err != nil { + t.Fatal(err) + } + defer holder.Unsubscribe() + + var logged bytes.Buffer + was := log.Writer() + log.SetOutput(&logged) + t.Cleanup(func() { log.SetOutput(was) }) + + const value = "pw-of-last-year-that-must-not-show" + words, err := acceptIntoVault(theVault(context.Background()), provisionArgs("novox", "postgres"), value, "jochen") + if err != nil { + t.Fatal(err) + } + if vault.taken["accept"] != value { + t.Fatalf("the vault opened %q", vault.taken["accept"]) + } + if err := conn.Flush(); err != nil { + t.Fatal(err) + } + tapMu.Lock() + crossed := tapped.String() + tapMu.Unlock() + if !strings.Contains(crossed, "mesh.seat.mesh-vault.call.accept.controller") { + t.Fatalf("the tap heard no call to the vault: %s", crossed) + } + for _, s := range subjects { + if !strings.HasSuffix(s, ".controller") { + t.Errorf("the vault was asked on %s, not in the controller's name", s) + } + } + for where, text := range map[string]string{"the bus": crossed, "the log": logged.String(), "the answer": words} { + if strings.Contains(text, value) || strings.Contains(text, "must-not-show") { + t.Errorf("the value crossed %s in the clear", where) + } + } + if !strings.Contains(words, "sealed at this terminal") { + t.Errorf("the answer: %q", words) + } + // A vault that refuses the first step: nothing is sealed or sent. + vault.refuse = errors.New("the vault has no operator key") + if _, err := acceptIntoVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), value, "jochen"); err == nil || + !strings.Contains(err.Error(), "nothing was read or sealed") { + t.Errorf("a refused call said %v", err) + } +} + +func TestAnIssuedValueForAProviderFilledTheNewWayGoesToTheVault(t *testing.T) { + var r issuedRecorder + vault := newStandInVault(t) + taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { + t.Fatal("the controller asked the provider's tool itself") + return link.Answer{}, nil + }) + taker.keep = func(string, string, string) (inventory.IssuedKept, error) { + t.Fatal("today's store was asked to keep it") + return inventory.IssuedKept{}, nil + } + taker.ofTheVault = switchOf(func(m string) bool { return m == "plex" }) + taker.vault = vault.ask + words, err := taker.take("plex-api") + if err != nil { + t.Fatal(err) + } + if len(r.sent) != 0 { + t.Errorf("machines were sent: %v", r.sent) + } + if len(r.acts) != 1 || r.acts[0].Verb != handActIssued || !strings.Contains(r.acts[0].Why, "kept in the vault") { + t.Errorf("the act: %+v", r.acts) + } + said := vault.said() + for _, want := range []string{`"verb":"issued"`, `"provider":"ace"`, `"module":"plex"`, `"tool":"plex_issued_credential"`} { + if !strings.Contains(said, want) { + t.Errorf("the vault was told %s, without %s", said, want) + } + } + if strings.Contains(said, "seal_to") || strings.Contains(said, "sealed") || strings.Contains(said, `"call"`) { + t.Errorf("the controller carried a key or a sealed value: %s", said) + } + if !strings.Contains(words, "provision/ace/plex-api") { + t.Errorf("said %q", words) + } + // The same value again: the vault writes nothing, and nothing is recorded. + again := newStandInVault(t) + again.again = true + r2 := issuedRecorder{} + t2 := r2.taker(t, issuedSealed(t, "the-servers-own-token")) + t2.ofTheVault, t2.vault = switchOf(func(string) bool { return true }), again.ask + if words, err := t2.take("plex-api"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { + t.Errorf("%q %v %+v", words, err, r2.acts) + } + // The vault refuses (its tool refused it): said, nothing recorded. + refused := newStandInVault(t) + refused.refuse = errors.New("its tool plex_issued_credential refused: this tool answers the vault alone") + r4 := issuedRecorder{} + t4 := r4.taker(t, issuedSealed(t, "x")) + t4.ofTheVault, t4.vault = switchOf(func(string) bool { return true }), refused.ask + if _, err := t4.take("plex-api"); err == nil || !strings.Contains(err.Error(), "answers the vault alone") || len(r4.acts) != 0 { + t.Errorf("a refusal: %v, acts %d", err, len(r4.acts)) + } + + // Filled the old way: today's path. + var r3 issuedRecorder + t3 := r3.taker(t, issuedSealed(t, "the-servers-own-token")) + t3.ofTheVault = switchOf(func(string) bool { return false }) + t3.vault = func(string, map[string]any) (json.RawMessage, error) { + t.Fatal("the vault was asked for a provider filled the old way") + return nil, nil + } + if _, err := t3.take("plex-api"); err != nil || len(r3.kept) != 1 { + t.Errorf("today's path: %v, kept %d", err, len(r3.kept)) + } +} + +func TestAnIssuedValuePerHolderNamesTheHoldersToTheVault(t *testing.T) { + var r issuedRecorder + vault := newStandInVault(t) + vault.perHolder = true + taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { + t.Fatal("the controller asked the provider's tool itself") + return link.Answer{}, nil + }) + taker.providers = func(string) ([]issuedProvider, bool, error) { + return []issuedProvider{{Node: "novox", Module: "gitea", Tool: "gitea_agent_token", Holders: []issuedHolder{ + {Node: "anchor", Module: "claude-code"}, {Node: "laptop", Module: "claude-code"}, + {Node: "shanks", Module: "claude-code"}}}}, true, nil + } + taker.ofTheVault = switchOf(func(m string) bool { return m == "gitea" }) + taker.vault = vault.ask + words, err := taker.take("gitea-read") + if err != nil { + t.Fatal(err) + } + said := vault.said() + if !strings.Contains(said, `"holders":[{"node":"anchor","module":"claude-code"},{"node":"laptop","module":"claude-code"},{"node":"shanks","module":"claude-code"}]`) { + t.Errorf("the vault was told %s", said) + } + for _, want := range []string{"provision/novox/gitea-read/anchor/claude-code", "provision/novox/gitea-read/laptop/claude-code", + "gave nothing for claude-code on shanks"} { + if !strings.Contains(words, want) { + t.Errorf("said %q, without %q", words, want) + } + } + if len(r.acts) != 1 || !strings.Contains(r.acts[0].Why, "claude-code on anchor") || !strings.Contains(r.acts[0].Why, "claude-code on laptop") { + t.Errorf("the act: %+v", r.acts) + } + if len(r.sent) != 0 || len(r.kept) != 0 { + t.Error("the controller kept or sent something") + } + // The same values again: nothing written, nothing recorded. + again := newStandInVault(t) + again.perHolder, again.again = true, true + r2 := issuedRecorder{} + t2 := r2.taker(t, nil) + t2.providers, t2.ofTheVault, t2.vault = taker.providers, taker.ofTheVault, again.ask + if words, err := t2.take("gitea-read"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { + t.Errorf("again: %q %v %+v", words, err, r2.acts) + } +} diff --git a/cmd/mesh-controller/vault_secrets_test.go b/cmd/mesh-controller/vault_secrets_test.go index 6ded75a4..e31041b7 100644 --- a/cmd/mesh-controller/vault_secrets_test.go +++ b/cmd/mesh-controller/vault_secrets_test.go @@ -1,24 +1,16 @@ package main import ( - "bytes" - "context" "encoding/json" "errors" "fmt" - "log" "strings" "sync" "testing" - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" "github.com/novox/mesh-controller/internal/secrets" - "github.com/novox/mesh-controller/internal/testbus" ) // The controller's secret verbs reach the vault only for a module filled the new way (design 57's V2, issue 675, @@ -128,17 +120,6 @@ func (v *standInVault) said() string { // A module's secrets are the vault's once the catalogue marks it moved (design 57 V4, issue 647): its machine's // client asks the vault for them. Every module not marked, and one the catalogue does not hold, keeps today's path. -func TestAModulesSecretsAreTheVaultsOnceItIsMarkedMoved(t *testing.T) { - open := aMesh(t) - register(t, open, aBot("moved-bot", true)) - register(t, open, aBot("kept-bot", false)) - for module, want := range map[string]bool{"moved-bot": true, "kept-bot": false, "nowhere": false} { - got, err := secretsOfTheVault(module) - if err != nil || got != want { - t.Errorf("%s: the vault's %v (%v), want %v", module, got, err, want) - } - } -} // A switch that cannot be read does nothing, and says why: never today's path by default. func TestASwitchThatCannotBeReadDoesNothing(t *testing.T) { @@ -175,89 +156,6 @@ func TestTheSwitchRunsOnePathByTheModulesForm(t *testing.T) { // `secret accept` for a module filled the new way: the value read at the terminal is sealed there to the key the // vault made for this one call; no clear value crosses the bus or reaches the controller's log. Over a real bus, // with a tap on every subject. -func TestAnAcceptedValueCrossesTheBusOnlySealedToTheVault(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - before := handActConn - handActConn = conn - t.Cleanup(func() { handActConn = before }) - - var tapped bytes.Buffer - var tapMu sync.Mutex - tap, err := conn.Subscribe(">", func(m *nats.Msg) { - tapMu.Lock() - defer tapMu.Unlock() - tapped.WriteString(m.Subject + " " + string(m.Data) + "\n") - }) - if err != nil { - t.Fatal(err) - } - defer tap.Unsubscribe() - - vault := newStandInVault(t) - var subjects []string - holder, err := conn.Subscribe(broker.CalledPattern(link.SeatToolSubject("mesh-vault", "accept")), func(m *nats.Msg) { - subjects = append(subjects, m.Subject) - var args map[string]any - _ = json.Unmarshal(m.Data, &args) - result, err := vault.ask("accept", args) - answer := link.Answer{Result: result} - if err != nil { - answer = link.Answer{Error: err.Error()} - } - raw, _ := json.Marshal(answer) - _ = m.Respond(raw) - }) - if err != nil { - t.Fatal(err) - } - defer holder.Unsubscribe() - - var logged bytes.Buffer - was := log.Writer() - log.SetOutput(&logged) - t.Cleanup(func() { log.SetOutput(was) }) - - const value = "pw-of-last-year-that-must-not-show" - words, err := acceptIntoVault(theVault(context.Background()), provisionArgs("novox", "postgres"), value, "jochen") - if err != nil { - t.Fatal(err) - } - if vault.taken["accept"] != value { - t.Fatalf("the vault opened %q", vault.taken["accept"]) - } - if err := conn.Flush(); err != nil { - t.Fatal(err) - } - tapMu.Lock() - crossed := tapped.String() - tapMu.Unlock() - if !strings.Contains(crossed, "mesh.seat.mesh-vault.call.accept.controller") { - t.Fatalf("the tap heard no call to the vault: %s", crossed) - } - for _, s := range subjects { - if !strings.HasSuffix(s, ".controller") { - t.Errorf("the vault was asked on %s, not in the controller's name", s) - } - } - for where, text := range map[string]string{"the bus": crossed, "the log": logged.String(), "the answer": words} { - if strings.Contains(text, value) || strings.Contains(text, "must-not-show") { - t.Errorf("the value crossed %s in the clear", where) - } - } - if !strings.Contains(words, "sealed at this terminal") { - t.Errorf("the answer: %q", words) - } - // A vault that refuses the first step: nothing is sealed or sent. - vault.refuse = errors.New("the vault has no operator key") - if _, err := acceptIntoVault(vault.ask, ownSecretArgs("g14", "dunst", "token"), value, "jochen"); err == nil || - !strings.Contains(err.Error(), "nothing was read or sealed") { - t.Errorf("a refused call said %v", err) - } -} // The desk's path for a module filled the new way (issue 723): the controller asks the vault once; the vault // records the ask, holds its bounds and asks the desk's prompt itself, so the controller never reaches the desk and @@ -341,122 +239,11 @@ func TestAnAskForAModuleFilledTheNewWayIsTheVaults(t *testing.T) { // provider's machine, module and tool; the vault asks the tool itself; the controller never reaches the tool and // nothing is kept, sealed or sent here. A provider filled the old way keeps today's path and the vault is never // asked. -func TestAnIssuedValueForAProviderFilledTheNewWayGoesToTheVault(t *testing.T) { - var r issuedRecorder - vault := newStandInVault(t) - taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { - t.Fatal("the controller asked the provider's tool itself") - return link.Answer{}, nil - }) - taker.keep = func(string, string, string) (inventory.IssuedKept, error) { - t.Fatal("today's store was asked to keep it") - return inventory.IssuedKept{}, nil - } - taker.ofTheVault = switchOf(func(m string) bool { return m == "plex" }) - taker.vault = vault.ask - words, err := taker.take("plex-api") - if err != nil { - t.Fatal(err) - } - if len(r.sent) != 0 { - t.Errorf("machines were sent: %v", r.sent) - } - if len(r.acts) != 1 || r.acts[0].Verb != handActIssued || !strings.Contains(r.acts[0].Why, "kept in the vault") { - t.Errorf("the act: %+v", r.acts) - } - said := vault.said() - for _, want := range []string{`"verb":"issued"`, `"provider":"ace"`, `"module":"plex"`, `"tool":"plex_issued_credential"`} { - if !strings.Contains(said, want) { - t.Errorf("the vault was told %s, without %s", said, want) - } - } - if strings.Contains(said, "seal_to") || strings.Contains(said, "sealed") || strings.Contains(said, `"call"`) { - t.Errorf("the controller carried a key or a sealed value: %s", said) - } - if !strings.Contains(words, "provision/ace/plex-api") { - t.Errorf("said %q", words) - } - // The same value again: the vault writes nothing, and nothing is recorded. - again := newStandInVault(t) - again.again = true - r2 := issuedRecorder{} - t2 := r2.taker(t, issuedSealed(t, "the-servers-own-token")) - t2.ofTheVault, t2.vault = switchOf(func(string) bool { return true }), again.ask - if words, err := t2.take("plex-api"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { - t.Errorf("%q %v %+v", words, err, r2.acts) - } - // The vault refuses (its tool refused it): said, nothing recorded. - refused := newStandInVault(t) - refused.refuse = errors.New("its tool plex_issued_credential refused: this tool answers the vault alone") - r4 := issuedRecorder{} - t4 := r4.taker(t, issuedSealed(t, "x")) - t4.ofTheVault, t4.vault = switchOf(func(string) bool { return true }), refused.ask - if _, err := t4.take("plex-api"); err == nil || !strings.Contains(err.Error(), "answers the vault alone") || len(r4.acts) != 0 { - t.Errorf("a refusal: %v, acts %d", err, len(r4.acts)) - } - - // Filled the old way: today's path. - var r3 issuedRecorder - t3 := r3.taker(t, issuedSealed(t, "the-servers-own-token")) - t3.ofTheVault = switchOf(func(string) bool { return false }) - t3.vault = func(string, map[string]any) (json.RawMessage, error) { - t.Fatal("the vault was asked for a provider filled the old way") - return nil, nil - } - if _, err := t3.take("plex-api"); err != nil || len(r3.kept) != 1 { - t.Errorf("today's path: %v, kept %d", err, len(r3.kept)) - } -} // `secret issued` names to the vault the provision's holders from that provider (issue 749): a tool may answer a // value for each, which the vault keeps as that holder's own; the controller says what was kept for whom and which // holder the tool gave nothing for, never a value. A provider answering one value is told the holders all the same // and keeps the provision's one credential. -func TestAnIssuedValuePerHolderNamesTheHoldersToTheVault(t *testing.T) { - var r issuedRecorder - vault := newStandInVault(t) - vault.perHolder = true - taker := r.taker(t, func(issuedProvider, map[string]any) (link.Answer, error) { - t.Fatal("the controller asked the provider's tool itself") - return link.Answer{}, nil - }) - taker.providers = func(string) ([]issuedProvider, bool, error) { - return []issuedProvider{{Node: "novox", Module: "gitea", Tool: "gitea_agent_token", Holders: []issuedHolder{ - {Node: "anchor", Module: "claude-code"}, {Node: "laptop", Module: "claude-code"}, - {Node: "shanks", Module: "claude-code"}}}}, true, nil - } - taker.ofTheVault = switchOf(func(m string) bool { return m == "gitea" }) - taker.vault = vault.ask - words, err := taker.take("gitea-read") - if err != nil { - t.Fatal(err) - } - said := vault.said() - if !strings.Contains(said, `"holders":[{"node":"anchor","module":"claude-code"},{"node":"laptop","module":"claude-code"},{"node":"shanks","module":"claude-code"}]`) { - t.Errorf("the vault was told %s", said) - } - for _, want := range []string{"provision/novox/gitea-read/anchor/claude-code", "provision/novox/gitea-read/laptop/claude-code", - "gave nothing for claude-code on shanks"} { - if !strings.Contains(words, want) { - t.Errorf("said %q, without %q", words, want) - } - } - if len(r.acts) != 1 || !strings.Contains(r.acts[0].Why, "claude-code on anchor") || !strings.Contains(r.acts[0].Why, "claude-code on laptop") { - t.Errorf("the act: %+v", r.acts) - } - if len(r.sent) != 0 || len(r.kept) != 0 { - t.Error("the controller kept or sent something") - } - // The same values again: nothing written, nothing recorded. - again := newStandInVault(t) - again.perHolder, again.again = true, true - r2 := issuedRecorder{} - t2 := r2.taker(t, nil) - t2.providers, t2.ofTheVault, t2.vault = taker.providers, taker.ofTheVault, again.ask - if words, err := t2.take("gitea-read"); err != nil || len(r2.acts) != 0 || !strings.Contains(words, "nothing was written") { - t.Errorf("again: %q %v %+v", words, err, r2.acts) - } -} // The holders named are the provision's holders from that provider, each consuming module on each machine once. func TestTheHoldersNamedAreThoseOfThatProvider(t *testing.T) { diff --git a/cmd/mesh-controller/vault_switch_test.go b/cmd/mesh-controller/vault_switch_test.go index 6cf12f65..98c9f189 100644 --- a/cmd/mesh-controller/vault_switch_test.go +++ b/cmd/mesh-controller/vault_switch_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/verb_looks_test.go b/cmd/mesh-controller/verb_looks_test.go index d16e190a..ced2f5c3 100644 --- a/cmd/mesh-controller/verb_looks_test.go +++ b/cmd/mesh-controller/verb_looks_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/waiting_provider_test.go b/cmd/mesh-controller/waiting_provider_test.go index e220363f..d324ad83 100644 --- a/cmd/mesh-controller/waiting_provider_test.go +++ b/cmd/mesh-controller/waiting_provider_test.go @@ -1,3 +1,5 @@ +//go:build integration + package main import ( diff --git a/cmd/mesh-controller/walk_said_integration_test.go b/cmd/mesh-controller/walk_said_integration_test.go new file mode 100644 index 00000000..eec2119c --- /dev/null +++ b/cmd/mesh-controller/walk_said_integration_test.go @@ -0,0 +1,266 @@ +//go:build integration + +package main + +import ( + "encoding/json" + "os" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + next := planMovedHeard(t) + register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "app"); err != nil { + t.Fatal(err) + } + cut := time.Now().UTC() + waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", + Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, + Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &waiting); err != nil { + t.Fatal(err) + } + if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { + t.Fatal(err) + } + started := next(inventory.WalkStarted) + touches, _ := json.Marshal(started["touches"]) + if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` || + started["budget_class"] != inventory.ClassLeaf || + started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) { + t.Fatalf("the start: %v", started) + } + if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil { + t.Fatal(err) + } + ended := next(inventory.WalkEnded) + touches, _ = json.Marshal(ended["touches"]) + if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` { + t.Fatalf("the end, which sent nothing: %v", ended) + } +} + +func planMovedHeard(t *testing.T) func(phase string) map[string]any { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + before := handActConn + handActConn = js.Conn() + t.Cleanup(func() { handActConn = before }) + heard := make(chan *nats.Msg, 64) + sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sub.Unsubscribe() }) + return func(phase string) map[string]any { + t.Helper() + deadline := time.After(5 * time.Second) + for { + select { + case m := <-heard: + var said map[string]any + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + if said["phase"] == phase { + return said + } + case <-deadline: + t.Fatalf("no walk was said %s", phase) + } + } + } +} + +func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + next := planMovedHeard(t) + register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) + if _, err := assign(ctx, open, "anchor", "app"); err != nil { + t.Fatal(err) + } + for _, verb := range []string{"stop", "close"} { + cut := time.Now().UTC() + walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main", + Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, + Modules: map[string]*inventory.PlanModule{"app": {}}, + Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &walk); err != nil { + t.Fatal(err) + } + if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil { + t.Fatal(err) + } + ended := next(inventory.WalkEnded) + if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped { + t.Fatalf("plans %s: the end is %v", verb, ended) + } + p, err := open.inventory.PlanByID(ctx, walk.ID) + if err != nil { + t.Fatal(err) + } + if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" { + t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery) + } + } +} + +func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asksRecorded(t) + next := planMovedHeard(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + ended := before.Add(time.Minute) + failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, + Note: "a failed to build in tier 0", + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.KilledByHand}}, + Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(ctx, &failed); err != nil { + t.Fatal(err) + } + if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil { + t.Fatal(err) + } + started := next(inventory.WalkStarted) + if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf { + t.Fatalf("the retried start: %v", started) + } +} + +func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) { + b := aBacklog(t) + next := planMovedHeard(t) + if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil { + t.Fatal(err) + } + started := next(inventory.WalkStarted) + if p := b.release(t); started["walk"] != p.ID { + t.Fatalf("the release's start: %v", started) + } +} + +func asTyped(t *testing.T, args ...string) error { + t.Helper() + was := os.Args + os.Args = append([]string{"mesh-controller"}, args...) + t.Cleanup(func() { os.Args = was }) + err := run() + // run gives back the lease as the process would end; the test's process goes on, and takes it again. + theLease.mu.Lock() + theLease.held, theLease.kv, theLease.noBus = nil, nil, false + theLease.mu.Unlock() + return err +} + +func TestAWalkEndedByCancelOrClearIsSaidEnded(t *testing.T) { + _, ask := aBuildQueue(t) + open := aMesh(t) + next := planMovedHeard(t) + twoTiers(t, open) + for _, verb := range [][]string{{"cancel"}, {"clear"}} { + id := link.NewBuildID(time.Now()) + ask(id, "https://forge.example/novox/a.git") + asked, _ := link.BuildAskedAt(id) + walk := aStartedWalkAsking(t, open, "plan-308-"+verb[0], id, asked) + args := verb + if verb[0] == "cancel" { + args = append(args, id) + } + if err := asTyped(t, args...); err != nil { + t.Fatal(err) + } + ended := next(inventory.WalkEnded) + if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkFailed { + t.Fatalf("%s: the end is %v", verb[0], ended) + } + // Ended, and so the next walk may be cut on the same repository. + p, _ := open.inventory.PlanByID(t.Context(), walk.ID) + if p.State != inventory.PlanFailed { + t.Fatalf("%s left the walk %s", verb[0], p.State) + } + } +} + +func TestAWalkRestartedByARebuildIsSaidStarted(t *testing.T) { + open := aMesh(t) + asksRecorded(t) + next := planMovedHeard(t) + twoTiers(t, open) + before := time.Now().UTC().Add(-time.Hour) + ended := before.Add(time.Minute) + failed := inventory.Plan{ID: "plan-308-rebuild", Repository: "novox/a", Branch: "main", Commit: "c0ffee", + Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, + Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", + Why: link.CancelledByHand}}, + Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} + if err := open.inventory.SavePlan(t.Context(), &failed); err != nil { + t.Fatal(err) + } + if err := asTyped(t, "rebuild", "a"); err != nil { + t.Fatal(err) + } + if started := next(inventory.WalkStarted); started["walk"] != failed.ID { + t.Fatalf("the rebuilt walk's start: %v", started) + } +} + +func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asksRecorded(t) + twoTiers(t, open) + for _, again := range [][]string{{"plans", "retry"}, {"rebuild", "a"}} { + before := time.Now().UTC().Add(-time.Hour) + walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) + if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { + t.Fatal(err) + } + // The build asked before the stop fails after it. + p, _ := open.inventory.PlanByID(ctx, walk.ID) + p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke" + if err := open.inventory.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + args := again + if again[0] == "plans" { + args = append(append([]string(nil), again...), walk.ID) + } + if err := asTyped(t, args...); err != nil { + t.Fatal(err) + } + p, _ = open.inventory.PlanByID(ctx, walk.ID) + if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { + t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) + } + asking, _ := link.BuildAskedAt(p.Modules["a"].Build) + planBuilt(ctx, open, "a", "c0ffee", "it broke again", asking, p.Modules["a"].Build) + if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { + t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) + } + } +} diff --git a/cmd/mesh-controller/walk_said_test.go b/cmd/mesh-controller/walk_said_test.go index 22ef9a15..aac59370 100644 --- a/cmd/mesh-controller/walk_said_test.go +++ b/cmd/mesh-controller/walk_said_test.go @@ -3,18 +3,13 @@ package main import ( "context" "encoding/json" - "os" "reflect" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // fakeWalkFacts is the inventory as far as a walk's touches read it. @@ -234,176 +229,21 @@ func TestPlanMovedCarriesTheWalkWholeAndOnItsStartWhatItTouches(t *testing.T) { // Live, with the store and a bus: a walk let go by the delivery's verb is said started with what it touches and // its bound, and stopped by the verb is said ended, stopped (novox/hq ADR 0308 How it is checked §1). -func TestAWalkLetGoAndStoppedByVerbsIsSaidStartedAndEnded(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - cut := time.Now().UTC() - waiting := inventory.Plan{ID: "plan-308-live", Repository: "novox/mesh-catalog", Branch: "main", Commit: "c8c8c8c8", - Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Delivery: &inventory.PlanDelivery{Awaits: catalogue.DeliverySeat}, - Times: &inventory.PlanTimes{Cut: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &waiting); err != nil { - t.Fatal(err) - } - if err := deliveryCommand(ctx, []string{"go", waiting.ID, "--why", "its turn"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - touches, _ := json.Marshal(started["touches"]) - if started["walk"] != waiting.ID || string(touches) != `[{"machine":"anchor","modules":["app"]}]` || - started["budget_class"] != inventory.ClassLeaf || - started["bound_seconds"] != float64(2*deliveryBudgets[inventory.ClassLeaf]/time.Second) { - t.Fatalf("the start: %v", started) - } - if err := deliveryCommand(ctx, []string{"stop", waiting.ID, "--why", "a test", "--by", "a person"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - touches, _ = json.Marshal(ended["touches"]) - if ended["walk"] != waiting.ID || ended["outcome"] != inventory.WalkStopped || string(touches) != `[]` { - t.Fatalf("the end, which sent nothing: %v", ended) - } -} // planMovedHeard raises a bus of the test's own, points the verbs' hand-act connection at it, and returns what // waits for the next plan-moved said with the given phase, failing the test when none comes within 5 s. -func planMovedHeard(t *testing.T) func(phase string) map[string]any { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - before := handActConn - handActConn = js.Conn() - t.Cleanup(func() { handActConn = before }) - heard := make(chan *nats.Msg, 64) - sub, err := js.Conn().ChanSubscribe(link.SeatEventSubject(link.MeshControllerSeat, link.KeyPlanMoved), heard) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = sub.Unsubscribe() }) - return func(phase string) map[string]any { - t.Helper() - deadline := time.After(5 * time.Second) - for { - select { - case m := <-heard: - var said map[string]any - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said["phase"] == phase { - return said - } - case <-deadline: - t.Fatalf("no walk was said %s", phase) - } - } - } -} // Live, with the store and a bus: a walk a person stops with `plans stop` is said ended, outcome stopped, by the // verb itself, and is kept stopped by that person; `plans close` ends one the same way (novox/hq ADR 0308 // decision 1, review 15 on mesh-controller#247). -func TestAWalkStoppedOrClosedByAPersonIsSaidEndedStopped(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - next := planMovedHeard(t) - register(t, open, catalogue.Manifest{Module: "app", Version: "1"}) - if _, err := assign(ctx, open, "anchor", "app"); err != nil { - t.Fatal(err) - } - for _, verb := range []string{"stop", "close"} { - cut := time.Now().UTC() - walk := inventory.Plan{ID: "plan-308-" + verb, Repository: "novox/mesh-catalog", Branch: "main", - Commit: "c8c8c8c8", Created: cut, State: inventory.PlanBuilding, Tiers: [][]string{{"app"}}, - Modules: map[string]*inventory.PlanModule{"app": {}}, - Times: &inventory.PlanTimes{Cut: &cut, Started: &cut, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &walk); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{verb, walk.ID, "--why", "a test"}); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkStopped { - t.Fatalf("plans %s: the end is %v", verb, ended) - } - p, err := open.inventory.PlanByID(ctx, walk.ID) - if err != nil { - t.Fatal(err) - } - if p.Outcome() != inventory.WalkStopped || p.Delivery == nil || p.Delivery.StoppedWhy != "a test" { - t.Fatalf("plans %s kept the walk %s, %+v", verb, p.Outcome(), p.Delivery) - } - } -} // Live, with the store and a bus: a failed walk retried with `plans retry` starts again, and the verb says its // start (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). -func TestAWalkRetriedByAPersonIsSaidStartedAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-retry", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tier: 0, Tiers: [][]string{{"a"}, {"b"}}, - Note: "a failed to build in tier 0", - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.KilledByHand}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(ctx, &failed); err != nil { - t.Fatal(err) - } - if err := plansCommand(ctx, []string{"retry", failed.ID}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if started["walk"] != failed.ID || started["budget_class"] != inventory.ClassLeaf { - t.Fatalf("the retried start: %v", started) - } -} // Live, with the store and a bus: the release plan `upgrade release-backlog` opens is said started by the verb // (novox/hq ADR 0308 decision 1, review 15 on mesh-controller#247). -func TestABacklogReleasedByAPersonIsSaidStarted(t *testing.T) { - b := aBacklog(t) - next := planMovedHeard(t) - if err := backlogCommand(t.Context(), "release-backlog", []string{"--why", "a test"}); err != nil { - t.Fatal(err) - } - started := next(inventory.WalkStarted) - if p := b.release(t); started["walk"] != p.ID { - t.Fatalf("the release's start: %v", started) - } -} // asTyped runs a verb as the controller's command line runs it, through run, as a person's terminal would. -func asTyped(t *testing.T, args ...string) error { - t.Helper() - was := os.Args - os.Args = append([]string{"mesh-controller"}, args...) - t.Cleanup(func() { os.Args = was }) - err := run() - // run gives back the lease as the process would end; the test's process goes on, and takes it again. - theLease.mu.Lock() - theLease.held, theLease.kv, theLease.noBus = nil, nil, false - theLease.mu.Unlock() - return err -} // aStartedWalkAsking keeps a live walk, said started, whose first tier asked a as build. func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time.Time) inventory.Plan { @@ -420,95 +260,10 @@ func aStartedWalkAsking(t *testing.T, open *stores, id, build string, asked time // Live, with the store and a bus: `cancel ` and `clear` fail the walk that asked the build, and the verb // says its end, failed (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). -func TestAWalkEndedByCancelOrClearIsSaidEnded(t *testing.T) { - _, ask := aBuildQueue(t) - open := aMesh(t) - next := planMovedHeard(t) - twoTiers(t, open) - for _, verb := range [][]string{{"cancel"}, {"clear"}} { - id := link.NewBuildID(time.Now()) - ask(id, "https://forge.example/novox/a.git") - asked, _ := link.BuildAskedAt(id) - walk := aStartedWalkAsking(t, open, "plan-308-"+verb[0], id, asked) - args := verb - if verb[0] == "cancel" { - args = append(args, id) - } - if err := asTyped(t, args...); err != nil { - t.Fatal(err) - } - ended := next(inventory.WalkEnded) - if ended["walk"] != walk.ID || ended["outcome"] != inventory.WalkFailed { - t.Fatalf("%s: the end is %v", verb[0], ended) - } - // Ended, and so the next walk may be cut on the same repository. - p, _ := open.inventory.PlanByID(t.Context(), walk.ID) - if p.State != inventory.PlanFailed { - t.Fatalf("%s left the walk %s", verb[0], p.State) - } - } -} // Live, with the store and a bus: `rebuild ` joining a failed walk sets it building again, and the verb // says its start (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). -func TestAWalkRestartedByARebuildIsSaidStarted(t *testing.T) { - open := aMesh(t) - asksRecorded(t) - next := planMovedHeard(t) - twoTiers(t, open) - before := time.Now().UTC().Add(-time.Hour) - ended := before.Add(time.Minute) - failed := inventory.Plan{ID: "plan-308-rebuild", Repository: "novox/a", Branch: "main", Commit: "c0ffee", - Created: before, State: inventory.PlanFailed, Tiers: [][]string{{"a"}, {"b"}}, - Modules: map[string]*inventory.PlanModule{"a": {State: "failed", AskedAt: &before, Build: "build-1", - Why: link.CancelledByHand}}, - Times: &inventory.PlanTimes{Cut: &before, Started: &before, Ended: &ended, Class: inventory.ClassLeaf}} - if err := open.inventory.SavePlan(t.Context(), &failed); err != nil { - t.Fatal(err) - } - if err := asTyped(t, "rebuild", "a"); err != nil { - t.Fatal(err) - } - if started := next(inventory.WalkStarted); started["walk"] != failed.ID { - t.Fatalf("the rebuilt walk's start: %v", started) - } -} // Live, with the store: a walk a person stopped and then retried, by `plans retry` or by `rebuild`, walks again // without the stop, so when it fails again it ends failed, not stopped, and the search for the merge that // brought the failure is not kept from it (novox/hq ADR 0308 decision 1, review 21 on mesh-controller#247). -func TestAWalkStoppedByHandAndRetriedEndsFailedWhenItFailsAgain(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asksRecorded(t) - twoTiers(t, open) - for _, again := range [][]string{{"plans", "retry"}, {"rebuild", "a"}} { - before := time.Now().UTC().Add(-time.Hour) - walk := aStartedWalkAsking(t, open, "plan-308-again-"+again[0], "build-1", before) - if err := asTyped(t, "plans", "stop", walk.ID, "--why", "a test"); err != nil { - t.Fatal(err) - } - // The build asked before the stop fails after it. - p, _ := open.inventory.PlanByID(ctx, walk.ID) - p.Modules["a"].State, p.Modules["a"].Why = "failed", "it broke" - if err := open.inventory.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - args := again - if again[0] == "plans" { - args = append(append([]string(nil), again...), walk.ID) - } - if err := asTyped(t, args...); err != nil { - t.Fatal(err) - } - p, _ = open.inventory.PlanByID(ctx, walk.ID) - if p.State != inventory.PlanBuilding || (p.Delivery != nil && p.Delivery.Stopped != "") { - t.Fatalf("%s: the walk is %s and keeps its stop: %+v", again[0], p.State, p.Delivery) - } - asking, _ := link.BuildAskedAt(p.Modules["a"].Build) - planBuilt(ctx, open, "a", "c0ffee", "it broke again", asking, p.Modules["a"].Build) - if p, _ = open.inventory.PlanByID(ctx, walk.ID); p.State != inventory.PlanFailed || p.Outcome() != inventory.WalkFailed { - t.Fatalf("%s: failing again, the walk is %s and ends %s", again[0], p.State, p.Outcome()) - } - } -} diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go new file mode 100644 index 00000000..ae027544 --- /dev/null +++ b/cmd/mesh-controller/walk_wait_rebuild_754_integration_test.go @@ -0,0 +1,63 @@ +//go:build integration + +package main + +import ( + "testing" + "time" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func TestARebuildInTheWaitIsNotTheWalksBuild(t *testing.T) { + open := aMesh(t) + ctx := t.Context() + asked := asksRecorded(t) + twoTiers(t, open) + created := time.Now().UTC().Add(-10 * time.Minute) + plan := inventory.Plan{ID: "plan-754", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: created, + State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{}, + Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} + if err := open.inventory.SavePlan(ctx, &plan); err != nil { + t.Fatal(err) + } + + // A rebuild of a, beside the waiting walk: not joined, and its outcome not taken. + joined, said, err := joinAPlan(ctx, open, "a") + if err != nil || joined || said == "" { + t.Fatalf("a rebuild joined a waiting walk: %v %q %v", joined, said, err) + } + planBuilt(ctx, open, "a", "c0ffee", "", time.Now().UTC(), "build-rebuild") + p, _ := open.inventory.PlanByID(ctx, plan.ID) + if s := p.Modules["a"]; s != nil && s.State != "" { + t.Fatalf("the waiting walk took a rebuild as its build: %+v", s) + } + if len(*asked) != 0 { + t.Fatalf("a waiting walk asked %v", *asked) + } + + // Let go: both are asked, and their outcomes complete the tier. + if _, err := letGo(ctx, open.inventory, plan.ID, "operator", "test"); err != nil { + t.Fatal(err) + } + advancePlans(ctx, open) + p, _ = open.inventory.PlanByID(ctx, plan.ID) + for _, m := range []string{"a", "b"} { + if s := p.Modules[m]; s == nil || s.State != "asked" || s.Build == "" { + t.Fatalf("let go, %s is %+v", m, s) + } + } + if len(*asked) != 2 { + t.Fatalf("asked %v", *asked) + } + for _, m := range []string{"a", "b"} { + id := p.Modules[m].Build + at, _ := link.BuildAskedAt(id) + planBuilt(ctx, open, m, "c0ffee", "", at, id) + } + advancePlans(ctx, open) + if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Open() && p.Tier == 0 { + t.Fatalf("the walk did not get past its tier: %s at %d, %q", p.State, p.Tier, p.Note) + } +} diff --git a/cmd/mesh-controller/walk_wait_rebuild_754_test.go b/cmd/mesh-controller/walk_wait_rebuild_754_test.go index 962fcebd..ee9616b3 100644 --- a/cmd/mesh-controller/walk_wait_rebuild_754_test.go +++ b/cmd/mesh-controller/walk_wait_rebuild_754_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/novox/mesh-controller/internal/inventory" - "github.com/novox/mesh-controller/internal/link" ) // novox/hq issue 754: a walk waiting for its delivery's word takes no build's outcome, nor does a walk take one for @@ -69,54 +68,3 @@ func TestATierPartlyAskedAsksTheRest(t *testing.T) { // novox/hq issue 754, as seen on 2026-10-14: a rebuild lands while the walk waits for its word; the walk is let go; // it asks every module of its tier, and completes it once those builds come in. -func TestARebuildInTheWaitIsNotTheWalksBuild(t *testing.T) { - open := aMesh(t) - ctx := t.Context() - asked := asksRecorded(t) - twoTiers(t, open) - created := time.Now().UTC().Add(-10 * time.Minute) - plan := inventory.Plan{ID: "plan-754", Repository: "novox/a", Branch: "main", Commit: "c0ffee", Created: created, - State: inventory.PlanBuilding, Tiers: [][]string{{"a", "b"}}, Modules: map[string]*inventory.PlanModule{}, - Delivery: &inventory.PlanDelivery{Awaits: "mesh-delivery"}} - if err := open.inventory.SavePlan(ctx, &plan); err != nil { - t.Fatal(err) - } - - // A rebuild of a, beside the waiting walk: not joined, and its outcome not taken. - joined, said, err := joinAPlan(ctx, open, "a") - if err != nil || joined || said == "" { - t.Fatalf("a rebuild joined a waiting walk: %v %q %v", joined, said, err) - } - planBuilt(ctx, open, "a", "c0ffee", "", time.Now().UTC(), "build-rebuild") - p, _ := open.inventory.PlanByID(ctx, plan.ID) - if s := p.Modules["a"]; s != nil && s.State != "" { - t.Fatalf("the waiting walk took a rebuild as its build: %+v", s) - } - if len(*asked) != 0 { - t.Fatalf("a waiting walk asked %v", *asked) - } - - // Let go: both are asked, and their outcomes complete the tier. - if _, err := letGo(ctx, open.inventory, plan.ID, "operator", "test"); err != nil { - t.Fatal(err) - } - advancePlans(ctx, open) - p, _ = open.inventory.PlanByID(ctx, plan.ID) - for _, m := range []string{"a", "b"} { - if s := p.Modules[m]; s == nil || s.State != "asked" || s.Build == "" { - t.Fatalf("let go, %s is %+v", m, s) - } - } - if len(*asked) != 2 { - t.Fatalf("asked %v", *asked) - } - for _, m := range []string{"a", "b"} { - id := p.Modules[m].Build - at, _ := link.BuildAskedAt(id) - planBuilt(ctx, open, m, "c0ffee", "", at, id) - } - advancePlans(ctx, open) - if p, _ = open.inventory.PlanByID(ctx, plan.ID); p.Open() && p.Tier == 0 { - t.Fatalf("the walk did not get past its tier: %s at %d, %q", p.State, p.Tier, p.Note) - } -} diff --git a/internal/artifacts/live_registry_test.go b/internal/artifacts/live_registry_test.go index c88b37f6..1bbf32f2 100644 --- a/internal/artifacts/live_registry_test.go +++ b/internal/artifacts/live_registry_test.go @@ -1,3 +1,5 @@ +//go:build integration + package artifacts import ( diff --git a/internal/artifacts/tagged_integration_test.go b/internal/artifacts/tagged_integration_test.go new file mode 100644 index 00000000..0595fa9c --- /dev/null +++ b/internal/artifacts/tagged_integration_test.go @@ -0,0 +1,72 @@ +//go:build integration + +package artifacts + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +func TestLiveADocumentPutUnderATagIsReadBackAndOnlyTheNewestIsKept(t *testing.T) { + address := os.Getenv("MESH_TEST_REGISTRY") + container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER") + if address == "" || container == "" { + t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see the collector test's comment for the registry to raise") + } + ctx := context.Background() + store := Store{Address: address} + repository := fmt.Sprintf("facts-live-%d", time.Now().UnixNano()) + + if _, _, err := store.GetTagged(ctx, repository, "latest"); !errors.Is(err, ErrNoTag) { + t.Fatalf("nothing was put and the read said %v, not that nothing is there", err) + } + first := []byte(`{"facts":1,"taken":"first"}`) + firstDigest, err := store.PutTagged(ctx, repository, "latest", "application/vnd.novox.mesh.facts.v1+json", first) + if err != nil { + t.Fatal(err) + } + second := []byte(`{"facts":1,"taken":"second"}`) + secondDigest, err := store.PutTagged(ctx, repository, "latest", "application/vnd.novox.mesh.facts.v1+json", second) + if err != nil { + t.Fatal(err) + } + got, digest, err := store.GetTagged(ctx, repository, "latest") + if err != nil { + t.Fatal(err) + } + if string(got) != string(second) || digest != secondDigest { + t.Fatalf("read back %q (%s), not the newest put %q (%s)", got, digest, second, secondDigest) + } + + // The collector, as the store's nightly step runs it — with no `--delete-untagged`: the newest kept, + // the replaced one taken because its manifest was let go of. + out, err := exec.Command("docker", "exec", container, "registry", "garbage-collect", + "/etc/docker/registry/config.yml").CombinedOutput() + if err != nil { + t.Fatalf("the collector did not run: %v\n%s", err, out) + } + if got, _, err := store.GetTagged(ctx, repository, "latest"); err != nil || string(got) != string(second) { + t.Fatalf("after collection the newest reads %q, %v", got, err) + } + // On the store's disk, not as the running server answers: it caches blob descriptors in memory. + onDisk := func(digest string) bool { + hex := strings.TrimPrefix(digest, "sha256:") + path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data" + return exec.Command("docker", "exec", container, "test", "-f", path).Run() == nil + } + if onDisk(firstDigest) { + t.Errorf("the replaced snapshot %s is still in the store after collection; nothing would ever take it", firstDigest) + } + if !onDisk(secondDigest) { + t.Errorf("the collector took the newest snapshot %s", secondDigest) + } + if !strings.HasPrefix(secondDigest, "sha256:") { + t.Errorf("the digest said %q", secondDigest) + } +} diff --git a/internal/artifacts/tagged_test.go b/internal/artifacts/tagged_test.go index 341553c5..6eba0bd2 100644 --- a/internal/artifacts/tagged_test.go +++ b/internal/artifacts/tagged_test.go @@ -2,13 +2,8 @@ package artifacts import ( "context" - "errors" - "fmt" - "os" - "os/exec" "strings" "testing" - "time" ) // The facts snapshot is put under a tag and read back by it (novox/hq to-be 45 §9). @@ -17,63 +12,6 @@ import ( // that a manifest put by tag is read by tag, that the layer comes back whole and checked, and that the // snapshot a newer one replaced is the collector's to take while the newest is kept. Raised as the // collector test above says, with MESH_TEST_REGISTRY and MESH_TEST_REGISTRY_CONTAINER. -func TestLiveADocumentPutUnderATagIsReadBackAndOnlyTheNewestIsKept(t *testing.T) { - address := os.Getenv("MESH_TEST_REGISTRY") - container := os.Getenv("MESH_TEST_REGISTRY_CONTAINER") - if address == "" || container == "" { - t.Skip("no MESH_TEST_REGISTRY / MESH_TEST_REGISTRY_CONTAINER; see the collector test's comment for the registry to raise") - } - ctx := context.Background() - store := Store{Address: address} - repository := fmt.Sprintf("facts-live-%d", time.Now().UnixNano()) - - if _, _, err := store.GetTagged(ctx, repository, "latest"); !errors.Is(err, ErrNoTag) { - t.Fatalf("nothing was put and the read said %v, not that nothing is there", err) - } - first := []byte(`{"facts":1,"taken":"first"}`) - firstDigest, err := store.PutTagged(ctx, repository, "latest", "application/vnd.novox.mesh.facts.v1+json", first) - if err != nil { - t.Fatal(err) - } - second := []byte(`{"facts":1,"taken":"second"}`) - secondDigest, err := store.PutTagged(ctx, repository, "latest", "application/vnd.novox.mesh.facts.v1+json", second) - if err != nil { - t.Fatal(err) - } - got, digest, err := store.GetTagged(ctx, repository, "latest") - if err != nil { - t.Fatal(err) - } - if string(got) != string(second) || digest != secondDigest { - t.Fatalf("read back %q (%s), not the newest put %q (%s)", got, digest, second, secondDigest) - } - - // The collector, as the store's nightly step runs it — with no `--delete-untagged`: the newest kept, - // the replaced one taken because its manifest was let go of. - out, err := exec.Command("docker", "exec", container, "registry", "garbage-collect", - "/etc/docker/registry/config.yml").CombinedOutput() - if err != nil { - t.Fatalf("the collector did not run: %v\n%s", err, out) - } - if got, _, err := store.GetTagged(ctx, repository, "latest"); err != nil || string(got) != string(second) { - t.Fatalf("after collection the newest reads %q, %v", got, err) - } - // On the store's disk, not as the running server answers: it caches blob descriptors in memory. - onDisk := func(digest string) bool { - hex := strings.TrimPrefix(digest, "sha256:") - path := "/var/lib/registry/docker/registry/v2/blobs/sha256/" + hex[:2] + "/" + hex + "/data" - return exec.Command("docker", "exec", container, "test", "-f", path).Run() == nil - } - if onDisk(firstDigest) { - t.Errorf("the replaced snapshot %s is still in the store after collection; nothing would ever take it", firstDigest) - } - if !onDisk(secondDigest) { - t.Errorf("the collector took the newest snapshot %s", secondDigest) - } - if !strings.HasPrefix(secondDigest, "sha256:") { - t.Errorf("the digest said %q", secondDigest) - } -} // A store with no address is said, not dialled. func TestADocumentWithNowhereToGoIsRefusedByName(t *testing.T) { diff --git a/internal/broker/callers_live_test.go b/internal/broker/callers_live_test.go index cacb0424..8d36d37f 100644 --- a/internal/broker/callers_live_test.go +++ b/internal/broker/callers_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/config_owner_live_test.go b/internal/broker/config_owner_live_test.go index 6d364778..b5f80422 100644 --- a/internal/broker/config_owner_live_test.go +++ b/internal/broker/config_owner_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/consumer_from_now_test.go b/internal/broker/consumer_from_now_test.go index 3500603c..fa64d770 100644 --- a/internal/broker/consumer_from_now_test.go +++ b/internal/broker/consumer_from_now_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/consumer_upgrade_test.go b/internal/broker/consumer_upgrade_test.go index 5cc0215a..1c7419e1 100644 --- a/internal/broker/consumer_upgrade_test.go +++ b/internal/broker/consumer_upgrade_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/controller_buckets_integration_test.go b/internal/broker/controller_buckets_integration_test.go new file mode 100644 index 00000000..106bb3b9 --- /dev/null +++ b/internal/broker/controller_buckets_integration_test.go @@ -0,0 +1,261 @@ +//go:build integration + +package broker + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestWhatTheControllerAskedIsBounded(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + kv, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + bucket, err := kv.KeyValue(ctx, AskedBucket) + if err != nil { + t.Fatal(err) + } + status, err := bucket.Status(ctx) + if err != nil { + t.Fatal(err) + } + info := status.(*jetstream.KeyValueBucketStatus).StreamInfo() + if status.History() != 1 || status.TTL() != AskedKeptFor || info.Config.MaxBytes <= 0 || info.Config.MaxBytes > 64<<20 { + t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes) + } +} + +func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) { + controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") { + t.Error("the controller may not write the merge window's bucket") + } + if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) { + t.Error("the merge window's bucket is not asserted as the controller's own") + } + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.KeyValue(ctx, MergeWindowBucket) + if err != nil { + t.Fatal(err) + } + w, err := kv.WatchAll(ctx) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { + t.Fatal(err) + } + for { + select { + case e := <-w.Updates(): + if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut { + return + } + case <-ctx.Done(): + t.Fatal("no watcher heard the key expire") + } + } +} + +func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) { + controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") { + t.Error("the controller may not write the rollout's silent keys' bucket") + } + if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) { + t.Error("the rollout's silent keys' bucket is not asserted as the controller's own") + } + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + kv, err := api.KeyValue(ctx, RolloutSilentBucket) + if err != nil { + t.Fatal(err) + } + w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) + if err != nil { + t.Fatal(err) + } + defer w.Stop() + if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil { + t.Fatal(err) + } + if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil { + t.Fatal(err) + } + if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { + t.Fatal(err) + } + for { + select { + case e := <-w.Updates(): + if e == nil || e.Operation() == jetstream.KeyValuePut { + continue + } + if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete { + t.Fatalf("a report's delete was heard as %v", e.Operation()) + } + if e.Key() == "plan-1.0.g14" { + if e.Operation() != jetstream.KeyValuePurge { + t.Fatalf("an expiry was heard as %v, not a purge", e.Operation()) + } + return + } + case <-ctx.Done(): + t.Fatal("no watcher heard the key expire") + } + } +} + +func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + // As the first build made it: no age. + if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1, + MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatalf("a bucket made without the age is not brought to it: %v", err) + } + kv, err := api.KeyValue(ctx, RolloutSilentBucket) + if err != nil { + t.Fatal(err) + } + status, err := kv.Status(ctx) + if err != nil { + t.Fatal(err) + } + if status.TTL() != RolloutSilentKeptFor { + t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor) + } +} + +func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := AssertMeshConsumers(js); err != nil { + t.Fatal(err) + } + for _, c := range MeshConsumers() { + if c.Stream == EventsStream && c.Name == MergesConsumerName { + if c.Resettable != "" || c.MaxDeliver != 0 || len(c.Filters) != 1 || c.Filters[0] != ControllerFollows[3] { + t.Errorf("the merges consumer is %+v; want one filter, no max-deliver, not resettable", c) + } + } + if c.Stream == EventsStream && c.Name == ControllerName { + for _, f := range c.Filters { + if f == ControllerFollows[3] { + t.Errorf("the events consumer still follows the merges, which the merges consumer takes") + } + } + } + } + if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil { + t.Fatal(err) + } + merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName) + if err != nil || merges.NumPending != 1 { + t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err) + } + if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil { + t.Fatal(err) + } + merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName) + if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 { + t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err) + } +} + +func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T) { + js, err := Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer js.Close() + if err := AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + old := &nats.ConsumerConfig{Durable: ControllerName, AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second, + MaxDeliver: 5, MaxAckPending: 1, DeliverSubject: "_DELIVER.controller.EVENTS", DeliverPolicy: nats.DeliverNewPolicy, + FilterSubjects: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName))} + if _, err := js.Context().AddConsumer(EventsStream, old); err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil { + t.Fatal(err) + } + if err := AssertMeshConsumers(js); err != nil { + t.Fatal(err) + } + merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName) + if err != nil || merges.NumPending != 1 { + t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err) + } +} diff --git a/internal/broker/controller_buckets_test.go b/internal/broker/controller_buckets_test.go index 4b337dc1..7b9d9f9b 100644 --- a/internal/broker/controller_buckets_test.go +++ b/internal/broker/controller_buckets_test.go @@ -1,16 +1,9 @@ package broker import ( - "context" - "github.com/nats-io/nats.go" "slices" "strings" "testing" - "time" - - "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/testbus" ) // **The controller may write every bucket it writes** (novox/hq to-be 45 §1, issue 269). Writing a @@ -100,34 +93,6 @@ func TestTheWatchedSignalsMayBeSaidAndHeard(t *testing.T) { // The controller's record of what it asked the operator is bounded (correctness review of 2026-10-08): one // value a key, a month's age, and a size it cannot outgrow. -func TestWhatTheControllerAskedIsBounded(t *testing.T) { - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - kv, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - bucket, err := kv.KeyValue(ctx, AskedBucket) - if err != nil { - t.Fatal(err) - } - status, err := bucket.Status(ctx) - if err != nil { - t.Fatal(err) - } - info := status.(*jetstream.KeyValueBucketStatus).StreamInfo() - if status.History() != 1 || status.TTL() != AskedKeptFor || info.Config.MaxBytes <= 0 || info.Config.MaxBytes > 64<<20 { - t.Errorf("history %d, age %s, bytes %d", status.History(), status.TTL(), info.Config.MaxBytes) - } -} // **A listed mirror is no undeclared bucket** (ADR 0356 §3, ADR 0201 §16): the controller's list of its mirrors // stands for their declaration, so the report of state nothing declares never names one. @@ -273,232 +238,19 @@ func TestTheControllerMayAskMeshConfigInItsOwnName(t *testing.T) { // **The merge window is a bucket whose keys expire** (novox/hq ADR 0407 decision 1, issue 973): a key made with a // time to live goes when it is up, and a watcher on the bucket hears it go. The controller may write it, and it is // no undeclared state. -func TestTheMergeWindowsKeysExpireAndAWatcherHearsIt(t *testing.T) { - controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(controller.Publish, "$KV."+MergeWindowBucket+".>") { - t.Error("the controller may not write the merge window's bucket") - } - if !IsControllerBucket(MergeWindowBucket) || !slices.Contains(ControllerBuckets(), MergeWindowBucket) { - t.Error("the merge window's bucket is not asserted as the controller's own") - } - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - kv, err := api.KeyValue(ctx, MergeWindowBucket) - if err != nil { - t.Fatal(err) - } - w, err := kv.WatchAll(ctx) - if err != nil { - t.Fatal(err) - } - defer w.Stop() - if _, err := kv.Create(ctx, "plan-1.quiet", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { - t.Fatal(err) - } - for { - select { - case e := <-w.Updates(): - if e != nil && e.Key() == "plan-1.quiet" && e.Operation() != jetstream.KeyValuePut { - return - } - case <-ctx.Done(): - t.Fatal("no watcher heard the key expire") - } - } -} // **A rollout's wait for a machine is a bucket whose keys expire** (novox/hq ADR 0406 §5, issue 974): a key made with a // time to live goes when it is up, and a watcher hears it go as a purge; a key a report deleted is a delete. The // controller may write it, and it is no undeclared state. -func TestTheRolloutsSilentKeysExpireAsAPurgeAndAReportsDeleteIsNot(t *testing.T) { - controller, err := PermissionsFor(Principal{Kind: KindController, PasswordHash: "x"}) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(controller.Publish, "$KV."+RolloutSilentBucket+".>") { - t.Error("the controller may not write the rollout's silent keys' bucket") - } - if !IsControllerBucket(RolloutSilentBucket) || !slices.Contains(ControllerBuckets(), RolloutSilentBucket) { - t.Error("the rollout's silent keys' bucket is not asserted as the controller's own") - } - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - kv, err := api.KeyValue(ctx, RolloutSilentBucket) - if err != nil { - t.Fatal(err) - } - w, err := kv.WatchAll(ctx, jetstream.UpdatesOnly()) - if err != nil { - t.Fatal(err) - } - defer w.Stop() - if _, err := kv.Create(ctx, "plan-1.0.ace", []byte("x"), jetstream.KeyTTL(time.Hour)); err != nil { - t.Fatal(err) - } - if err := kv.Delete(ctx, "plan-1.0.ace"); err != nil { - t.Fatal(err) - } - if _, err := kv.Create(ctx, "plan-1.0.g14", []byte("x"), jetstream.KeyTTL(time.Second)); err != nil { - t.Fatal(err) - } - for { - select { - case e := <-w.Updates(): - if e == nil || e.Operation() == jetstream.KeyValuePut { - continue - } - if e.Key() == "plan-1.0.ace" && e.Operation() != jetstream.KeyValueDelete { - t.Fatalf("a report's delete was heard as %v", e.Operation()) - } - if e.Key() == "plan-1.0.g14" { - if e.Operation() != jetstream.KeyValuePurge { - t.Fatalf("an expiry was heard as %v, not a purge", e.Operation()) - } - return - } - case <-ctx.Done(): - t.Fatal("no watcher heard the key expire") - } - } -} // **The silent keys' bucket ages**, or its delete markers would fill it (issue 974, review 1): every report deletes a key, // the delete leaves a marker, and with one value per key and no age the markers stay for ever. A bucket made before the // age was set is brought to it by the next start, which the server accepts as an update of an existing bucket. -func TestTheSilentKeysBucketAgesAndAnOlderOneIsBroughtToIt(t *testing.T) { - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - // As the first build made it: no age. - if _, err := api.CreateKeyValue(ctx, jetstream.KeyValueConfig{Bucket: RolloutSilentBucket, History: 1, - MaxValueSize: 4 << 10, MaxBytes: 8 << 20, Storage: jetstream.FileStorage, LimitMarkerTTL: RolloutSilentMarkerTTL}); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatalf("a bucket made without the age is not brought to it: %v", err) - } - kv, err := api.KeyValue(ctx, RolloutSilentBucket) - if err != nil { - t.Fatal(err) - } - status, err := kv.Status(ctx) - if err != nil { - t.Fatal(err) - } - if status.TTL() != RolloutSilentKeptFor { - t.Errorf("the bucket ages at %v; want %v", status.TTL(), RolloutSilentKeptFor) - } -} // **The merges have a consumer of their own, which a reset of the events consumer leaves alone** (issue 973, novox/hq // ADR 0407 decision 3): healer H4 re-makes the events consumer from now, dropping what it holds pending; a merge // waiting in the merges consumer is not that consumer's, and nothing would catch it up. -func TestAResetOfTheEventsConsumerLeavesTheMergesAndTheirConsumerAlone(t *testing.T) { - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := AssertMeshConsumers(js); err != nil { - t.Fatal(err) - } - for _, c := range MeshConsumers() { - if c.Stream == EventsStream && c.Name == MergesConsumerName { - if c.Resettable != "" || c.MaxDeliver != 0 || len(c.Filters) != 1 || c.Filters[0] != ControllerFollows[3] { - t.Errorf("the merges consumer is %+v; want one filter, no max-deliver, not resettable", c) - } - } - if c.Stream == EventsStream && c.Name == ControllerName { - for _, f := range c.Filters { - if f == ControllerFollows[3] { - t.Errorf("the events consumer still follows the merges, which the merges consumer takes") - } - } - } - } - if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ControllerFollows[0], []byte(`{}`)); err != nil { - t.Fatal(err) - } - merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName) - if err != nil || merges.NumPending != 1 { - t.Fatalf("the merges consumer holds %v (%v); want the merge", merges, err) - } - if _, _, err := js.ResetConsumer(EventsStream, ControllerName); err != nil { - t.Fatal(err) - } - merges, err = js.Context().ConsumerInfo(EventsStream, MergesConsumerName) - if err != nil || merges.NumPending != 1 || merges.Delivered.Stream != 0 { - t.Fatalf("after the reset the merges consumer holds %+v (%v); want its merge left alone", merges, err) - } -} // **Delivering this change loses no merge** (review of issue 973): a merge published under the old filter of the // `controller` consumer and never acknowledged by it is delivered by the merges consumer once the assertion has // made it, because it starts where `controller` has acknowledged up to, not from now. -func TestAMergeTheOldConsumerHadNotAcknowledgedIsTheMergesConsumers(t *testing.T) { - js, err := Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer js.Close() - if err := AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - old := &nats.ConsumerConfig{Durable: ControllerName, AckPolicy: nats.AckExplicitPolicy, AckWait: 30 * time.Second, - MaxDeliver: 5, MaxAckPending: 1, DeliverSubject: "_DELIVER.controller.EVENTS", DeliverPolicy: nats.DeliverNewPolicy, - FilterSubjects: append(append([]string(nil), ControllerFollows...), AgainFilter(ControllerName))} - if _, err := js.Context().AddConsumer(EventsStream, old); err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ControllerFollows[3], []byte(`{"commit":"c1"}`)); err != nil { - t.Fatal(err) - } - if err := AssertMeshConsumers(js); err != nil { - t.Fatal(err) - } - merges, err := js.Context().ConsumerInfo(EventsStream, MergesConsumerName) - if err != nil || merges.NumPending != 1 { - t.Fatalf("the merges consumer holds %v (%v); want the merge the old consumer never acknowledged", merges, err) - } -} diff --git a/internal/broker/jetstream_test.go b/internal/broker/jetstream_test.go index 5124696d..4cf84a33 100644 --- a/internal/broker/jetstream_test.go +++ b/internal/broker/jetstream_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/log_integration_test.go b/internal/broker/log_integration_test.go new file mode 100644 index 00000000..69c62f0b --- /dev/null +++ b/internal/broker/log_integration_test.go @@ -0,0 +1,69 @@ +//go:build integration + +package broker + +import ( + "context" + "slices" + "testing" + "time" + + "github.com/nats-io/nats.go/jetstream" +) + +func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) { + bus := aLiveBus(t) + l := Log{Module: "logtest", Name: "changes", MaxMiB: 1} + if _, err := RaiseLogs(bus, []Log{l}); err != nil { + t.Fatalf("a real server refused a module's log: %v", err) + } + js, err := jetstream.New(bus.Conn()) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`)) + if err != nil { + t.Fatalf("an append to the log was refused: %v", err) + } + stream, err := js.Stream(ctx, l.Stream()) + if err != nil { + t.Fatal(err) + } + have := stream.CachedInfo().Config + want := l.Config() + if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 || + have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize || + have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge || + have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) { + t.Fatalf("the server holds the log as %+v", have) + } + if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil { + t.Fatal("an entry was deleted from a log") + } + l.MaxMiB = 2 + if _, err := RaiseLogs(bus, []Log{l}); err != nil { + t.Fatalf("asserting the log again failed, so a restart would: %v", err) + } + info, err := stream.Info(ctx) + if err != nil { + t.Fatal(err) + } + if info.Config.MaxBytes != 2*1024*1024 { + t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes) + } + if info.State.Msgs < 1 { + t.Fatal("asserting the log again lost what it held") + } + undeclared, err := RaiseLogs(bus, nil) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(undeclared, l.Stream()) { + t.Fatalf("a log nothing declares was not reported: %v", undeclared) + } + if _, err := js.Stream(ctx, l.Stream()); err != nil { + t.Fatalf("a log nothing declares is gone: %v", err) + } +} diff --git a/internal/broker/log_test.go b/internal/broker/log_test.go index 95043e1c..b1988cdd 100644 --- a/internal/broker/log_test.go +++ b/internal/broker/log_test.go @@ -1,7 +1,6 @@ package broker import ( - "context" "encoding/json" "go/ast" "go/parser" @@ -11,7 +10,6 @@ import ( "slices" "strings" "testing" - "time" "github.com/nats-io/nats.go/jetstream" ) @@ -240,59 +238,3 @@ func stringsIn(f *ast.File) []string { // Against a real server: a log is created as its configuration says, asserting it again keeps what // it holds, a changed cap is brought to match in place, an entry cannot be deleted from it, and one // nothing declares any more is reported and stays. -func TestALogIsAssertedInPlaceAndNeverRemoved(t *testing.T) { - bus := aLiveBus(t) - l := Log{Module: "logtest", Name: "changes", MaxMiB: 1} - if _, err := RaiseLogs(bus, []Log{l}); err != nil { - t.Fatalf("a real server refused a module's log: %v", err) - } - js, err := jetstream.New(bus.Conn()) - if err != nil { - t.Fatal(err) - } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - ack, err := js.Publish(ctx, l.Subject()+".7", []byte(`{"op":"opened"}`)) - if err != nil { - t.Fatalf("an append to the log was refused: %v", err) - } - stream, err := js.Stream(ctx, l.Stream()) - if err != nil { - t.Fatal(err) - } - have := stream.CachedInfo().Config - want := l.Config() - if have.Storage != want.Storage || have.Retention != want.Retention || have.MaxAge != 0 || - have.MaxMsgsPerSubject != -1 || have.MaxBytes != want.MaxBytes || have.MaxMsgSize != want.MaxMsgSize || - have.Discard != jetstream.DiscardNew || !have.AllowDirect || !have.DenyDelete || !have.DenyPurge || - have.Replicas != 1 || !slices.Equal(have.Subjects, want.Subjects) { - t.Fatalf("the server holds the log as %+v", have) - } - if err := stream.DeleteMsg(ctx, ack.Sequence); err == nil { - t.Fatal("an entry was deleted from a log") - } - l.MaxMiB = 2 - if _, err := RaiseLogs(bus, []Log{l}); err != nil { - t.Fatalf("asserting the log again failed, so a restart would: %v", err) - } - info, err := stream.Info(ctx) - if err != nil { - t.Fatal(err) - } - if info.Config.MaxBytes != 2*1024*1024 { - t.Fatalf("the changed cap was not brought to match: %d", info.Config.MaxBytes) - } - if info.State.Msgs < 1 { - t.Fatal("asserting the log again lost what it held") - } - undeclared, err := RaiseLogs(bus, nil) - if err != nil { - t.Fatal(err) - } - if !slices.Contains(undeclared, l.Stream()) { - t.Fatalf("a log nothing declares was not reported: %v", undeclared) - } - if _, err := js.Stream(ctx, l.Stream()); err != nil { - t.Fatalf("a log nothing declares is gone: %v", err) - } -} diff --git a/internal/broker/modstreams_live_test.go b/internal/broker/modstreams_live_test.go index 68f42df6..4532d913 100644 --- a/internal/broker/modstreams_live_test.go +++ b/internal/broker/modstreams_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/msgttl_live_test.go b/internal/broker/msgttl_live_test.go index 02589420..777d522d 100644 --- a/internal/broker/msgttl_live_test.go +++ b/internal/broker/msgttl_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/prov_applied_live_test.go b/internal/broker/prov_applied_live_test.go index f7c4bd6f..2b8069f7 100644 --- a/internal/broker/prov_applied_live_test.go +++ b/internal/broker/prov_applied_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/provisioner_live_test.go b/internal/broker/provisioner_live_test.go index c5fe6892..9c48880f 100644 --- a/internal/broker/provisioner_live_test.go +++ b/internal/broker/provisioner_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/provisioner_server_live_test.go b/internal/broker/provisioner_server_live_test.go index 95847566..554b1bb9 100644 --- a/internal/broker/provisioner_server_live_test.go +++ b/internal/broker/provisioner_server_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/raise_live_test.go b/internal/broker/raise_live_test.go index 1d550c70..fee13277 100644 --- a/internal/broker/raise_live_test.go +++ b/internal/broker/raise_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/snapshot_live_test.go b/internal/broker/snapshot_live_test.go index af9d33c5..528ee2d1 100644 --- a/internal/broker/snapshot_live_test.go +++ b/internal/broker/snapshot_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/broker/state_integration_test.go b/internal/broker/state_integration_test.go new file mode 100644 index 00000000..fa92d0cb --- /dev/null +++ b/internal/broker/state_integration_test.go @@ -0,0 +1,55 @@ +//go:build integration + +package broker + +import ( + "testing" + + "github.com/nats-io/nats.go" +) + +func TestABucketIsAssertedInPlace(t *testing.T) { + js := aLiveBus(t) + b := Bucket{Module: "statetest", Name: "servers"} + if _, err := RaiseBuckets(js, []Bucket{b}); err != nil { + t.Fatalf("a real server refused a module's bucket: %v", err) + } + kv, err := js.Context().KeyValue(b.Bucket()) + if err != nil { + t.Fatal(err) + } + if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil { + t.Fatal(err) + } + b.History = 3 + if _, err := RaiseBuckets(js, []Bucket{b}); err != nil { + t.Fatalf("asserting the bucket again failed, so a restart would: %v", err) + } + got, err := kv.Get("all.one") + if err != nil || string(got.Value()) != `{"kept":true}` { + t.Fatalf("asserting again lost what the bucket held: %v %v", got, err) + } + status, err := kv.Status() + if err != nil { + t.Fatal(err) + } + if status.History() != 3 { + t.Fatalf("history is %d, the owner declared 3", status.History()) + } + if s, ok := status.(*nats.KeyValueBucketStatus); ok { + if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes { + t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes) + } + } +} + +func TestABucketIsAssertedOverAHeldConnection(t *testing.T) { + js := aLiveBus(t) + held := OnConn(js.Conn()) + if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil { + t.Fatalf("asserting over a held connection failed: %v", err) + } + if _, err := js.Context().KeyValue("statetest_held"); err != nil { + t.Fatalf("the bucket is not there: %v", err) + } +} diff --git a/internal/broker/state_test.go b/internal/broker/state_test.go index 402cd317..8d4d7aeb 100644 --- a/internal/broker/state_test.go +++ b/internal/broker/state_test.go @@ -4,8 +4,6 @@ import ( "slices" "strings" "testing" - - "github.com/nats-io/nats.go" ) // The grants measured against a running server (novox/hq research 024): an owner reads and writes @@ -131,54 +129,10 @@ func TestRaisingStateReportsWhatNothingDeclares(t *testing.T) { // Against a real server: a bucket is created with the owner's options and the mesh's caps, // asserting it again changes nothing and keeps what it holds, and a changed option is brought to // match in place. -func TestABucketIsAssertedInPlace(t *testing.T) { - js := aLiveBus(t) - b := Bucket{Module: "statetest", Name: "servers"} - if _, err := RaiseBuckets(js, []Bucket{b}); err != nil { - t.Fatalf("a real server refused a module's bucket: %v", err) - } - kv, err := js.Context().KeyValue(b.Bucket()) - if err != nil { - t.Fatal(err) - } - if _, err := kv.Put("all.one", []byte(`{"kept":true}`)); err != nil { - t.Fatal(err) - } - b.History = 3 - if _, err := RaiseBuckets(js, []Bucket{b}); err != nil { - t.Fatalf("asserting the bucket again failed, so a restart would: %v", err) - } - got, err := kv.Get("all.one") - if err != nil || string(got.Value()) != `{"kept":true}` { - t.Fatalf("asserting again lost what the bucket held: %v %v", got, err) - } - status, err := kv.Status() - if err != nil { - t.Fatal(err) - } - if status.History() != 3 { - t.Fatalf("history is %d, the owner declared 3", status.History()) - } - if s, ok := status.(*nats.KeyValueBucketStatus); ok { - if c := s.StreamInfo().Config; c.MaxMsgSize != StateMaxValueBytes || c.MaxBytes != StateMaxBytes { - t.Fatalf("the mesh's caps are not on the bucket: value %d, bucket %d", c.MaxMsgSize, c.MaxBytes) - } - } -} // Against a real server: the handle over a connection the control plane already holds asserts a // bucket as Dial's does — what a push uses, so a module registered since the last start has its // bucket before its membership names it. -func TestABucketIsAssertedOverAHeldConnection(t *testing.T) { - js := aLiveBus(t) - held := OnConn(js.Conn()) - if _, err := RaiseBuckets(held, []Bucket{{Module: "statetest", Name: "held"}}); err != nil { - t.Fatalf("asserting over a held connection failed: %v", err) - } - if _, err := js.Context().KeyValue("statetest_held"); err != nil { - t.Fatalf("the bucket is not there: %v", err) - } -} // **State keyed by machine reaches its machine's key alone** (novox/hq ADR 0260): the owner's instance on // a machine writes and reads that key, and a bar granted a key reads that key — a direct get of its diff --git a/internal/broker/vault_calls_live_integration_test.go b/internal/broker/vault_calls_live_integration_test.go new file mode 100644 index 00000000..65cac495 --- /dev/null +++ b/internal/broker/vault_calls_live_integration_test.go @@ -0,0 +1,130 @@ +//go:build integration + +package broker + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/nats-io/nats-server/v2/server" + "github.com/nats-io/nats.go" + "golang.org/x/crypto/bcrypt" +) + +func TestOnARealBusTheVaultsAccountIsAnsweredByTheDeskAndTheProviderAndNothingElse(t *testing.T) { + users, err := Users(vaultRecords()) + if err != nil { + t.Fatal(err) + } + hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost) + if err != nil { + t.Fatal(err) + } + for i := range users { + users[i].PasswordHash = string(hash) + } + accounts, err := ComposeAccounts(users) + if err != nil { + t.Fatal(err) + } + conf := filepath.Join(t.TempDir(), "bus.conf") + if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+ + `"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil { + t.Fatal(err) + } + opts, err := server.ProcessConfigFile(conf) + if err != nil { + t.Fatalf("the composed accounts do not parse: %v", err) + } + opts.NoLog, opts.NoSigs = true, true + s, err := server.NewServer(opts) + if err != nil { + t.Fatal(err) + } + go s.Start() + if !s.ReadyForConnections(10 * time.Second) { + t.Fatal("the bus did not come up") + } + defer s.Shutdown() + + // anchor's runtime serves the desk's prompt and plex's issuing tool, on every subject it is granted, and says + // which subject it heard. + holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("anchor.node-tools", "pw")) + if err != nil { + t.Fatal(err) + } + defer holder.Close() + heard := make(chan string, 8) + answer := func(m *nats.Msg) { + heard <- m.Subject + _ = m.Respond([]byte(`{"result":{"sealed":"c2VhbGVkLXRvLXRoZS1jYWxs"}}`)) + } + for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor.*", + "mesh.seat.node-launcher.tool.secret.anchor", "mesh.mod.plex.call.plex_issued_credential.anchor.*", + "mesh.mod.other.call.tool.anchor.*", "mesh.seat.node-launcher.call.menu.anchor.*"} { + sub, err := holder.Subscribe(subject, answer) + if err != nil { + t.Fatal(err) + } + _ = holder.Flush() + if !sub.IsValid() { + t.Fatalf("the desk's runtime may not hear %s", subject) + } + } + + refusals := make(chan error, 16) + vault, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.mesh-vault", "pw"), + nats.CustomInboxPrefix("_INBOX.novox.mesh-vault"), + nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err })) + if err != nil { + t.Fatalf("the vault's own account cannot connect: %v", err) + } + defer vault.Close() + token := CallerToken("novox.mesh-vault") + for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor." + token, + "mesh.mod.plex.call.plex_issued_credential.anchor." + token} { + if _, err := vault.Request(subject, []byte(`{"seal_to":"a2V5LW9mLXRoZS1jYWxs"}`), 3*time.Second); err != nil { + t.Fatalf("the vault's call %s was not answered: %v", subject, err) + } + if got := <-heard; got != subject { + t.Fatalf("the holder heard %s, want %s", got, subject) + } + } + for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.anchor", + "mesh.seat.node-launcher.call.secret.anchor.controller", + "mesh.seat.node-launcher.call.menu.anchor." + token, + "mesh.mod.other.call.tool.anchor." + token} { + if err := vault.Publish(subject, []byte(`{}`)); err != nil { + t.Fatal(err) + } + _ = vault.Flush() + select { + case err := <-refusals: + if !errors.Is(err, nats.ErrPermissionViolation) { + t.Errorf("%s: the server said %v, want a permissions violation", subject, err) + } + case <-time.After(3 * time.Second): + t.Errorf("the server did not refuse the vault %s", subject) + } + } + select { + case got := <-heard: + t.Errorf("a call the vault may not make reached the holder: %s", got) + case <-time.After(200 * time.Millisecond): + } + + // The controller is still answered on today's path, for the modules filled the old way (until R1). + controller, err := nats.Connect(s.ClientURL(), nats.UserInfo("controller", "pw"), + nats.CustomInboxPrefix("_INBOX.controller")) + if err != nil { + t.Fatal(err) + } + defer controller.Close() + if _, err := controller.Request("mesh.seat.node-launcher.tool.secret.anchor", []byte(`{}`), 3*time.Second); err != nil { + t.Errorf("the controller's call to the desk on today's path: %v", err) + } + <-heard +} diff --git a/internal/broker/vault_calls_live_test.go b/internal/broker/vault_calls_live_test.go index 7a769870..17a72747 100644 --- a/internal/broker/vault_calls_live_test.go +++ b/internal/broker/vault_calls_live_test.go @@ -1,16 +1,8 @@ package broker import ( - "errors" - "os" - "path/filepath" "strings" "testing" - "time" - - "github.com/nats-io/nats-server/v2/server" - "github.com/nats-io/nats.go" - "golang.org/x/crypto/bcrypt" ) // vaultRecords is a mesh of two machines: anchor, a desk whose runtime carries the desk's prompt (rofi holding @@ -111,117 +103,3 @@ func TestTheVaultAsksTheDeskAndTheIssuingToolsInItsOwnNameAlone(t *testing.T) { // **On a real server, as composed** (issue 723): the vault's account asks the desk's prompt and an issuing tool // and is answered, naming itself; the server refuses it any other call; and the controller is still answered on // today's path. That only sealed values cross is the vault's test (mesh-catalog, mesh-vault intake_test.go). -func TestOnARealBusTheVaultsAccountIsAnsweredByTheDeskAndTheProviderAndNothingElse(t *testing.T) { - users, err := Users(vaultRecords()) - if err != nil { - t.Fatal(err) - } - hash, err := bcrypt.GenerateFromPassword([]byte("pw"), bcrypt.MinCost) - if err != nil { - t.Fatal(err) - } - for i := range users { - users[i].PasswordHash = string(hash) - } - accounts, err := ComposeAccounts(users) - if err != nil { - t.Fatal(err) - } - conf := filepath.Join(t.TempDir(), "bus.conf") - if err := os.WriteFile(conf, []byte("listen: 127.0.0.1:-1\njetstream { store_dir: "+ - `"`+t.TempDir()+`"`+" }\n"+accounts), 0o600); err != nil { - t.Fatal(err) - } - opts, err := server.ProcessConfigFile(conf) - if err != nil { - t.Fatalf("the composed accounts do not parse: %v", err) - } - opts.NoLog, opts.NoSigs = true, true - s, err := server.NewServer(opts) - if err != nil { - t.Fatal(err) - } - go s.Start() - if !s.ReadyForConnections(10 * time.Second) { - t.Fatal("the bus did not come up") - } - defer s.Shutdown() - - // anchor's runtime serves the desk's prompt and plex's issuing tool, on every subject it is granted, and says - // which subject it heard. - holder, err := nats.Connect(s.ClientURL(), nats.UserInfo("anchor.node-tools", "pw")) - if err != nil { - t.Fatal(err) - } - defer holder.Close() - heard := make(chan string, 8) - answer := func(m *nats.Msg) { - heard <- m.Subject - _ = m.Respond([]byte(`{"result":{"sealed":"c2VhbGVkLXRvLXRoZS1jYWxs"}}`)) - } - for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor.*", - "mesh.seat.node-launcher.tool.secret.anchor", "mesh.mod.plex.call.plex_issued_credential.anchor.*", - "mesh.mod.other.call.tool.anchor.*", "mesh.seat.node-launcher.call.menu.anchor.*"} { - sub, err := holder.Subscribe(subject, answer) - if err != nil { - t.Fatal(err) - } - _ = holder.Flush() - if !sub.IsValid() { - t.Fatalf("the desk's runtime may not hear %s", subject) - } - } - - refusals := make(chan error, 16) - vault, err := nats.Connect(s.ClientURL(), nats.UserInfo("novox.mesh-vault", "pw"), - nats.CustomInboxPrefix("_INBOX.novox.mesh-vault"), - nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) { refusals <- err })) - if err != nil { - t.Fatalf("the vault's own account cannot connect: %v", err) - } - defer vault.Close() - token := CallerToken("novox.mesh-vault") - for _, subject := range []string{"mesh.seat.node-launcher.call.secret.anchor." + token, - "mesh.mod.plex.call.plex_issued_credential.anchor." + token} { - if _, err := vault.Request(subject, []byte(`{"seal_to":"a2V5LW9mLXRoZS1jYWxs"}`), 3*time.Second); err != nil { - t.Fatalf("the vault's call %s was not answered: %v", subject, err) - } - if got := <-heard; got != subject { - t.Fatalf("the holder heard %s, want %s", got, subject) - } - } - for _, subject := range []string{"mesh.seat.node-launcher.tool.secret.anchor", - "mesh.seat.node-launcher.call.secret.anchor.controller", - "mesh.seat.node-launcher.call.menu.anchor." + token, - "mesh.mod.other.call.tool.anchor." + token} { - if err := vault.Publish(subject, []byte(`{}`)); err != nil { - t.Fatal(err) - } - _ = vault.Flush() - select { - case err := <-refusals: - if !errors.Is(err, nats.ErrPermissionViolation) { - t.Errorf("%s: the server said %v, want a permissions violation", subject, err) - } - case <-time.After(3 * time.Second): - t.Errorf("the server did not refuse the vault %s", subject) - } - } - select { - case got := <-heard: - t.Errorf("a call the vault may not make reached the holder: %s", got) - case <-time.After(200 * time.Millisecond): - } - - // The controller is still answered on today's path, for the modules filled the old way (until R1). - controller, err := nats.Connect(s.ClientURL(), nats.UserInfo("controller", "pw"), - nats.CustomInboxPrefix("_INBOX.controller")) - if err != nil { - t.Fatal(err) - } - defer controller.Close() - if _, err := controller.Request("mesh.seat.node-launcher.tool.secret.anchor", []byte(`{}`), 3*time.Second); err != nil { - t.Errorf("the controller's call to the desk on today's path: %v", err) - } - <-heard -} diff --git a/internal/broker/worker_type_change_test.go b/internal/broker/worker_type_change_test.go index 7ddd588d..f3bd7712 100644 --- a/internal/broker/worker_type_change_test.go +++ b/internal/broker/worker_type_change_test.go @@ -1,3 +1,5 @@ +//go:build integration + package broker import ( diff --git a/internal/builder/build_source_test.go b/internal/builder/build_source_test.go index 5c1fdc48..e356ccd7 100644 --- a/internal/builder/build_source_test.go +++ b/internal/builder/build_source_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/builder_integration_test.go b/internal/builder/builder_integration_test.go new file mode 100644 index 00000000..56eac672 --- /dev/null +++ b/internal/builder/builder_integration_test.go @@ -0,0 +1,362 @@ +//go:build integration + +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/theme.conf": "dark", + }) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Commit != "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" { + t.Fatalf("the commit was not recorded: %q", got.Commit) + } + if got.Manifest.Resources[0]["image"] == nil { + t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) + } + digest, _ := got.Manifest.Resources[1]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("the archive was not pinned: %v", got.Manifest.Resources[1]) + } +} + +func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { + // Or nothing downstream can tell "this changed" from "this was built again", and every + // rebuild looks like a change to every machine holding it. + var digests []string + for i := 0; i < 2; i++ { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/a.conf": "one", "files/b.conf": "two", + }) + // A year apart, so a packer carrying timestamps cannot accidentally agree. + r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + for _, b := range got.Built { + if b.Kind == catalogue.ArtifactArchive { + digests = append(digests, b.Digest) + } + } + } + if digests[0] != digests[1] { + t.Fatalf("two builds of one commit produced %s and %s", digests[0], digests[1]) + } +} + +func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { + // Half a module in the store under a digest the mesh never records is reachable, + // unreferenced, and indistinguishable from something in use. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) + // `files` is missing, so packing the archive fails — after the image would have been pushed. + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a build with a missing input succeeded") + } + if len(r.archives) != 0 { + t.Fatalf("an archive was published by a failed build: %v", r.archives) + } +} + +func TestARepositoryWithNoManifestSaysSo(t *testing.T) { + workspace := t.TempDir() + r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a repository with nothing saying what it is was built") + } + if !strings.Contains(err.Error(), ManifestName) { + t.Fatalf("the failure does not name what is missing: %v", err) + } +} + +func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { + // Most of what a person installs is configuration. + r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ + {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if len(got.Built) != 0 { + t.Fatalf("something was built: %v", got.Built) + } + if got.Manifest.Module != "shell" || len(got.Manifest.Resources) != 1 { + t.Fatalf("got %+v", got.Manifest) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker") { + t.Fatalf("docker was run for a module that builds nothing: %q", line) + } + } +} + +func TestTheTreeIsFreshEveryTime(t *testing.T) { + // A build that reuses a working tree can succeed because of something a previous build left + // behind, and that is a build nobody can reproduce. + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/a": "b", + }) + leftover := filepath.Join(workspace, "source", "files", "from-last-time") + if err := os.MkdirAll(filepath.Dir(leftover), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { + t.Fatal(err) + } + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(leftover); err == nil { + t.Fatal("a previous build's file survived into this one") + } +} + +func TestABuildThatCannotPushFails(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/a": "b", + }) + r.failPush = true + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err == nil { + t.Fatal("a build that could publish nothing reported success") + } +} + +func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { + // A module usually runs software it did not write. Naming the upstream reference directly + // would need every machine to reach a public registry, and would pin to a tag somebody else + // can move. + const mirrors = `{"module":"postgres","version":"1", + "build":{"artifacts":[{"name":"store","kind":"upstream","from":"postgres:17-alpine"}]}, + "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` + + r, workspace := aRepository(t, mirrors, nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + // Pulled, not built. + var pulled, built bool + for _, line := range r.ran { + if strings.HasPrefix(line, "docker pull postgres:17-alpine") { + pulled = true + } + if strings.HasPrefix(line, "docker build") { + built = true + } + } + if !pulled { + t.Fatalf("the upstream image was not fetched: %v", r.ran) + } + if built { + t.Fatalf("something was built for an image that is mirrored: %v", r.ran) + } + // And the resource names what this registry serves, pinned by the digest it assigned. + image, _ := got.Manifest.Resources[0]["image"].(string) + if !strings.Contains(image, "@sha256:") { + t.Fatalf("the mirrored image is not pinned by digest: %q", image) + } + if strings.Contains(image, "17-alpine") { + t.Fatalf("the resource still names the upstream tag: %q", image) + } +} + +func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { + r := &recorded{contents: map[string]string{ + "README.md": "this repository holds several modules", + "modules/shell/" + ManifestName: withBoth, + "modules/shell/Dockerfile": "FROM scratch", + "modules/shell/files/theme.conf": "dark", + // A second module beside it, so what is built is chosen by the path rather than by + // happening to be the only manifest in the clone. + "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, + }} + got, err := Build(context.Background(), r.run, r, + "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Manifest.Module != "meshboard" { + t.Fatalf("built %q, which is not the module at the path asked for", got.Manifest.Module) + } + if got.Manifest.Resources[0]["image"] == nil { + t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) + } +} + +func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { + for _, escaping := range []string{"../../etc", "/etc"} { + r := &recorded{contents: map[string]string{ManifestName: withBoth}} + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatalf("%q was accepted as a module's path", escaping) + } + if !strings.Contains(err.Error(), "leaves the repository") && + !strings.Contains(err.Error(), "absolute path") { + t.Fatalf("the refusal of %q does not say why: %v", escaping, err) + } + } +} + +func TestAnArtifactWithItsOwnContextIsBuiltFromThere(t *testing.T) { + const withContext = `{"module":"route-proxy","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile", + "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` + r := &recorded{ + contents: map[string]string{ + ManifestName: withContext, + // The recipe lives with the packaging, not the source — read from here regardless of + // where the build context comes from. FROM scratch declares no base, so what is under + // test — where the context comes from — is not entangled with ADR 0097's own checks. + "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", + }, + secondary: map[string]map[string]string{ + // go.mod exists only in the second repository. A build context taken from the wrong + // place would never find it, which a real docker build would refuse on — the fake + // does not read files, so what is checked below is that the build was even pointed + // at the right place, not that COPY would have succeeded. + "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, + }, + } + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + + var clonedSource bool + for _, line := range r.ran { + if strings.HasPrefix(line, "git clone") && strings.Contains(line, "https://forge.invalid/source.git") { + clonedSource = true + } + } + if !clonedSource { + t.Fatalf("the artifact's own context was never cloned: %v", r.ran) + } + + buildIndex := -1 + for i, line := range r.ran { + if strings.HasPrefix(line, "docker build ") { + buildIndex = i + } + } + if buildIndex == -1 { + t.Fatal("no docker build was run") + } + build := r.ran[buildIndex] + buildDir := r.dirs[buildIndex] + + if !strings.Contains(buildDir, "context-server") { + t.Errorf("docker build ran from %q, not the artifact's own cloned context", buildDir) + } + recipe := strings.SplitN(strings.SplitN(build, "-f ", 2)[1], " ", 2)[0] + if !filepath.IsAbs(recipe) { + t.Errorf("the recipe %q is not an absolute path, so it is read relative to whatever "+ + "directory the build context moved to rather than where it actually is", recipe) + } + if !strings.HasSuffix(recipe, string(filepath.Separator)+"Dockerfile") { + t.Errorf("the recipe is not the module's own Dockerfile: %q", recipe) + } + if !strings.HasSuffix(build, " .") { + t.Errorf("the build was not given a context: %s", build) + } +} + +func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/theme.conf": "dark", + }) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", + workspace, nil, Npmrc{}, + GitCredential{URL: "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000"}, nil) + if err != nil { + t.Fatal(err) + } + clone := r.ran[0] + stored := storeNamedIn(t, clone) + for _, line := range r.ran { + if strings.Contains(line, "sw0rdfi5h") { + t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line) + } + } + // Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what + // it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential). + if withinDir(workspace, stored) { + t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace) + } + if _, err := os.Stat(stored); !os.IsNotExist(err) { + t.Fatalf("the credential store outlives the build: %v", err) + } + if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { + t.Fatal("a credential file is left in the workspace") + } +} + +func TestABuildWithNoCredentialClonesExactlyAsBefore(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{ + "Dockerfile": "FROM scratch", "files/theme.conf": "dark", + }) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", + workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(r.ran[0], "git clone --quiet ") { + t.Fatalf("a credential-less clone grew flags: %s", r.ran[0]) + } + if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { + t.Fatal("a credential file was written with no credential to put in it") + } +} + +func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) { + const withContext = `{"module":"route-proxy","version":"1", + "build":{"artifacts":[ + {"name":"server","kind":"image","from":"Dockerfile", + "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` + r := &recorded{ + contents: map[string]string{ + ManifestName: withContext, + "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", + }, + secondary: map[string]map[string]string{ + "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, + }, + } + workspace := t.TempDir() + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/catalogue.git", "", "", workspace, nil, Npmrc{}, + GitCredential{URL: "https://builder:s3cret@forge.invalid"}, nil) + if err != nil { + t.Fatal(err) + } + stored := storeNamedIn(t, r.ran[0]) + var contextClone string + for _, line := range r.ran { + if strings.Contains(line, "clone") && strings.Contains(line, "source.git") { + contextClone = line + } + } + if contextClone == "" { + t.Fatalf("the context was never cloned: %v", r.ran) + } + if !strings.Contains(contextClone, "credential.helper=store --file="+stored) { + t.Fatalf("the context clone does not name the credential store: %s", contextClone) + } +} diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 4bfe447f..1cc165ac 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -130,168 +130,6 @@ const withBoth = `{"module":"meshboard","version":"1", {"id":"svc","type":"container","name":"meshboard","artifact":"server"}, {"id":"theme","type":"archive","path":"/opt/meshboard","artifact":"look"}]}` -func TestABuildProducesAManifestThePinsAreIn(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/theme.conf": "dark", - }) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Commit != "c0ffeec0ffeec0ffeec0ffeec0ffeec0ffeec0ff" { - t.Fatalf("the commit was not recorded: %q", got.Commit) - } - if got.Manifest.Resources[0]["image"] == nil { - t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) - } - digest, _ := got.Manifest.Resources[1]["digest"].(string) - if !strings.HasPrefix(digest, "sha256:") { - t.Fatalf("the archive was not pinned: %v", got.Manifest.Resources[1]) - } -} - -func TestTwoBuildsOfOneCommitProduceOneDigest(t *testing.T) { - // Or nothing downstream can tell "this changed" from "this was built again", and every - // rebuild looks like a change to every machine holding it. - var digests []string - for i := 0; i < 2; i++ { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/a.conf": "one", "files/b.conf": "two", - }) - // A year apart, so a packer carrying timestamps cannot accidentally agree. - r.stamped = time.Date(2020+i, time.March, 3, 4, 5, 6, 0, time.UTC) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - for _, b := range got.Built { - if b.Kind == catalogue.ArtifactArchive { - digests = append(digests, b.Digest) - } - } - } - if digests[0] != digests[1] { - t.Fatalf("two builds of one commit produced %s and %s", digests[0], digests[1]) - } -} - -func TestNothingIsPublishedUntilEverythingIsBuilt(t *testing.T) { - // Half a module in the store under a digest the mesh never records is reachable, - // unreferenced, and indistinguishable from something in use. - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch"}) - // `files` is missing, so packing the archive fails — after the image would have been pushed. - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a build with a missing input succeeded") - } - if len(r.archives) != 0 { - t.Fatalf("an archive was published by a failed build: %v", r.archives) - } -} - -func TestARepositoryWithNoManifestSaysSo(t *testing.T) { - workspace := t.TempDir() - r := &recorded{contents: map[string]string{"README.md": "nothing to see"}} - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a repository with nothing saying what it is was built") - } - if !strings.Contains(err.Error(), ManifestName) { - t.Fatalf("the failure does not name what is missing: %v", err) - } -} - -func TestAModuleThatBuildsNothingStillProducesAManifest(t *testing.T) { - // Most of what a person installs is configuration. - r, workspace := aRepository(t, `{"module":"shell","version":"1","resources":[ - {"id":"rc","type":"file","path":"/etc/zsh/zshrc","content":"setopt"}]}`, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/shell.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if len(got.Built) != 0 { - t.Fatalf("something was built: %v", got.Built) - } - if got.Manifest.Module != "shell" || len(got.Manifest.Resources) != 1 { - t.Fatalf("got %+v", got.Manifest) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker") { - t.Fatalf("docker was run for a module that builds nothing: %q", line) - } - } -} - -func TestTheTreeIsFreshEveryTime(t *testing.T) { - // A build that reuses a working tree can succeed because of something a previous build left - // behind, and that is a build nobody can reproduce. - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/a": "b", - }) - leftover := filepath.Join(workspace, "source", "files", "from-last-time") - if err := os.MkdirAll(filepath.Dir(leftover), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(leftover, []byte("stale"), 0o644); err != nil { - t.Fatal(err) - } - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - if _, err := os.Stat(leftover); err == nil { - t.Fatal("a previous build's file survived into this one") - } -} - -func TestABuildThatCannotPushFails(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/a": "b", - }) - r.failPush = true - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/x.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err == nil { - t.Fatal("a build that could publish nothing reported success") - } -} - -func TestAnUpstreamImageIsMirroredRatherThanBuilt(t *testing.T) { - // A module usually runs software it did not write. Naming the upstream reference directly - // would need every machine to reach a public registry, and would pin to a tag somebody else - // can move. - const mirrors = `{"module":"postgres","version":"1", - "build":{"artifacts":[{"name":"store","kind":"upstream","from":"postgres:17-alpine"}]}, - "resources":[{"id":"db","type":"container","name":"mesh-postgres","artifact":"store"}]}` - - r, workspace := aRepository(t, mirrors, nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/postgres.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - // Pulled, not built. - var pulled, built bool - for _, line := range r.ran { - if strings.HasPrefix(line, "docker pull postgres:17-alpine") { - pulled = true - } - if strings.HasPrefix(line, "docker build") { - built = true - } - } - if !pulled { - t.Fatalf("the upstream image was not fetched: %v", r.ran) - } - if built { - t.Fatalf("something was built for an image that is mirrored: %v", r.ran) - } - // And the resource names what this registry serves, pinned by the digest it assigned. - image, _ := got.Manifest.Resources[0]["image"].(string) - if !strings.Contains(image, "@sha256:") { - t.Fatalf("the mirrored image is not pinned by digest: %q", image) - } - if strings.Contains(image, "17-alpine") { - t.Fatalf("the resource still names the upstream tag: %q", image) - } -} - func TestAnUpstreamImageWithNoTagIsRefused(t *testing.T) { // What gets mirrored would be whatever `latest` means today, and a module pinned to that is // not pinned. @@ -317,148 +155,19 @@ func TestAnUpstreamReferenceIsNotAPathInTheRepository(t *testing.T) { // **A module is a repository and a path within it** (novox/hq ADR 0069). The catalogue holds its // modules one to a directory and the system this replaces has always built one that way, so a // builder that could only read a repository's root could build none of what exists. -func TestAModuleIsBuiltFromItsPathWithinTheRepository(t *testing.T) { - r := &recorded{contents: map[string]string{ - "README.md": "this repository holds several modules", - "modules/shell/" + ManifestName: withBoth, - "modules/shell/Dockerfile": "FROM scratch", - "modules/shell/files/theme.conf": "dark", - // A second module beside it, so what is built is chosen by the path rather than by - // happening to be the only manifest in the clone. - "modules/other/" + ManifestName: `{"module":"other","version":"1"}`, - }} - got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "modules/shell", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Manifest.Module != "meshboard" { - t.Fatalf("built %q, which is not the module at the path asked for", got.Manifest.Module) - } - if got.Manifest.Resources[0]["image"] == nil { - t.Fatalf("the image was not pinned: %v", got.Manifest.Resources[0]) - } -} // A build reads only its own tree. A path climbing out of the clone would otherwise let a build // read — and an archive artifact publish — whatever the build machine happens to hold, which is // the one thing a machine that builds other people's repositories must not do. -func TestAPathThatLeavesTheRepositoryIsRefused(t *testing.T) { - for _, escaping := range []string{"../../etc", "/etc"} { - r := &recorded{contents: map[string]string{ManifestName: withBoth}} - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/x.git", escaping, "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatalf("%q was accepted as a module's path", escaping) - } - if !strings.Contains(err.Error(), "leaves the repository") && - !strings.Contains(err.Error(), "absolute path") { - t.Fatalf("the refusal of %q does not say why: %v", escaping, err) - } - } -} // **Packaging and source are allowed to live apart** — a module that ships only the recipe for // source that lives in a second repository (the reference route-proxy, packaged in the catalogue // but built from mesh-controller's own repository) names where that source actually is, rather // than vendoring a second copy the two could drift from. -func TestAnArtifactWithItsOwnContextIsBuiltFromThere(t *testing.T) { - const withContext = `{"module":"route-proxy","version":"1", - "build":{"artifacts":[ - {"name":"server","kind":"image","from":"Dockerfile", - "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` - r := &recorded{ - contents: map[string]string{ - ManifestName: withContext, - // The recipe lives with the packaging, not the source — read from here regardless of - // where the build context comes from. FROM scratch declares no base, so what is under - // test — where the context comes from — is not entangled with ADR 0097's own checks. - "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", - }, - secondary: map[string]map[string]string{ - // go.mod exists only in the second repository. A build context taken from the wrong - // place would never find it, which a real docker build would refuse on — the fake - // does not read files, so what is checked below is that the build was even pointed - // at the right place, not that COPY would have succeeded. - "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, - }, - } - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "", "", t.TempDir(), nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - - var clonedSource bool - for _, line := range r.ran { - if strings.HasPrefix(line, "git clone") && strings.Contains(line, "https://forge.invalid/source.git") { - clonedSource = true - } - } - if !clonedSource { - t.Fatalf("the artifact's own context was never cloned: %v", r.ran) - } - - buildIndex := -1 - for i, line := range r.ran { - if strings.HasPrefix(line, "docker build ") { - buildIndex = i - } - } - if buildIndex == -1 { - t.Fatal("no docker build was run") - } - build := r.ran[buildIndex] - buildDir := r.dirs[buildIndex] - - if !strings.Contains(buildDir, "context-server") { - t.Errorf("docker build ran from %q, not the artifact's own cloned context", buildDir) - } - recipe := strings.SplitN(strings.SplitN(build, "-f ", 2)[1], " ", 2)[0] - if !filepath.IsAbs(recipe) { - t.Errorf("the recipe %q is not an absolute path, so it is read relative to whatever "+ - "directory the build context moved to rather than where it actually is", recipe) - } - if !strings.HasSuffix(recipe, string(filepath.Separator)+"Dockerfile") { - t.Errorf("the recipe is not the module's own Dockerfile: %q", recipe) - } - if !strings.HasSuffix(build, " .") { - t.Errorf("the build was not given a context: %s", build) - } -} // The forge credential is offered through git's own credential store — a file, never argv — and // git decides when it applies. What is checked: the clone names the store, the secret never // appears in a command line, and the file holds exactly the URL at 0600. -func TestABuildOffersTheForgesCredentialThroughGitsOwnStore(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/theme.conf": "dark", - }) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", - workspace, nil, Npmrc{}, - GitCredential{URL: "http://mesh_novox_builder:sw0rdfi5h@forge.invalid:20000"}, nil) - if err != nil { - t.Fatal(err) - } - clone := r.ran[0] - stored := storeNamedIn(t, clone) - for _, line := range r.ran { - if strings.Contains(line, "sw0rdfi5h") { - t.Fatalf("the secret is in a command line, readable by anything that can list processes: %s", line) - } - } - // Outside the workspace a check's container mounts, and gone once the build is (novox/hq issue 462); what - // it held while git read it is checked with the check's clones (TestACheckContainerSeesNoForgeCredential). - if withinDir(workspace, stored) { - t.Fatalf("the credential store %s is inside the workspace %s", stored, workspace) - } - if _, err := os.Stat(stored); !os.IsNotExist(err) { - t.Fatalf("the credential store outlives the build: %v", err) - } - if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { - t.Fatal("a credential file is left in the workspace") - } -} // storeNamedIn is the credential store a git command line offers. func storeNamedIn(t *testing.T, line string) string { @@ -472,57 +181,6 @@ func storeNamedIn(t *testing.T, line string) string { // Without a credential, a clone is exactly the invocation it always was, and no credential file // appears — the builder a mesh of public repositories runs is unchanged. -func TestABuildWithNoCredentialClonesExactlyAsBefore(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{ - "Dockerfile": "FROM scratch", "files/theme.conf": "dark", - }) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", - workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !strings.HasPrefix(r.ran[0], "git clone --quiet ") { - t.Fatalf("a credential-less clone grew flags: %s", r.ran[0]) - } - if _, err := os.Stat(filepath.Join(workspace, "git-credentials")); !os.IsNotExist(err) { - t.Fatal("a credential file was written with no credential to put in it") - } -} // An artifact's own context is cloned with the same offer: a private module whose context is a // second private repository on the same forge builds, and the secret still never reaches argv. -func TestAContextCloneCarriesTheSameCredentialStore(t *testing.T) { - const withContext = `{"module":"route-proxy","version":"1", - "build":{"artifacts":[ - {"name":"server","kind":"image","from":"Dockerfile", - "context":{"repository":"https://forge.invalid/source.git","ref":"main"}}]}}` - r := &recorded{ - contents: map[string]string{ - ManifestName: withContext, - "Dockerfile": "FROM scratch\nCOPY go.mod ./\n", - }, - secondary: map[string]map[string]string{ - "https://forge.invalid/source.git": {"go.mod": "module route-proxy\n"}, - }, - } - workspace := t.TempDir() - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/catalogue.git", "", "", workspace, nil, Npmrc{}, - GitCredential{URL: "https://builder:s3cret@forge.invalid"}, nil) - if err != nil { - t.Fatal(err) - } - stored := storeNamedIn(t, r.ran[0]) - var contextClone string - for _, line := range r.ran { - if strings.Contains(line, "clone") && strings.Contains(line, "source.git") { - contextClone = line - } - } - if contextClone == "" { - t.Fatalf("the context was never cloned: %v", r.ran) - } - if !strings.Contains(contextClone, "credential.helper=store --file="+stored) { - t.Fatalf("the context clone does not name the credential store: %s", contextClone) - } -} diff --git a/internal/builder/bundle_integration_test.go b/internal/builder/bundle_integration_test.go new file mode 100644 index 00000000..80a69e81 --- /dev/null +++ b/internal/builder/bundle_integration_test.go @@ -0,0 +1,166 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) + } + + // Compiled in the toolchain the mesh chose, not in one the module named. + var compiled string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + compiled = line + } + } + if compiled == "" { + t.Fatalf("nothing was compiled:\n%s", strings.Join(r.ran, "\n")) + } + if !strings.Contains(compiled, "mesh-tools/build@sha256:") { + t.Fatalf("the compile did not run in the mesh's own toolchain: %s", compiled) + } + if strings.Contains(strings.Join(r.ran, "\n"), "docker build") { + t.Fatalf("a bundle invoked a Dockerfile build, which is the thing it exists to avoid:\n%s", + strings.Join(r.ran, "\n")) + } + + // And pinned by a digest of what came out, like any other artifact. + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") { + t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) + } + + // **One file per entrypoint and launcher, in the toolchain's bundler** (novox/hq ADR 0193). A + // second run in the same toolchain image bundles each into the artifact's bundled output, the SDK + // inlined, refusing by name in an image that predates the bundler; and the toolchain's + // node_modules is no longer copied into a bundle that keeps nothing external. + var bundling []string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") && strings.Contains(line, "esbuild") { + bundling = append(bundling, line) + } + } + if len(bundling) != 2 { + t.Fatalf("want one bundling run for the entrypoints and one for the launchers:\n%s", strings.Join(r.ran, "\n")) + } + for _, want := range []string{"mesh-tools/build@sha256:", "predates one-file bundles", "--bundle", "--format=esm", + "--platform=node", "--outdir=" + Out("code") + ".bundled", Out("code") + "/index.js"} { + if !strings.Contains(bundling[0], want) { + t.Errorf("the entrypoints' bundling lacks %q: %s", want, bundling[0]) + } + } + if !strings.Contains(bundling[1], Out("code")+"/index.serve.mjs") || !strings.Contains(bundling[1], "--out-extension:.js=.mjs") { + t.Errorf("the launcher is not bundled under its own name: %s", bundling[1]) + } + if strings.Contains(strings.Join(r.ran, "\n"), "/app/runtime") { + t.Errorf("the toolchain's node_modules was copied into a bundle that keeps nothing external:\n%s", strings.Join(r.ran, "\n")) + } + if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "esbuild") { + t.Fatal("the bundler ran before the compile wrote its output") + } +} + +func TestABundleKeepingAPackageExternalCarriesTheToolchainsModules(t *testing.T) { + manifest := strings.Replace(aBundle, `"entrypoints":["index.js"]`, `"entrypoints":["index.js"],"external":["sharp"]`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + if _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + all := strings.Join(r.ran, "\n") + if !strings.Contains(all, "--external:sharp") || !strings.Contains(all, "/app/runtime") { + t.Errorf("an external package was not kept as an import with the toolchain's modules beside it:\n%s", all) + } +} + +func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { + r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a bundle was built with no toolchain to compile it in") + } + if !strings.Contains(err.Error(), "mesh-tools") { + t.Fatalf("the refusal does not name what has to be built first: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + t.Fatalf("a compile was attempted before the refusal: %s", line) + } + } +} + +func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { + manifest := strings.Replace(aBundle, `"language":"typescript"`, `"language":"cobol"`, 1) + r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "x"}) + + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, + map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a language nothing can compile was accepted") + } + if !strings.Contains(err.Error(), "typescript") { + t.Fatalf("the refusal does not say what would have worked: %v", err) + } +} + +func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { + const two = `{"module":"greeter","version":"1", + "build":{"artifacts":[ + {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]}, + {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, + "resources":[ + {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"}, + {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}` + + r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"}) + held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} + + got, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a module with two bundles did not build: %v", err) + } + + // Compiled into two different places. Only the compile lines: the copy of each bundle's + // dependencies names the same directory again, deliberately. + var outputs []string + for _, line := range r.ran { + if !strings.Contains(line, "--outDir") { + continue + } + for _, part := range strings.Fields(line) { + if strings.HasPrefix(part, ".mesh-build/") { + outputs = append(outputs, part) + } + } + } + if len(outputs) != 2 || outputs[0] == outputs[1] { + t.Fatalf("two bundles did not get their own output directories: %v", outputs) + } + + // And published as two artifacts, each with its own digest. + if len(r.archives) != 2 { + t.Fatalf("expected two archives published, got %v", r.archives) + } + first, _ := got.Manifest.Resources[0]["digest"].(string) + second, _ := got.Manifest.Resources[1]["digest"].(string) + if first == "" || second == "" { + t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources) + } +} diff --git a/internal/builder/bundle_test.go b/internal/builder/bundle_test.go index a29225e6..ccabe300 100644 --- a/internal/builder/bundle_test.go +++ b/internal/builder/bundle_test.go @@ -4,7 +4,6 @@ import ( "context" "os" "path/filepath" - "strings" "testing" ) @@ -49,85 +48,9 @@ func (c compiling) run(ctx context.Context, dir, name string, args ...string) (s // **A module says what it is written in, and needs no Dockerfile.** This is the whole point of the // bundle recipe: the same module previously needed a hand-written recipe repeating an incantation // that is easy to get wrong in ways that fail somewhere else. -func TestABundleIsCompiledAndPackedWithNoDockerfile(t *testing.T) { - r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) - held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} - - got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a module with a language and no Dockerfile did not build: %v", err) - } - - // Compiled in the toolchain the mesh chose, not in one the module named. - var compiled string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") { - compiled = line - } - } - if compiled == "" { - t.Fatalf("nothing was compiled:\n%s", strings.Join(r.ran, "\n")) - } - if !strings.Contains(compiled, "mesh-tools/build@sha256:") { - t.Fatalf("the compile did not run in the mesh's own toolchain: %s", compiled) - } - if strings.Contains(strings.Join(r.ran, "\n"), "docker build") { - t.Fatalf("a bundle invoked a Dockerfile build, which is the thing it exists to avoid:\n%s", - strings.Join(r.ran, "\n")) - } - - // And pinned by a digest of what came out, like any other artifact. - digest, _ := got.Manifest.Resources[0]["digest"].(string) - if !strings.HasPrefix(digest, "sha256:") { - t.Fatalf("the bundle was not pinned: %v", got.Manifest.Resources[0]) - } - - // **One file per entrypoint and launcher, in the toolchain's bundler** (novox/hq ADR 0193). A - // second run in the same toolchain image bundles each into the artifact's bundled output, the SDK - // inlined, refusing by name in an image that predates the bundler; and the toolchain's - // node_modules is no longer copied into a bundle that keeps nothing external. - var bundling []string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") && strings.Contains(line, "esbuild") { - bundling = append(bundling, line) - } - } - if len(bundling) != 2 { - t.Fatalf("want one bundling run for the entrypoints and one for the launchers:\n%s", strings.Join(r.ran, "\n")) - } - for _, want := range []string{"mesh-tools/build@sha256:", "predates one-file bundles", "--bundle", "--format=esm", - "--platform=node", "--outdir=" + Out("code") + ".bundled", Out("code") + "/index.js"} { - if !strings.Contains(bundling[0], want) { - t.Errorf("the entrypoints' bundling lacks %q: %s", want, bundling[0]) - } - } - if !strings.Contains(bundling[1], Out("code")+"/index.serve.mjs") || !strings.Contains(bundling[1], "--out-extension:.js=.mjs") { - t.Errorf("the launcher is not bundled under its own name: %s", bundling[1]) - } - if strings.Contains(strings.Join(r.ran, "\n"), "/app/runtime") { - t.Errorf("the toolchain's node_modules was copied into a bundle that keeps nothing external:\n%s", strings.Join(r.ran, "\n")) - } - if strings.Index(strings.Join(r.ran, "\n"), "--outDir") > strings.Index(strings.Join(r.ran, "\n"), "esbuild") { - t.Fatal("the bundler ran before the compile wrote its output") - } -} // A bundle naming packages it keeps external is bundled with them as imports, and carries the // toolchain's node_modules for them — the one case it still does. -func TestABundleKeepingAPackageExternalCarriesTheToolchainsModules(t *testing.T) { - manifest := strings.Replace(aBundle, `"entrypoints":["index.js"]`, `"entrypoints":["index.js"],"external":["sharp"]`, 1) - r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "console.log(1)"}) - held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} - if _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - all := strings.Join(r.ran, "\n") - if !strings.Contains(all, "--external:sharp") || !strings.Contains(all, "/app/runtime") { - t.Errorf("an external package was not kept as an import with the toolchain's modules beside it:\n%s", all) - } -} // A language whose bundle carries its own dependencies copies nothing in: a Go binary is static. func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) { @@ -146,39 +69,8 @@ func TestOnlyALanguageWithARuntimeDirectoryCopiesDependenciesIn(t *testing.T) { // **Refused before anything is built, naming what to build first.** A base the mesh has not built // is not a compile that fails on its first line — it is a question somebody can answer, and saying // it early is the difference between a fixable message and one about a missing image. -func TestABundleWhoseToolchainIsNotHeldIsRefusedFirst(t *testing.T) { - r, workspace := aRepository(t, aBundle, map[string]string{"index.ts": "console.log(1)"}) - - _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a bundle was built with no toolchain to compile it in") - } - if !strings.Contains(err.Error(), "mesh-tools") { - t.Fatalf("the refusal does not name what has to be built first: %v", err) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") { - t.Fatalf("a compile was attempted before the refusal: %s", line) - } - } -} // A language the mesh does not build is refused the same way, and names what it can build. -func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { - manifest := strings.Replace(aBundle, `"language":"typescript"`, `"language":"cobol"`, 1) - r, workspace := aRepository(t, manifest, map[string]string{"index.ts": "x"}) - - _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, - map[string]string{"mesh-tools/build": "registry.invalid/x@sha256:" + strings.Repeat("c", 64)}, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a language nothing can compile was accepted") - } - if !strings.Contains(err.Error(), "typescript") { - t.Fatalf("the refusal does not say what would have worked: %v", err) - } -} // **One module, two bundles, and neither packs the other.** // @@ -186,48 +78,3 @@ func TestABundleInAnUnknownLanguageIsRefused(t *testing.T) { // daemon in one language and tools in another (ADR 0040). An earlier version of this compiled // every bundle into the toolchain's single output directory, so two of them would overwrite each // other and then be packed together — one artifact containing both, twice. -func TestTwoBundlesInOneModuleArePackedSeparately(t *testing.T) { - const two = `{"module":"greeter","version":"1", - "build":{"artifacts":[ - {"name":"daemon","kind":"bundle","language":"typescript","entrypoints":["index.js"]}, - {"name":"tools","kind":"bundle","language":"typescript","entrypoints":["index.js"]}]}, - "resources":[ - {"id":"a","type":"archive","path":"/opt/greeter/daemon","artifact":"daemon"}, - {"id":"b","type":"archive","path":"/opt/greeter/tools","artifact":"tools"}]}` - - r, workspace := aRepository(t, two, map[string]string{"index.ts": "console.log(1)"}) - held := map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)} - - got, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/greeter.git", "", "", workspace, held, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a module with two bundles did not build: %v", err) - } - - // Compiled into two different places. Only the compile lines: the copy of each bundle's - // dependencies names the same directory again, deliberately. - var outputs []string - for _, line := range r.ran { - if !strings.Contains(line, "--outDir") { - continue - } - for _, part := range strings.Fields(line) { - if strings.HasPrefix(part, ".mesh-build/") { - outputs = append(outputs, part) - } - } - } - if len(outputs) != 2 || outputs[0] == outputs[1] { - t.Fatalf("two bundles did not get their own output directories: %v", outputs) - } - - // And published as two artifacts, each with its own digest. - if len(r.archives) != 2 { - t.Fatalf("expected two archives published, got %v", r.archives) - } - first, _ := got.Manifest.Resources[0]["digest"].(string) - second, _ := got.Manifest.Resources[1]["digest"].(string) - if first == "" || second == "" { - t.Fatalf("a bundle was not pinned: %v", got.Manifest.Resources) - } -} diff --git a/internal/builder/check.go b/internal/builder/check.go index 3f272fcd..d25d2687 100644 --- a/internal/builder/check.go +++ b/internal/builder/check.go @@ -91,16 +91,10 @@ type CheckSpec struct { // Judge is who judges the gate: "" the controller the mesh runs, "self" the change's own, "validator" // the running one with the change's validator. Judge string - // Release is the release branch the pull request merges into (novox/hq ADR 0358), empty for any other base. The - // gate is then judged by the controller beside it as mesh-controller — mesh-controller's release branch, never - // the controller the mesh runs nor its main — with mesh-host's validator from beside it when mesh-host was read at - // the release branch too; a change to the controller still judges itself and a change to the node-engine brings - // its own validator. The verdict names the branch, the judge's commit and the branch each beside was read at. - Release string // JudgeCache is the judge cache (novox/hq ADR 0358 §6): a directory of the build agent's own, outside the // workspace, that only judge builds mount — never a check's containers, so no pull request's script can write it. - // It holds the Go build cache every judge is built with (judges stay warm), the release-branch judges kept by the - // commits they are built from (judges/, one per release-branch head), and each judge build's fresh home (homes/, + // It holds the Go build cache every judge is built with (judges stay warm), the group's judges kept by the + // commits they are built from (judges/), and each judge build's fresh home (homes/, // removed after the build). Empty: each judge is built cold under the check's own root, which every check empties // before it clones, and nothing is kept. JudgeCache string @@ -206,12 +200,10 @@ func ToolchainsOf(held map[string]string) map[string]string { return out } -// Beside is one repository cloned next to the one checked: at Ref, or at Else when Else is said and the repository -// has no branch Ref (novox/hq ADR 0358 §3). +// Beside is one repository cloned next to the one checked, at Ref. type Beside struct { Repository string Ref string - Else string } // CheckVerdict is what came of one check. Verdict and Summary are the gate's; Gate and Repo each layer. @@ -234,19 +226,12 @@ type CheckVerdict struct { Members []JudgedHead // Built are the builds it made for its merge (novox/hq ADR 0350 §1), each with its key, or why not. Built []CheckBuilt - // Release is the release branch it was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the controller - // that judged its gate (twelve characters), and BesideAt the branch each repository beside it was read at, by - // directory. Empty for a check of any other base. - Release string - JudgedBy string - BesideAt map[string]string - // Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2): - // mesh-host's of the release branch with its commit, the one the release branch's controller vendors, the - // change's own, or the one the change's own controller vendors. Empty for a check of any other base. + // Validator says which node-engine validator the group's judge was built with (ADR 0382 §3), empty for any other + // judge. Validator string // Siblings are the pull requests of the delivery group the check read beside it, each merged onto its base commit // (novox/hq ADR 0382 §4); SiblingsUnknown is why it could not learn them; JudgeFrom says where the judge came from - // when the group made it (§3), the validator's source being in Validator. + // when the group made it (§3). Siblings []JudgedSibling SiblingsUnknown string JudgeFrom string @@ -393,22 +378,15 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry if replay.base != "" { defer os.RemoveAll(replay.base) } - // cloneAt clones a repository and checks out ref — or, when the repository has no branch ref and orElse is said, - // orElse (novox/hq ADR 0358 §3); readAt keeps which, by directory. + // clone clones a repository and checks out ref; readAt keeps which, by directory. readAt := map[string]string{} - cloneAt := func(repository, ref, orElse, dir string) error { + clone := func(repository, ref, dir string) error { if _, err := run(ctx, root, "git", cloneWith(credentials, "clone", "--quiet", repository, dir)...); err != nil { return fmt.Errorf("cannot clone %s: %s", redact.redact(repository), redact.redact(err.Error())) } if err := recordedWithoutUserinfo(ctx, run, filepath.Join(root, dir), repository); err != nil { return err } - if orElse != "" && ref != "" { - if _, err := run(ctx, filepath.Join(root, dir), "git", "rev-parse", "--verify", "--quiet", - "refs/remotes/origin/"+ref); err != nil { - ref = orElse - } - } readAt[dir] = ref if ref != "" { if _, err := run(ctx, filepath.Join(root, dir), "git", "checkout", "--quiet", ref); err != nil { @@ -417,9 +395,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } return nil } - clone := func(repository, ref, dir string) error { - return cloneAt(repository, ref, "", dir) - } name := spec.Repo if name == "" || !safeName.MatchString(name) { name = "checked" @@ -455,8 +430,8 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } named := func(v CheckVerdict) CheckVerdict { v.Head, v.Trunk, v.Merge, v.Beside, v.Members = judged.Head, judged.Trunk, judged.Merge, judged.Beside, judged.Members - v.Release, v.JudgedBy, v.BesideAt, v.Validator = spec.Release, judged.JudgedBy, judged.BesideAt, judged.Validator v.Siblings, v.SiblingsUnknown, v.JudgeFrom = judged.Siblings, spec.SiblingsUnknown, judged.JudgeFrom + v.Validator = judged.Validator return v } // **What the merge would move is built from a checkout nothing of the check touches** (novox/hq issue 703): the @@ -500,22 +475,10 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry if dir == name || !safeName.MatchString(dir) { continue } - if dir == "mesh-lab" { - // **mesh-lab is never read at a release branch** (novox/hq ADR 0358): its replays run with the container - // runtime's socket, so a branch of a release's name — which anyone may create — is never what runs there, - // whether it is asked with a fallback or without. - if b.Else != "" || (spec.Release != "" && b.Ref == spec.Release) { - b.Ref = b.Else - if b.Ref == "" { - b.Ref = "main" - } - } - b.Else = "" - } if sibling := siblingIn(spec, dir); sibling != nil && dir != "mesh-lab" { // **A sibling's repository is read at the sibling's merge** (novox/hq ADR 0382 §2): its head merged onto its // base's head, as its own check merges it — in place of the commit the mesh runs. - if err := cloneAt(b.Repository, "", "", dir); err != nil { + if err := clone(b.Repository, "", dir); err != nil { return CheckVerdict{}, fmt.Errorf("beside it, %w", err) } m, err := mergeIntoTrunk(ctx, run, filepath.Join(root, dir), sibling.Base, sibling.Head) @@ -532,7 +495,7 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry Gate: &Layer{Verdict: "fail", Summary: said}, Repo: &Layer{Verdict: "fail", Summary: said}}), nil } readAt[dir] = sibling.Base - } else if err := cloneAt(b.Repository, b.Ref, b.Else, dir); err != nil { + } else if err := clone(b.Repository, b.Ref, dir); err != nil { return CheckVerdict{}, fmt.Errorf("beside it, %w", err) } if dir == "mesh-lab" { @@ -542,13 +505,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry return CheckVerdict{}, fmt.Errorf("mesh-lab could not be copied for its replays: %w", err) } } - if spec.Release != "" { - // The branch each was read at is named with its commit (ADR 0358 §4). - if judged.BesideAt == nil { - judged.BesideAt = map[string]string{} - } - judged.BesideAt[dir] = readAt[dir] - } // The commit each one read is named in the verdict (ADR 0323 §3): its move is issue 568's question. if at := headAt(ctx, run, filepath.Join(root, dir)); at != "" { if judged.Beside == nil { @@ -756,32 +712,12 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry // The judge. Its failing to build is the change's fault when the change is the judge or its validator, // and the check's when it is the controller the mesh runs. var siblingJudge siblingJudgement - gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, judged.BesideAt, judged, &siblingJudge, inJudgeBuild, say) + gate, judgeFault, err := judgeFor(ctx, labelled, run, spec, root, tree, judged, &siblingJudge, inJudgeBuild, say) if err != nil { return CheckVerdict{}, err } judged.JudgeFrom = siblingJudge.From - if siblingJudge.Validator != "" { - judged.Validator = siblingJudge.Validator - } - if spec.Release != "" { - // **A release-branch judgement says so** (novox/hq ADR 0358 §4): the branch, the controller's commit that - // judged, and each repository read beside it at its branch and commit. - judgedBy := filepath.Join(root, "mesh-controller") - if spec.Judge == "self" { - judgedBy = tree - } - if siblingJudge.Dir != "" { - judgedBy = siblingJudge.Dir - } - judged.JudgedBy = headAt(ctx, run, judgedBy) - if siblingJudge.Validator == "" { - judged.Validator = releaseValidator(spec, judged.BesideAt, judged.Beside) - } - said := releaseWords(spec.Release, whoJudges(siblingJudge), judged.JudgedBy, judged.Validator, judged.BesideAt, judged.Beside) - fmt.Fprintf(&out, "--- %s\n", said) - say("check", "%s", said) - } + judged.Validator = siblingJudge.Validator verdictFile := filepath.Join(root, "verdict.json") env := []string{EnvFacts + "=" + factsFile, EnvGate + "=" + gate, EnvGateStore + "=" + storeURL, EnvTestStore + "=" + storeURL, EnvTestBus + "=nats://" + bus, EnvRepository + "=" + spec.Owner + "/" + spec.Repo, @@ -847,11 +783,6 @@ func Check(ctx context.Context, run Runner, spec CheckSpec, workspace, registry } } - if spec.Release != "" { - // The statuses name the release branch and the judge first, so the forge's short description keeps them. - mark := releaseMark(spec.Release, whoJudges(siblingJudge), judged.JudgedBy) - v.Gate.Summary, v.Repo.Summary = mark+v.Gate.Summary, mark+v.Repo.Summary - } // What the check printed travels in the verdict to the forge and the controller: redacted as the log is. v.Gate.Summary, v.Repo.Summary, v.Repo.Failed = redact.redact(v.Gate.Summary), redact.redact(v.Repo.Summary), redact.redact(v.Repo.Failed) @@ -1234,12 +1165,7 @@ func gateLayer(ctx context.Context, spec CheckSpec, tree, root, gate, verdictFil // the mesh runs. It answers the judge's path; or, when the change makes its own judge unbuildable, why — // the change's fault, a failing gate; or an error when the check cannot build a judge at all. A check // whose gate does not run builds a judge only to hand its scripts one, and goes on without when it cannot. -// -// **A pull request into a release branch is judged by that branch's controller** (novox/hq ADR 0358 §2): the one -// beside it as mesh-controller, read at the release branch — with mesh-host's validator from beside it when mesh-host -// has that branch too, or with the change's own validator for a change to the node-engine — never the controller the -// mesh runs. besideAt is the branch each repository beside it was read at. -func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, besideAt map[string]string, +func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, tree string, judged CheckVerdict, siblings *siblingJudgement, inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) { gated := spec.Gated() @@ -1256,14 +1182,7 @@ func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, t say("check", "judged by the change's own controller") return bin, "", nil } - var bin, judgeTree string - var err error - if spec.Release != "" { - withHost := spec.Judge != "validator" && besideAt["mesh-host"] == spec.Release - bin, judgeTree, err = releaseJudge(ctx, labelled, run, spec, root, withHost, inToolchain, say) - } else { - bin, judgeTree, err = judge(ctx, labelled, run, root, inToolchain, say) - } + bin, judgeTree, err := judge(ctx, labelled, run, root, inToolchain, say) if err != nil { if !gated { say("check", "no judge for its script: %v", err) @@ -1284,69 +1203,12 @@ func judgeFor(ctx context.Context, labelled, run Runner, spec CheckSpec, root, t withValidator := filepath.Join(root, "bin", "judge-with-this-validator") if _, err := labelled(ctx, root, "docker", inToolchain(filepath.Join(root, judgeTree), nil, "go", "build", "-o", withValidator, "./cmd/mesh-controller")...); err != nil { - if spec.Release != "" { - return "", spec.Release + "'s controller does not build with this change's validator: " + - firstLine(err.Error()), nil - } return "", "the controller the mesh runs does not build with this change's validator: " + firstLine(err.Error()), nil } - if spec.Release != "" { - say("check", "judged by %s's controller, with this change's validator", spec.Release) - return withValidator, "", nil - } say("check", "judged by the controller the mesh runs, with this change's validator") return withValidator, "", nil } -// releaseJudge builds the controller that judges a pull request into a release branch (novox/hq ADR 0358 §2): the -// one beside the check as mesh-controller, read at that branch — with mesh-host's validator from beside it in place -// of the one it vendors when withHost. **Built once per release-branch head**: kept in the spec's judge cache by the -// commits it is built from and taken from there while they stand. It answers the binary and the directory it was -// built from. -func releaseJudge(ctx context.Context, run, plain Runner, spec CheckSpec, root string, withHost bool, - inToolchain func(string, []string, ...string) []string, say func(step, format string, args ...any)) (string, string, error) { - const dir = "mesh-controller" - if _, err := os.Stat(filepath.Join(root, dir)); err != nil { - return "", "", fmt.Errorf("no controller of %s beside the check to judge it with", spec.Release) - } - key := fullHeadAt(ctx, plain, filepath.Join(root, dir)) - if key == "" { - return "", "", fmt.Errorf("the controller of %s beside the check names no commit", spec.Release) - } - with := "" - if withHost { - host := fullHeadAt(ctx, plain, filepath.Join(root, "mesh-host")) - if host == "" { - return "", "", fmt.Errorf("mesh-host of %s beside the check names no commit", spec.Release) - } - key += "-with-mesh-host-" + host - with = ", with mesh-host's validator of " + spec.Release + " at " + twelve(host) - } - bin := filepath.Join(root, "bin", "mesh-controller") - if took := judgeKept(keptJudges(spec.JudgeCache), key, bin); took { - say("check", "judged by %s's controller at %s%s, built for an earlier check of that head", spec.Release, - twelve(key), with) - return bin, dir, nil - } - if withHost { - vendored := filepath.Join(root, dir, "vendor", "github.com", "novox", "mesh-host") - for _, pkg := range []string{"validate", filepath.Join("internal", "declaration")} { - if err := replaceGoFiles(filepath.Join(root, "mesh-host", pkg), filepath.Join(vendored, pkg)); err != nil { - return "", "", fmt.Errorf("mesh-host's validator of %s could not be put in its controller: %w", spec.Release, err) - } - } - } - // Built where no pull request reaches (JudgeBuildEnv): a judge kept for this head is the head's, nothing else. - if _, err := run(ctx, root, "docker", inToolchain(filepath.Join(root, dir), nil, - "go", "build", "-o", bin, "./cmd/mesh-controller")...); err != nil { - return "", "", fmt.Errorf("the controller of %s does not build%s: %w", spec.Release, with, err) - } - // Kept before anything of the change runs: nothing a pull request's script writes can reach the cache. - keepJudge(keptJudges(spec.JudgeCache), key, bin, say) - say("check", "judged by %s's controller at %s%s", spec.Release, twelve(key), with) - return bin, dir, nil -} - // judgeBuildEnv is JudgeBuildEnv, a seam only a test replaces: to show what the plain environment would let a planted // workspace do to a judge's build. var judgeBuildEnv = JudgeBuildEnv @@ -1492,7 +1354,7 @@ func copyCheckout(ctx context.Context, run Runner, from, dir string) error { return err } -// keptJudges is where a judge cache keeps the release-branch judges, empty when there is no judge cache. +// keptJudges is where a judge cache keeps the group's judges, empty when there is no judge cache. func keptJudges(cache string) string { if cache == "" { return "" @@ -1500,7 +1362,7 @@ func keptJudges(cache string) string { return filepath.Join(cache, "judges") } -// judgesKept is how many release-branch judges a judge cache keeps: a head moved past is not built from again. +// judgesKept is how many judges a judge cache keeps: a head moved past is not built from again. const judgesKept = 4 // judgeKept puts the judge kept under key in the cache at bin, and says whether there was one. @@ -1574,47 +1436,6 @@ func fullHeadAt(ctx context.Context, run Runner, tree string) string { return strings.TrimSpace(out) } -// releaseValidator says which node-engine validator a release-branch check's judge is built with (novox/hq ADR 0358 -// §2), by the rule judgeFor builds it by. -func releaseValidator(spec CheckSpec, at, commits map[string]string) string { - switch { - case spec.Judge == "self": - return "the validator the change's own controller vendors" - case spec.Judge == "validator": - return "the change's own validator" - case at["mesh-host"] == spec.Release: - return fmt.Sprintf("mesh-host's validator of %s at %s", spec.Release, commits["mesh-host"]) - } - return fmt.Sprintf("the validator %s's controller vendors (mesh-host has no branch %s)", spec.Release, spec.Release) -} - -// releaseWords is how a release-branch check says what judged it and what it read (novox/hq ADR 0358 §4). -func releaseWords(release, controller, judgedBy, validator string, at, commits map[string]string) string { - var read []string - for _, dir := range sortedKeys(at) { - read = append(read, fmt.Sprintf("%s %s at %s", dir, refWords(at[dir]), commits[dir])) - } - said := fmt.Sprintf("judged for the release branch %s by %s at %s, with %s", release, controller, judgedBy, validator) - if len(read) > 0 { - said += "; read beside it: " + strings.Join(read, ", ") - } - return said -} - -// whoJudges says which controller judged: the release branch's own ("its controller"), or — when a delivery group's -// member in mesh-controller judged (novox/hq ADR 0239 §14) — the group's. -func whoJudges(j siblingJudgement) string { - if j.Dir != "" { - return "the group's controller" - } - return "its controller" -} - -// releaseMark opens each layer's summary of a release-branch check, so a status says it first. -func releaseMark(release, controller, judgedBy string) string { - return fmt.Sprintf("[release %s, judged by %s %s] ", release, controller, judgedBy) -} - // refWords is a ref as a line says it: a commit by its first eight characters, a branch whole. func refWords(ref string) string { if commitLike.MatchString(ref) { @@ -1626,15 +1447,6 @@ func refWords(ref string) string { // commitLike is a ref that names a commit. var commitLike = regexp.MustCompile(`^[0-9a-f]{7,40}$`) -func sortedKeys(m map[string]string) []string { - keys := make([]string, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return keys -} - // replaceGoFiles puts a package's Go files — never its tests — in place of another copy's. func replaceGoFiles(from, into string) error { old, err := filepath.Glob(filepath.Join(into, "*.go")) @@ -2004,8 +1816,7 @@ func judgeFromTheGroup(ctx context.Context, labelled, run Runner, spec CheckSpec switch { case len(spec.Group) > 0 && groupDir(spec, root, tree, "mesh-controller") != "": // **A delivery group's composed check is judged by the group's own controller** (novox/hq ADR 0239 §14), whatever - // base it merges into — a release branch included (ADR 0358 §2 changes the judge of one pull request, not of a - // group): the member that moves mesh-controller, as merged, with the validator of the group's mesh-host member's + // base it merges into: the member that moves mesh-controller, as merged, with the validator of the group's mesh-host member's // merge when it has one, else the one that controller vendors. source, validatorDir = groupDir(spec, root, tree, "mesh-controller"), groupDir(spec, root, tree, "mesh-host") from = "the group's own controller, merge " + headAt(ctx, run, source) diff --git a/internal/builder/check_base_test.go b/internal/builder/check_base_test.go index 16aa5484..e1887137 100644 --- a/internal/builder/check_base_test.go +++ b/internal/builder/check_base_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/check_credential_integration_test.go b/internal/builder/check_credential_integration_test.go new file mode 100644 index 00000000..6e56036b --- /dev/null +++ b/internal/builder/check_credential_integration_test.go @@ -0,0 +1,180 @@ +//go:build integration + +package builder + +import ( + "context" + "errors" + "io/fs" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestACheckContainerSeesNoForgeCredential(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) + besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"}) + // A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the + // clone's .git/config, which the container reads. + besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo + workspace := t.TempDir() + + var stores []string + reached := false + var leaks []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + switch name { + case "git": + for _, a := range args { + if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok { + stores = append(stores, f) + raw, err := os.ReadFile(f) + if err != nil || strings.TrimSpace(string(raw)) != forgeURL { + t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err) + } + if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 { + t.Errorf("the credential store is readable beyond its owner: %v", info.Mode()) + } + } + } + if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") { + source := args[len(args)-2] + if u, err := url.Parse(source); err == nil && u.User != nil { + // Git cannot reach a file:// URL with userinfo; clone it without, then record it as git + // would have: as given. + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source) + } + } + return Command(ctx, dir, name, args...) + case "docker": + if !reached { + reached = true + // The first container: everything the workspace holds is what the toolchain container sees. + filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + raw, _ := os.ReadFile(path) + s := string(raw) + if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || + strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" || + strings.Contains(s, "credential.helper") { + leaks = append(leaks, path) + } + return nil + }) + for _, f := range stores { + if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) { + leaks = append(leaks, f+" (still there when the first container runs)") + } + if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") { + leaks = append(leaks, f+" (inside the workspace the container mounts)") + } + } + } + if len(args) > 0 && args[0] == "ps" { + return "", nil + } + return "", errors.New("no container runtime in this test") + } + return "", errors.New("unexpected command " + name) + } + // A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a + // build wrote into the tree it cloned. + if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, dir := range []string{"source/x", "context-server"} { + if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"), + []byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil { + t.Fatal(err) + } + } + _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ + "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), + GitCredential{URL: forgeURL}, nil) + if err == nil { + t.Fatal("the check ran past its first container in a test with none") + } + if !reached { + t.Fatalf("the check never reached its first container: %v", err) + } + if len(stores) == 0 { + t.Fatal("no clone was offered the forge credential") + } + if len(leaks) > 0 { + t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n")) + } +} + +func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) { + var said []string + say := func(step, format string, args ...any) { + if step == "output" && len(args) > 0 { + said = append(said, args[0].(string)) + } + } + var out tail + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}}, + t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd { + return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+ + "echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789") + }, say) + if layer == nil || layer.Verdict != "pass" { + t.Fatalf("the check answered %+v\n%s", layer, out.String()) + } + joined := strings.Join(said, "\n") + if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") { + t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined) + } + if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") { + t.Fatalf("the line is not said with what was there named:\n%s", joined) + } +} + +func TestABuildsCloneRecordsNoUserinfo(t *testing.T) { + repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`}) + source := "file://build-user:" + besideSecret + "@" + repo + workspace := t.TempDir() + tree := filepath.Join(workspace, "source") + var configs []string + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "git" && hasString(args, "clone") && args[len(args)-2] == source { + clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) + if out, err := Command(ctx, dir, "git", clone...); err != nil { + return out, err + } + return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source) + } + if name == "git" && len(args) > 0 && args[0] == "rev-parse" { + raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config")) + configs = append(configs, string(raw)) + } + return Command(ctx, dir, name, args...) + } + if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if len(configs) == 0 { + t.Fatal("the build never read its clone") + } + for _, c := range configs { + if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") { + t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c) + } + } + if _, err := os.Stat(tree); !os.IsNotExist(err) { + t.Fatalf("the build's tree outlives the build: %v", err) + } +} diff --git a/internal/builder/check_credential_test.go b/internal/builder/check_credential_test.go index 7fc52aa9..f0c63f1c 100644 --- a/internal/builder/check_credential_test.go +++ b/internal/builder/check_credential_test.go @@ -1,18 +1,12 @@ package builder import ( - "context" "crypto/sha256" "encoding/hex" "encoding/json" - "errors" - "io/fs" "net/http" "net/http/httptest" "net/url" - "os" - "os/exec" - "path/filepath" "strings" "testing" "time" @@ -67,135 +61,7 @@ func bareURL(t *testing.T, raw string) string { return u.String() } -func TestACheckContainerSeesNoForgeCredential(t *testing.T) { - repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) - besideRepo, besideHead := aCheckedRepository(t, map[string]string{"README": "beside"}) - // A clone source that carries userinfo, as a forge's clone URL may: git records it as given in the - // clone's .git/config, which the container reads. - besideURL := "file://beside-user:" + besideSecret + "@" + besideRepo - workspace := t.TempDir() - - var stores []string - reached := false - var leaks []string - run := func(ctx context.Context, dir, name string, args ...string) (string, error) { - switch name { - case "git": - for _, a := range args { - if f, ok := strings.CutPrefix(a, "credential.helper=store --file="); ok { - stores = append(stores, f) - raw, err := os.ReadFile(f) - if err != nil || strings.TrimSpace(string(raw)) != forgeURL { - t.Errorf("git is offered a store that does not hold the credential as given: %q, %v", raw, err) - } - if info, err := os.Stat(f); err == nil && info.Mode().Perm() != 0o600 { - t.Errorf("the credential store is readable beyond its owner: %v", info.Mode()) - } - } - } - if len(args) >= 2 && args[len(args)-3] == "--quiet" && hasString(args, "clone") { - source := args[len(args)-2] - if u, err := url.Parse(source); err == nil && u.User != nil { - // Git cannot reach a file:// URL with userinfo; clone it without, then record it as git - // would have: as given. - clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) - if out, err := Command(ctx, dir, "git", clone...); err != nil { - return out, err - } - return Command(ctx, filepath.Join(dir, args[len(args)-1]), "git", "remote", "set-url", "origin", source) - } - } - return Command(ctx, dir, name, args...) - case "docker": - if !reached { - reached = true - // The first container: everything the workspace holds is what the toolchain container sees. - filepath.WalkDir(workspace, func(path string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return nil - } - raw, _ := os.ReadFile(path) - s := string(raw) - if strings.Contains(s, forgeSecret) || strings.Contains(s, besideSecret) || - strings.Contains(s, npmSecret) || d.Name() == "git-credentials" || d.Name() == ".npmrc" || - strings.Contains(s, "credential.helper") { - leaks = append(leaks, path) - } - return nil - }) - for _, f := range stores { - if _, err := os.Stat(f); !errors.Is(err, os.ErrNotExist) { - leaks = append(leaks, f+" (still there when the first container runs)") - } - if rel, err := filepath.Rel(workspace, f); err == nil && !strings.HasPrefix(rel, "..") { - leaks = append(leaks, f+" (inside the workspace the container mounts)") - } - } - } - if len(args) > 0 && args[0] == "ps" { - return "", nil - } - return "", errors.New("no container runtime in this test") - } - return "", errors.New("unexpected command " + name) - } - // A credential an older builder left in the workspace is removed too: the forge's, and the .npmrc a - // build wrote into the tree it cloned. - if err := os.WriteFile(filepath.Join(workspace, "git-credentials"), []byte(forgeURL+"\n"), 0o600); err != nil { - t.Fatal(err) - } - for _, dir := range []string{"source/x", "context-server"} { - if err := os.MkdirAll(filepath.Join(workspace, dir), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(workspace, dir, ".npmrc"), - []byte("//forge.invalid/api/packages/novox/npm/:_authToken="+npmSecret+"\n"), 0o600); err != nil { - t.Fatal(err) - } - } - _, err := Check(t.Context(), run, CheckSpec{ID: "check-462", Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-controller", Number: 1, Toolchain: "golang", Beside: map[string]Beside{ - "mesh-catalog": {Repository: besideURL, Ref: besideHead}}}, workspace, aFactsRegistry(t), - GitCredential{URL: forgeURL}, nil) - if err == nil { - t.Fatal("the check ran past its first container in a test with none") - } - if !reached { - t.Fatalf("the check never reached its first container: %v", err) - } - if len(stores) == 0 { - t.Fatal("no clone was offered the forge credential") - } - if len(leaks) > 0 { - t.Fatalf("the check's container sees the credential:\n%s", strings.Join(leaks, "\n")) - } -} - // Every line a repository's own check prints is published to the build's log redacted. -func TestACheckLinePublishedToTheLogIsRedacted(t *testing.T) { - var said []string - say := func(step, format string, args ...any) { - if step == "output" && len(args) > 0 { - said = append(said, args[0].(string)) - } - } - var out tail - layer := ownCheck(t.Context(), CheckSpec{Toolchain: "golang"}, []ScriptPart{{Toolchain: "go", Script: CheckScript}}, - t.TempDir(), &out, func() bool { return false }, func(string, string) *exec.Cmd { - return exec.CommandContext(t.Context(), "sh", "-c", "echo cloning http://mesh_builder:t0ps3cret-forge@forge.invalid/novox/x.git; "+ - "echo token ghp_abcdefghijklmnopqrstuvwxyz0123456789") - }, say) - if layer == nil || layer.Verdict != "pass" { - t.Fatalf("the check answered %+v\n%s", layer, out.String()) - } - joined := strings.Join(said, "\n") - if strings.Contains(joined, "t0ps3cret-forge") || strings.Contains(joined, "ghp_abcdef") { - t.Fatalf("a secret the check printed is published to the build's log:\n%s", joined) - } - if !strings.Contains(joined, "http://mesh_builder:[redacted: a password in a URI]@forge.invalid/novox/x.git") { - t.Fatalf("the line is not said with what was there named:\n%s", joined) - } -} func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { r := redactorFor(GitCredential{URL: forgeURL}) @@ -214,38 +80,3 @@ func TestTheRedactorHidesTheForgeCredentialAndShapes(t *testing.T) { // A build's clone of a URL carrying userinfo records it without, so no build leaves a credential in // workspace/source/.git/config while it runs either (novox/hq issue 462); the tree itself is gone when the // build ends. -func TestABuildsCloneRecordsNoUserinfo(t *testing.T) { - repo, _ := aCheckedRepository(t, map[string]string{ManifestName: `{"module":"plain","version":"1"}`}) - source := "file://build-user:" + besideSecret + "@" + repo - workspace := t.TempDir() - tree := filepath.Join(workspace, "source") - var configs []string - run := func(ctx context.Context, dir, name string, args ...string) (string, error) { - if name == "git" && hasString(args, "clone") && args[len(args)-2] == source { - clone := append(append([]string{}, args[:len(args)-2]...), bareURL(t, source), args[len(args)-1]) - if out, err := Command(ctx, dir, "git", clone...); err != nil { - return out, err - } - return Command(ctx, args[len(args)-1], "git", "remote", "set-url", "origin", source) - } - if name == "git" && len(args) > 0 && args[0] == "rev-parse" { - raw, _ := os.ReadFile(filepath.Join(tree, ".git", "config")) - configs = append(configs, string(raw)) - } - return Command(ctx, dir, name, args...) - } - if _, err := Build(t.Context(), run, &recorded{}, source, "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - if len(configs) == 0 { - t.Fatal("the build never read its clone") - } - for _, c := range configs { - if strings.Contains(c, besideSecret) || strings.Contains(c, "build-user") { - t.Fatalf("the build's clone records the credential it was cloned with:\n%s", c) - } - } - if _, err := os.Stat(tree); !os.IsNotExist(err) { - t.Fatalf("the build's tree outlives the build: %v", err) - } -} diff --git a/internal/builder/check_integration_test.go b/internal/builder/check_integration_test.go new file mode 100644 index 00000000..6e9f9992 --- /dev/null +++ b/internal/builder/check_integration_test.go @@ -0,0 +1,399 @@ +//go:build integration + +package builder + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestTheThrowawayStoreFlushesNothing(t *testing.T) { + said := strings.Join(StoreSettings, " ") + for _, want := range []string{"-c fsync=off", "-c synchronous_commit=off", "-c full_page_writes=off"} { + if !strings.Contains(said, want) { + t.Errorf("the store is raised with %q, not %q", said, want) + } + } + if os.Getenv("MESH_TEST_DOCKER") != "1" { + t.Skip("MESH_TEST_DOCKER=1: the settings are proven on a raised store") + } + id := fmt.Sprintf("check-store-%d", time.Now().UnixNano()) + labelledRun := Labelled(Command, id) + t.Cleanup(func() { _, _ = RemoveContainersOf(context.Background(), Command, id) }) + if _, err := throwaway(t.Context(), labelledRun, Command, id+"-store", StoreImage("17.11"), 5432, + []string{"-e", "POSTGRES_PASSWORD=check"}, StoreSettings); err != nil { + t.Fatal(err) + } + if err := waitFor(t.Context(), Command, id+"-store", []string{"pg_isready", "-U", "postgres"}); err != nil { + t.Fatal(err) + } + for setting, want := range map[string]string{"fsync": "off", "synchronous_commit": "off", "full_page_writes": "off"} { + var got string + for try := 0; try < 10; try++ { + out, err := exec.Command("docker", "exec", id+"-store", "psql", "-U", "postgres", "-tAc", "SHOW "+setting).CombinedOutput() + if got = strings.TrimSpace(string(out)); err == nil { + break + } + time.Sleep(500 * time.Millisecond) + } + if got != want { + t.Errorf("%s is %q on the raised store", setting, got) + } + } +} + +func aCheckedRepository(t *testing.T, files map[string]string) (string, string) { + t.Helper() + dir := t.TempDir() + git := func(args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("init", "--quiet", "-b", "main") + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + git("add", "-A") + git("commit", "--quiet", "-m", "x") + return dir, git("rev-parse", "HEAD") +} + +func labelled(id string) []string { + out, _ := exec.Command("docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id).Output() + return strings.Fields(string(out)) +} + +func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) { + registry := checkEnvironment(t) + // The script proves what it was given: the facts, a store that answers, a bus that answers — and no + // container runtime socket. + script := `set -e +test -s "$MESH_FACTS" +grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS" +case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac +case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac +test "$MESH_CHECK_REPOSITORY" = "novox/hq" +test "$MESH_CHECK_CHANGED" = "a.go,b.go" +test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; } +echo checked +` + repo, head := aCheckedRepository(t, map[string]string{CheckScript: script}) + id := fmt.Sprintf("check-test-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + // Nothing of the graph touched: the gate a pass that says so, the repository's own check run. + if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule { + t.Errorf("the gate answered %+v", v.Gate) + } + if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") { + t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } +} + +func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) { + registry := checkEnvironment(t) + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"}) + v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") { + t.Fatalf("a failing script answered %+v", v.Repo) + } + + // A script in a toolchain the mesh does not hold: an error, never a pass. + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"}) + v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil || v.Repo == nil || v.Repo.Verdict != "error" { + t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err) + } + + // Past its bound: an error, not a pass. + was := CheckTimeout + CheckTimeout = 25 * time.Second + t.Cleanup(func() { CheckTimeout = was }) + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"}) + v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()), + Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") { + t.Fatalf("a check past its bound answered %+v", v.Repo) + } + + // No facts: it cannot run, and says so. + repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"}) + _, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox", + Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "facts snapshot") { + t.Fatalf("a check with no facts said %v", err) + } +} + +func TestACheckWithNothingToRunSaysSo(t *testing.T) { + repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"}) + v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox", + Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" || + !strings.Contains(v.Repo.Summary, CheckScript) { + t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo) + } + // A gated change never takes that path: with no store to read the facts from, it cannot run. + _, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil) + if err == nil { + t.Fatal("a change touching a module ran nothing and was not refused") + } +} + +func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) { + registry := checkEnvironment(t) + judgeRepo, judgeHead := aCheckedRepository(t, aJudge) + beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}} + check := func(files map[string]string, judge string) CheckVerdict { + t.Helper() + repo, head := aCheckedRepository(t, files) + id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside, + Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge, + Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } + return v + } + v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "") + if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" || + strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" { + t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report) + } + if v.Repo == nil || v.Repo.Verdict != "pass" { + t.Fatalf("its own check answered %+v", v.Repo) + } + + v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "") + if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" { + t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report) + } + + // A fault the base branch already had is said and fails nothing; one the change brings fails. + repo, _ := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}) + git := func(args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = repo + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git("checkout", "--quiet", "-b", "change") + if err := os.WriteFile(filepath.Join(repo, "modules/gitea/index.ts"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + git("add", "-A") + git("commit", "--quiet", "-m", "y") + id := fmt.Sprintf("check-base-%d", time.Now().UnixNano()) + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: git("rev-parse", "HEAD"), Owner: "novox", + Repo: "mesh-catalog", Base: "main", Paths: []string{"modules/gitea/index.ts"}, Beside: beside, + Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain}, + t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate.Verdict != "warning" || !strings.Contains(v.Report, "already") { + t.Fatalf("a fault the base already had failed the change: %+v\n%s", v.Gate, v.Report) + } + + // A delivery group (novox/hq ADR 0239): the other heads cloned beside it at their heads and handed to the + // gate to compose with this one; the repository's own check is each pull request's, not the group's. + app, appHead := aCheckedRepository(t, map[string]string{"module.json": `{"module":"app"}`}) + repo2, head2 := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, + CheckScript: "exit 1\n"}) + gid := fmt.Sprintf("check-group-%d", time.Now().UnixNano()) + v, err = Check(t.Context(), Command, CheckSpec{ID: gid, Repository: repo2, Ref: head2, Owner: "novox", + Repo: "mesh-catalog", Paths: []string{"modules/gitea/module.json"}, Beside: beside, Modules: []string{"gitea"}, + Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain, + Group: []GroupHead{{Owner: "novox", Repo: "app", Repository: app, Ref: appHead, Paths: []string{"module.json"}}}}, + t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if v.Gate.Verdict != "pass" || v.Gate.Summary != "composed with novox/app" { + t.Fatalf("the group's other head was not composed: %+v\n%s", v.Gate, v.Report) + } + if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Repo.Summary, "group") { + t.Fatalf("a group's check ran a member's own script: %+v", v.Repo) + } + + // A change to the controller judges itself: one that does not build fails its own gate. + v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n", + "modules/gitea/module.json": "{}"}, "self") + if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") { + t.Fatalf("a controller that does not build judged itself %+v", v.Gate) + } +} + +func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) { + repo, onMain := aCheckedRepository(t, map[string]string{"module.json": `{"module":"x","version":"1"}`}) + git := func(dir string, args ...string) string { + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + git(repo, "checkout", "--quiet", "-b", "feature") + if err := os.WriteFile(filepath.Join(repo, "x"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + git(repo, "add", "-A") + git(repo, "commit", "--quiet", "-m", "off the trunk") + offMain := git(repo, "rev-parse", "HEAD") + git(repo, "checkout", "--quiet", "main") + + clone := filepath.Join(t.TempDir(), "clone") + git(filepath.Dir(clone), "clone", "--quiet", repo, clone) + if trunk, on := trunkOf(t.Context(), Command, clone, onMain); trunk != "main" || !on { + t.Errorf("a commit on main reads as on %q: %v", trunk, on) + } + if trunk, on := trunkOf(t.Context(), Command, clone, offMain); trunk != "main" || on { + t.Errorf("a feature branch's commit reads as on %q: %v", trunk, on) + } + if got := strings.Join(branchesHolding(t.Context(), Command, clone, offMain), ","); got != "feature" { + t.Errorf("the feature branch's commit is said to be on %q", got) + } + if got := strings.Join(branchesHolding(t.Context(), Command, clone, onMain), ","); got != "feature,main" { + t.Errorf("main's commit is said to be on %q", got) + } + // A tree that says no trunk is not known — never read as on it. + if trunk, on := trunkOf(t.Context(), Command, repo, onMain); trunk != "" || on { + t.Errorf("a repository with no origin reads as trunk %q, on %v", trunk, on) + } +} + +func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) { + registry := checkEnvironment(t) + repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) + id := fmt.Sprintf("check-again-%d", time.Now().UnixNano()) + if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store", + "postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil { + t.Skipf("no container for the earlier delivery: %v %s", err, out) + } + v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", + Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) + if err != nil { + t.Fatalf("a redelivered check could not run: %v", err) + } + if v.Repo == nil || v.Repo.Verdict != "pass" { + t.Errorf("the repository's check answered %+v", v.Repo) + } + if left := labelled(id); len(left) > 0 { + t.Errorf("the check left %d container(s) behind", len(left)) + } +} + +func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { + script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n" + parts := ScriptParts([]byte(script)) + if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) || + parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) { + t.Fatalf("the parts read as %+v", parts) + } + if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) { + t.Fatalf("a script declaring nothing reads as %+v", got) + } + + // Each part run where its toolchain is: here, the image the part was given is what the command says. + held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}} + run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) { + t.Helper() + tree := t.TempDir() + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + var images []string + var out tail + layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false }, + func(image, script string) *exec.Cmd { + images = append(images, image+" "+script) + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = tree + return cmd + }, func(string, string, ...any) {}) + return layer, images + } + twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}} + + layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) + if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") || + strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" { + t.Errorf("both parts passing answered %+v, ran %v", layer, images) + } + layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n", + "replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts) + if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") || + !strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") { + t.Errorf("a failing Go part answered %+v", layer) + } + layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) + if layer.Verdict != "fail" || len(images) != 1 { + t.Errorf("a failing first part answered %+v and ran %v", layer, images) + } + layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts) + if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") { + t.Errorf("a declared part the repository lacks answered %+v", layer) + } + layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript}, + {"go", "../elsewhere.sh"}}) + if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") { + t.Errorf("a part outside the repository answered %+v", layer) + } + layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}}, + map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) + if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") { + t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) + } +} diff --git a/internal/builder/check_merge_test.go b/internal/builder/check_merge_test.go index 86ad2da0..3bc42bc7 100644 --- a/internal/builder/check_merge_test.go +++ b/internal/builder/check_merge_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/check_test.go b/internal/builder/check_test.go index 1eb5890a..b712b25d 100644 --- a/internal/builder/check_test.go +++ b/internal/builder/check_test.go @@ -1,11 +1,8 @@ package builder import ( - "context" "encoding/json" - "fmt" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -35,69 +32,8 @@ func TestTheStoreAndBusAChecksStandsOnAreTheOnesTheMeshRuns(t *testing.T) { // **Issue 306**: the throwaway store is raised without durability — it lives for one check — so a suite // that makes a database per test does not wait on the disk of a busy machine for every flush. -func TestTheThrowawayStoreFlushesNothing(t *testing.T) { - said := strings.Join(StoreSettings, " ") - for _, want := range []string{"-c fsync=off", "-c synchronous_commit=off", "-c full_page_writes=off"} { - if !strings.Contains(said, want) { - t.Errorf("the store is raised with %q, not %q", said, want) - } - } - if os.Getenv("MESH_TEST_DOCKER") != "1" { - t.Skip("MESH_TEST_DOCKER=1: the settings are proven on a raised store") - } - id := fmt.Sprintf("check-store-%d", time.Now().UnixNano()) - labelledRun := Labelled(Command, id) - t.Cleanup(func() { _, _ = RemoveContainersOf(context.Background(), Command, id) }) - if _, err := throwaway(t.Context(), labelledRun, Command, id+"-store", StoreImage("17.11"), 5432, - []string{"-e", "POSTGRES_PASSWORD=check"}, StoreSettings); err != nil { - t.Fatal(err) - } - if err := waitFor(t.Context(), Command, id+"-store", []string{"pg_isready", "-U", "postgres"}); err != nil { - t.Fatal(err) - } - for setting, want := range map[string]string{"fsync": "off", "synchronous_commit": "off", "full_page_writes": "off"} { - var got string - for try := 0; try < 10; try++ { - out, err := exec.Command("docker", "exec", id+"-store", "psql", "-U", "postgres", "-tAc", "SHOW "+setting).CombinedOutput() - if got = strings.TrimSpace(string(out)); err == nil { - break - } - time.Sleep(500 * time.Millisecond) - } - if got != want { - t.Errorf("%s is %q on the raised store", setting, got) - } - } -} // aRepository is a git repository holding these files, committed, and its head. -func aCheckedRepository(t *testing.T, files map[string]string) (string, string) { - t.Helper() - dir := t.TempDir() - git := func(args ...string) string { - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git("init", "--quiet", "-b", "main") - for name, body := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(dir, name)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o755); err != nil { - t.Fatal(err) - } - } - git("add", "-A") - git("commit", "--quiet", "-m", "x") - return dir, git("rev-parse", "HEAD") -} func checkEnvironment(t *testing.T) string { t.Helper() @@ -121,104 +57,8 @@ func checkEnvironment(t *testing.T) string { // goToolchain is the Go image a check's script runs in here: the base the controller's own image is built on. const goToolchain = "golang@sha256:8ac98ca534ac3f51e1f420a1dd2c15e74c75cfa0f23f3ad27eb5d7236c349a0c" -func labelled(id string) []string { - out, _ := exec.Command("docker", "ps", "-aq", "--filter", "label="+BuildLabel+"="+id).Output() - return strings.Fields(string(out)) -} - -func TestACheckRunsTheRepositorysOwnScriptBesideTheMeshsVersionsAndLeavesNothing(t *testing.T) { - registry := checkEnvironment(t) - // The script proves what it was given: the facts, a store that answers, a bus that answers — and no - // container runtime socket. - script := `set -e -test -s "$MESH_FACTS" -grep -q '"bus": "2.11.17"' "$MESH_FACTS" || grep -q '"bus":"2.11.17"' "$MESH_FACTS" -case "$MESH_TEST_POSTGRES" in postgres://*127.0.0.1:*) ;; *) echo "no store: $MESH_TEST_POSTGRES"; exit 1;; esac -case "$MESH_TEST_NATS" in nats://127.0.0.1:*) ;; *) echo "no bus: $MESH_TEST_NATS"; exit 1;; esac -test "$MESH_CHECK_REPOSITORY" = "novox/hq" -test "$MESH_CHECK_CHANGED" = "a.go,b.go" -test ! -S /var/run/docker.sock || { echo "the check holds the container runtime's socket"; exit 1; } -echo checked -` - repo, head := aCheckedRepository(t, map[string]string{CheckScript: script}) - id := fmt.Sprintf("check-test-%d", time.Now().UnixNano()) - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "hq", Number: 7, Paths: []string{"a.go", "b.go"}, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - // Nothing of the graph touched: the gate a pass that says so, the repository's own check run. - if v.Gate == nil || v.Gate.Verdict != "pass" || v.Gate.Summary != noModule { - t.Errorf("the gate answered %+v", v.Gate) - } - if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Report, "checked") { - t.Fatalf("the repository's check answered %+v\n%s", v.Repo, v.Report) - } - if left := labelled(id); len(left) > 0 { - t.Errorf("the check left %d container(s) behind", len(left)) - } -} - -func TestAFailingCheckFailsAndOneThatCannotRunIsNeverAPass(t *testing.T) { - registry := checkEnvironment(t) - repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo 'resource \"x.service\": refused'; exit 3\n"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-fail-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Repo == nil || v.Repo.Verdict != "fail" || !strings.Contains(v.Repo.Summary, "refused") { - t.Fatalf("a failing script answered %+v", v.Repo) - } - - // A script in a toolchain the mesh does not hold: an error, never a pass. - repo, head = aCheckedRepository(t, map[string]string{CheckScript: "# mesh-check-toolchain: cobol\nexit 0\n"}) - v, err = Check(t.Context(), Command, CheckSpec{ID: fmt.Sprintf("check-cobol-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil || v.Repo == nil || v.Repo.Verdict != "error" { - t.Fatalf("a script in a toolchain nobody holds answered %+v, %v", v.Repo, err) - } - - // Past its bound: an error, not a pass. - was := CheckTimeout - CheckTimeout = 25 * time.Second - t.Cleanup(func() { CheckTimeout = was }) - repo, head = aCheckedRepository(t, map[string]string{CheckScript: "sleep 120\n"}) - v, err = Check(context.Background(), Command, CheckSpec{ID: fmt.Sprintf("check-slow-%d", time.Now().UnixNano()), - Repository: repo, Ref: head, Owner: "novox", Repo: "hq", Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err == nil && (v.Repo == nil || v.Repo.Verdict != "error") { - t.Fatalf("a check past its bound answered %+v", v.Repo) - } - - // No facts: it cannot run, and says so. - repo, head = aCheckedRepository(t, map[string]string{CheckScript: "exit 0\n"}) - _, err = Check(t.Context(), Command, CheckSpec{ID: "check-nofacts", Repository: repo, Ref: head, Owner: "novox", - Repo: "hq", Toolchain: goToolchain}, t.TempDir(), "127.0.0.1:1", GitCredential{}, nil) - if err == nil || !strings.Contains(err.Error(), "facts snapshot") { - t.Fatalf("a check with no facts said %v", err) - } -} - // A repository that touches nothing of the graph and has no script of its own runs nothing, and says // both: the gate a pass that names why, the repository a warning — never silent. -func TestACheckWithNothingToRunSaysSo(t *testing.T) { - repo, head := aCheckedRepository(t, map[string]string{"README.md": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-nothing", Repository: repo, Ref: head, Owner: "novox", - Repo: "hq"}, t.TempDir(), "", GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || v.Gate.Verdict != "pass" || v.Repo == nil || v.Repo.Verdict != "warning" || - !strings.Contains(v.Repo.Summary, CheckScript) { - t.Fatalf("a check with nothing to run said %+v / %+v", v.Gate, v.Repo) - } - // A gated change never takes that path: with no store to read the facts from, it cannot run. - _, err = Check(t.Context(), Command, CheckSpec{ID: "check-gated", Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-catalog", Modules: []string{"gitea"}}, t.TempDir(), "", GitCredential{}, nil) - if err == nil { - t.Fatal("a change touching a module ran nothing and was not refused") - } -} // A script declares its toolchain among its first lines; go when it declares none. func TestAScriptDeclaresItsToolchain(t *testing.T) { @@ -314,143 +154,8 @@ func main() { `, } -func TestTheGateRunsWhenTheGraphIsTouchedBesideTheRepositorysOwnCheck(t *testing.T) { - registry := checkEnvironment(t) - judgeRepo, judgeHead := aCheckedRepository(t, aJudge) - beside := map[string]Beside{"mesh-controller": {Repository: judgeRepo, Ref: judgeHead}} - check := func(files map[string]string, judge string) CheckVerdict { - t.Helper() - repo, head := aCheckedRepository(t, files) - id := fmt.Sprintf("check-gate-%d", time.Now().UnixNano()) - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "mesh-catalog", Number: 9, Paths: []string{"modules/gitea/index.ts"}, Beside: beside, - Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Judge: judge, - Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if left := labelled(id); len(left) > 0 { - t.Errorf("the check left %d container(s) behind", len(left)) - } - return v - } - v := check(map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "echo own; exit 0\n"}, "") - if v.Gate == nil || v.Gate.Verdict != "warning" || v.Gate.Summary != "a merge rebuilds 14 module(s)" || - strings.Join(v.Gate.Modules, ",") != "gitea" || v.Verdict != "warning" { - t.Fatalf("the gate answered %+v\n%s", v.Gate, v.Report) - } - if v.Repo == nil || v.Repo.Verdict != "pass" { - t.Fatalf("its own check answered %+v", v.Repo) - } - - v = check(map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}, "") - if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "module check") || v.Repo.Verdict != "warning" { - t.Fatalf("a manifest the judge refuses answered %+v / %+v\n%s", v.Gate, v.Repo, v.Report) - } - - // A fault the base branch already had is said and fails nothing; one the change brings fails. - repo, _ := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea","broken":true}`}) - git := func(args ...string) string { - cmd := exec.Command("git", args...) - cmd.Dir = repo - cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git("checkout", "--quiet", "-b", "change") - if err := os.WriteFile(filepath.Join(repo, "modules/gitea/index.ts"), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - git("add", "-A") - git("commit", "--quiet", "-m", "y") - id := fmt.Sprintf("check-base-%d", time.Now().UnixNano()) - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: git("rev-parse", "HEAD"), Owner: "novox", - Repo: "mesh-catalog", Base: "main", Paths: []string{"modules/gitea/index.ts"}, Beside: beside, - Modules: []string{"gitea"}, Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain}, - t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate.Verdict != "warning" || !strings.Contains(v.Report, "already") { - t.Fatalf("a fault the base already had failed the change: %+v\n%s", v.Gate, v.Report) - } - - // A delivery group (novox/hq ADR 0239): the other heads cloned beside it at their heads and handed to the - // gate to compose with this one; the repository's own check is each pull request's, not the group's. - app, appHead := aCheckedRepository(t, map[string]string{"module.json": `{"module":"app"}`}) - repo2, head2 := aCheckedRepository(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "exit 1\n"}) - gid := fmt.Sprintf("check-group-%d", time.Now().UnixNano()) - v, err = Check(t.Context(), Command, CheckSpec{ID: gid, Repository: repo2, Ref: head2, Owner: "novox", - Repo: "mesh-catalog", Paths: []string{"modules/gitea/module.json"}, Beside: beside, Modules: []string{"gitea"}, - Manifests: []string{"modules/gitea/module.json"}, Toolchain: goToolchain, - Group: []GroupHead{{Owner: "novox", Repo: "app", Repository: app, Ref: appHead, Paths: []string{"module.json"}}}}, - t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate.Verdict != "pass" || v.Gate.Summary != "composed with novox/app" { - t.Fatalf("the group's other head was not composed: %+v\n%s", v.Gate, v.Report) - } - if v.Repo == nil || v.Repo.Verdict != "pass" || !strings.Contains(v.Repo.Summary, "group") { - t.Fatalf("a group's check ran a member's own script: %+v", v.Repo) - } - - // A change to the controller judges itself: one that does not build fails its own gate. - v = check(map[string]string{"go.mod": "module x\n\ngo 1.22\n", "cmd/mesh-controller/main.go": "package main\nfunc main() { nope }\n", - "modules/gitea/module.json": "{}"}, "self") - if v.Gate.Verdict != "fail" || !strings.Contains(v.Gate.Summary, "does not build") { - t.Fatalf("a controller that does not build judged itself %+v", v.Gate) - } -} - // **Only a commit on the trunk is published** (novox/hq ADR 0238): the build seat reads, from the clone it // just made, the branch the forge names its default and whether the commit built is on it. -func TestABuildSaysWhetherItsCommitIsOnTheTrunk(t *testing.T) { - repo, onMain := aCheckedRepository(t, map[string]string{"module.json": `{"module":"x","version":"1"}`}) - git := func(dir string, args ...string) string { - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.org", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.org") - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git(repo, "checkout", "--quiet", "-b", "feature") - if err := os.WriteFile(filepath.Join(repo, "x"), []byte("x"), 0o644); err != nil { - t.Fatal(err) - } - git(repo, "add", "-A") - git(repo, "commit", "--quiet", "-m", "off the trunk") - offMain := git(repo, "rev-parse", "HEAD") - git(repo, "checkout", "--quiet", "main") - - clone := filepath.Join(t.TempDir(), "clone") - git(filepath.Dir(clone), "clone", "--quiet", repo, clone) - if trunk, on := trunkOf(t.Context(), Command, clone, onMain); trunk != "main" || !on { - t.Errorf("a commit on main reads as on %q: %v", trunk, on) - } - if trunk, on := trunkOf(t.Context(), Command, clone, offMain); trunk != "main" || on { - t.Errorf("a feature branch's commit reads as on %q: %v", trunk, on) - } - if got := strings.Join(branchesHolding(t.Context(), Command, clone, offMain), ","); got != "feature" { - t.Errorf("the feature branch's commit is said to be on %q", got) - } - if got := strings.Join(branchesHolding(t.Context(), Command, clone, onMain), ","); got != "feature,main" { - t.Errorf("main's commit is said to be on %q", got) - } - // A tree that says no trunk is not known — never read as on it. - if trunk, on := trunkOf(t.Context(), Command, repo, onMain); trunk != "" || on { - t.Errorf("a repository with no origin reads as trunk %q, on %v", trunk, on) - } -} // **Issue 286**: a failed merge-check.sh was said by its last line — a bare "FAIL", or the name of a file // gofmt listed — which named nothing a reader could act on. The status says what failed. @@ -497,99 +202,12 @@ func TestTheSeatCallsAGateThatRaisedNoMachineAnError(t *testing.T) { // **Issue 285**: an ask redelivered after its holder stopped mid-check found its own throwaway store still // there under its name, and the check said it could not run. What an earlier delivery left goes first. -func TestARedeliveredCheckRemovesWhatItsEarlierDeliveryLeft(t *testing.T) { - registry := checkEnvironment(t) - repo, head := aCheckedRepository(t, map[string]string{CheckScript: "echo checked\n"}) - id := fmt.Sprintf("check-again-%d", time.Now().UnixNano()) - if out, err := exec.Command("docker", "run", "-d", "--rm", "--label", BuildLabel+"="+id, "--name", id+"-store", - "postgres:17-alpine", "sleep", "300").CombinedOutput(); err != nil { - t.Skipf("no container for the earlier delivery: %v %s", err, out) - } - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: repo, Ref: head, Owner: "novox", - Repo: "hq", Number: 7, Toolchain: goToolchain}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatalf("a redelivered check could not run: %v", err) - } - if v.Repo == nil || v.Repo.Verdict != "pass" { - t.Errorf("the repository's check answered %+v", v.Repo) - } - if left := labelled(id); len(left) > 0 { - t.Errorf("the check left %d container(s) behind", len(left)) - } -} // **A repository in two languages checks both, each in its own toolchain** (novox/hq issue 302): the lab's // merge-check.sh runs in the TypeScript toolchain, and its Go replays were said "NOT CHECKED HERE" on every // pull request, so a register that did not compile would have merged green. Its script declares the Go // part; the check runs it in the Go toolchain's container after the script, and the layer passes only // when both do. -func TestARepositoryInTwoLanguagesIsCheckedInBoth(t *testing.T) { - script := "#!/bin/sh\n# mesh-check-toolchain: typescript\n# mesh-check-also: go replays/merge-check.sh\nnpm test\n" - parts := ScriptParts([]byte(script)) - if len(parts) != 2 || parts[0] != (ScriptPart{"typescript", CheckScript}) || - parts[1] != (ScriptPart{"go", "replays/merge-check.sh"}) { - t.Fatalf("the parts read as %+v", parts) - } - if got := ScriptParts([]byte("#!/bin/sh\nset -eu\n")); len(got) != 1 || got[0] != (ScriptPart{"go", CheckScript}) { - t.Fatalf("a script declaring nothing reads as %+v", got) - } - - // Each part run where its toolchain is: here, the image the part was given is what the command says. - held := CheckSpec{Toolchain: "go-image", Toolchains: map[string]string{"typescript": "ts-image", "go": "go-image"}} - run := func(t *testing.T, spec CheckSpec, files map[string]string, parts []ScriptPart) (*Layer, []string) { - t.Helper() - tree := t.TempDir() - for name, body := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(tree, name)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tree, name), []byte(body), 0o755); err != nil { - t.Fatal(err) - } - } - var images []string - var out tail - layer := ownCheck(t.Context(), spec, parts, tree, &out, func() bool { return false }, - func(image, script string) *exec.Cmd { - images = append(images, image+" "+script) - cmd := exec.CommandContext(t.Context(), "sh", script) - cmd.Dir = tree - return cmd - }, func(string, string, ...any) {}) - return layer, images - } - twoParts := []ScriptPart{{"typescript", CheckScript}, {"go", "replays/merge-check.sh"}} - - layer, images := run(t, held, map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) - if layer.Verdict != "pass" || !strings.Contains(layer.Summary, "replays/merge-check.sh (go)") || - strings.Join(images, ", ") != "ts-image merge-check.sh, go-image replays/merge-check.sh" { - t.Errorf("both parts passing answered %+v, ran %v", layer, images) - } - layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n", - "replays/merge-check.sh": "echo 'not gofmt'\"'\"'d:'; echo register.go; exit 1\n"}, twoParts) - if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "replays/merge-check.sh failed") || - !strings.Contains(layer.Summary, "register.go") || !strings.Contains(layer.Summary, "merge-check.sh (typescript) passed") { - t.Errorf("a failing Go part answered %+v", layer) - } - layer, images = run(t, held, map[string]string{CheckScript: "exit 2\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) - if layer.Verdict != "fail" || len(images) != 1 { - t.Errorf("a failing first part answered %+v and ran %v", layer, images) - } - layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, twoParts) - if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "does not hold") { - t.Errorf("a declared part the repository lacks answered %+v", layer) - } - layer, _ = run(t, held, map[string]string{CheckScript: "exit 0\n"}, []ScriptPart{{"typescript", CheckScript}, - {"go", "../elsewhere.sh"}}) - if layer.Verdict != "fail" || !strings.Contains(layer.Summary, "not a path inside") { - t.Errorf("a part outside the repository answered %+v", layer) - } - layer, _ = run(t, CheckSpec{Toolchains: map[string]string{"typescript": "ts-image"}}, - map[string]string{CheckScript: "exit 0\n", "replays/merge-check.sh": "exit 0\n"}, twoParts) - if layer.Verdict != "error" || !strings.Contains(layer.Summary, "go toolchain") { - t.Errorf("a part in a toolchain the mesh does not hold answered %+v — never a pass", layer) - } -} // A branch tried before its pull request may narrow what the repository's script runs (novox/hq issue 686); a // pull request's check is handed nothing, and runs whole. diff --git a/internal/builder/dependencies_test.go b/internal/builder/dependencies_test.go index fbcade29..c70c77bc 100644 --- a/internal/builder/dependencies_test.go +++ b/internal/builder/dependencies_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/failures_integration_test.go b/internal/builder/failures_integration_test.go new file mode 100644 index 00000000..b714f1bb --- /dev/null +++ b/internal/builder/failures_integration_test.go @@ -0,0 +1,152 @@ +//go:build integration + +package builder + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func ownCheckOf(t *testing.T, printed string) (*Layer, []string) { + t.Helper() + tree := t.TempDir() + if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + var out tail + var logged []string + layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out, + func() bool { return false }, + func(_, script string) *exec.Cmd { + cmd := exec.CommandContext(t.Context(), "sh", script) + cmd.Dir = tree + return cmd + }, func(step, format string, args ...any) { + if step == "output" { + logged = append(logged, fmt.Sprintf(format, args...)) + } + }) + if layer == nil || layer.Verdict != "fail" { + t.Fatalf("a failing check answered %+v", layer) + } + return layer, logged +} + +func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") { + t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary) + } + if !strings.Contains(layer.Summary, "TestSubtests/the_hub") { + t.Errorf("the second failing test is not named: %q", layer.Summary) + } +} + +func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt")) + if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") || + !strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") { + t.Errorf("a run failing in four packages is said as %q", layer.Summary) + } +} + +func TestABuildErrorIsNamedByTheError(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt")) + if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") { + t.Errorf("a build error is said as %q", layer.Summary) + } +} + +func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) { + printed := captured(t, "a-long-run-failing-early.txt") + _, logged := ownCheckOf(t, printed) + want := strings.Split(strings.TrimRight(printed, "\n"), "\n") + if len(logged) != len(want) { + t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want)) + } + for i := range want { + if logged[i] != want[i] { + t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i]) + } + } +} + +func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) { + layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) + want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" + + " early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" + + " early_test.go:10: a second line of the same failure\n" + + "--- FAIL: TestSubtests (0.00s)\n" + + " --- FAIL: TestSubtests/the_hub (0.00s)\n" + + " early_test.go:14: the hub does not compose\n" + + "FAIL\texample.com/cap/early\t" + if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") { + t.Errorf("the early failure is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-panic.txt")) + for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map", + "[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7", + "FAIL\texample.com/cap/panics\t"} { + if !strings.Contains(layer.Failed, line) { + t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed) + } + } + + layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt")) + race := captured(t, "a-data-race.txt") + report := race[:strings.Index(race, "--- FAIL:")] + if !strings.HasPrefix(layer.Failed, report) || + !strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") { + t.Errorf("a data race is kept as:\n%s", layer.Failed) + } + + layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt")) + if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+ + "FAIL\texample.com/cap/broken [build failed]" { + t.Errorf("a build error is kept as:\n%s", layer.Failed) + } +} + +func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) { + var b strings.Builder + for i := range 3000 { + fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600)) + fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i) + } + b.WriteString("FAIL\tx/everything\t1s\nFAIL\n") + layer, logged := ownCheckOf(t, b.String()) + if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") { + t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed), + layer.Failed[max(0, len(layer.Failed)-200):]) + } + if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") || + !strings.Contains(layer.Summary, "and 2994 more") { + t.Errorf("a run failing 3000 tests is said as %q", layer.Summary) + } + + var out tail + out.Write([]byte(b.String())) + report := withWhatFailed(out.String(), layer) + const maxCheckReport = 60 << 10 // as the controller cuts it, from the front + if len(report) > maxCheckReport { + report = report[len(report)-maxCheckReport:] + } + if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") { + t.Error("what failed did not survive the controller's cut of the report") + } + + // The build's log keeps its first logLines lines, says how many it left out, then what failed. + if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" || + logged[logLines+1] != "--- what failed, picked from the whole of its output" || + logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" { + t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)]) + } +} diff --git a/internal/builder/failures_test.go b/internal/builder/failures_test.go index e6278dfa..65a81beb 100644 --- a/internal/builder/failures_test.go +++ b/internal/builder/failures_test.go @@ -1,43 +1,13 @@ package builder import ( - "fmt" "os" - "os/exec" "path/filepath" - "strings" "testing" ) // ownCheckOf runs a repository's own check whose script prints a captured output and fails, as the build // seat runs it, and answers the layer and every line it said in the build's log. -func ownCheckOf(t *testing.T, printed string) (*Layer, []string) { - t.Helper() - tree := t.TempDir() - if err := os.WriteFile(filepath.Join(tree, "printed.txt"), []byte(printed), 0o644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(tree, CheckScript), []byte("cat printed.txt\nexit 1\n"), 0o755); err != nil { - t.Fatal(err) - } - var out tail - var logged []string - layer := ownCheck(t.Context(), CheckSpec{Toolchain: "go-image"}, []ScriptPart{{"go", CheckScript}}, tree, &out, - func() bool { return false }, - func(_, script string) *exec.Cmd { - cmd := exec.CommandContext(t.Context(), "sh", script) - cmd.Dir = tree - return cmd - }, func(step, format string, args ...any) { - if step == "output" { - logged = append(logged, fmt.Sprintf(format, args...)) - } - }) - if layer == nil || layer.Verdict != "fail" { - t.Fatalf("a failing check answered %+v", layer) - } - return layer, logged -} func captured(t *testing.T, name string) string { t.Helper() @@ -52,124 +22,18 @@ func captured(t *testing.T, name string) string { // name: the package's own logs, printed after its `--- FAIL:` lines, pushed them out of the last 200 lines, // which was all the summary was named from. The first failing test is named, and the others after it, // however much came after them. -func TestAFailureEarlyInALongRunIsStillNamed(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) - if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") { - t.Errorf("a failure 600 lines from the end is said as %q", layer.Summary) - } - if !strings.Contains(layer.Summary, "TestSubtests/the_hub") { - t.Errorf("the second failing test is not named: %q", layer.Summary) - } -} // One -race run of every package: the first failure is the earliest, not the last within the tail. -func TestARaceRunOfEveryPackageNamesItsFirstFailure(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-race-run-of-every-package.txt")) - if !strings.Contains(layer.Summary, "failed: --- FAIL: TestTheEnvelopeIsPinned (0.00s)") || - !strings.Contains(layer.Summary, "TestAMapIsNil") || !strings.Contains(layer.Summary, "TestACounterIsShared") { - t.Errorf("a run failing in four packages is said as %q", layer.Summary) - } -} // A package that does not build is named by its error, not by its bare "[build failed]". -func TestABuildErrorIsNamedByTheError(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-build-error.txt")) - if !strings.HasSuffix(layer.Summary, "failed: broken/broken.go:3:28: undefined: undefinedThing") { - t.Errorf("a build error is said as %q", layer.Summary) - } -} // The whole of what the check printed is in the build's log, line by line — where before none of it was. -func TestTheWholeOutputIsInTheBuildsLog(t *testing.T) { - printed := captured(t, "a-long-run-failing-early.txt") - _, logged := ownCheckOf(t, printed) - want := strings.Split(strings.TrimRight(printed, "\n"), "\n") - if len(logged) != len(want) { - t.Fatalf("the build's log holds %d line(s) of the %d printed", len(logged), len(want)) - } - for i := range want { - if logged[i] != want[i] { - t.Fatalf("line %d is logged as %q, printed as %q", i+1, logged[i], want[i]) - } - } -} // What failed is kept whole in the verdict: each --- FAIL with its messages, a panic with its first frames, // a data race report from rule to rule — and none of the log lines around them. -func TestTheVerdictKeepsWhatFailedWhole(t *testing.T) { - layer, _ := ownCheckOf(t, captured(t, "a-long-run-failing-early.txt")) - want := "--- FAIL: TestTheEnvelopeIsPinned (0.00s)\n" + - " early_test.go:9: the envelope's subject is \"mesh.a\", not \"mesh.b\"\n" + - " early_test.go:10: a second line of the same failure\n" + - "--- FAIL: TestSubtests (0.00s)\n" + - " --- FAIL: TestSubtests/the_hub (0.00s)\n" + - " early_test.go:14: the hub does not compose\n" + - "FAIL\texample.com/cap/early\t" - if !strings.HasPrefix(layer.Failed, want) || strings.Contains(layer.Failed, "kept what home-server says") { - t.Errorf("the early failure is kept as:\n%s", layer.Failed) - } - - layer, _ = ownCheckOf(t, captured(t, "a-panic.txt")) - for _, line := range []string{"--- FAIL: TestAMapIsNil (0.00s)", "panic: assignment to entry in nil map", - "[running]:", "example.com/cap/panics.TestAMapIsNil(", "/src/cap/panics/panics_test.go:7", - "FAIL\texample.com/cap/panics\t"} { - if !strings.Contains(layer.Failed, line) { - t.Errorf("a panic is kept without %q:\n%s", line, layer.Failed) - } - } - - layer, _ = ownCheckOf(t, captured(t, "a-data-race.txt")) - race := captured(t, "a-data-race.txt") - report := race[:strings.Index(race, "--- FAIL:")] - if !strings.HasPrefix(layer.Failed, report) || - !strings.Contains(layer.Failed, "--- FAIL: TestACounterIsShared (0.00s)\n testing.go:1865: race detected") { - t.Errorf("a data race is kept as:\n%s", layer.Failed) - } - - layer, _ = ownCheckOf(t, captured(t, "a-build-error.txt")) - if layer.Failed != "# example.com/cap/broken\nbroken/broken.go:3:28: undefined: undefinedThing\n"+ - "FAIL\texample.com/cap/broken [build failed]" { - t.Errorf("a build error is kept as:\n%s", layer.Failed) - } -} // A run that fails everywhere is bounded in what it keeps, and what it keeps survives the controller's cut // of a report to its last 60 KiB, said at the report's end. -func TestWhatFailedIsBoundedAndSurvivesTheReportsCut(t *testing.T) { - var b strings.Builder - for i := range 3000 { - fmt.Fprintf(&b, "--- FAIL: TestNumber%d (0.00s)\n x_test.go:1: %s\n", i, strings.Repeat("why ", 600)) - fmt.Fprintf(&b, "2026/10/11 01:30:03 a log line %d\n", i) - } - b.WriteString("FAIL\tx/everything\t1s\nFAIL\n") - layer, logged := ownCheckOf(t, b.String()) - if len(layer.Failed) > failureBytes+2*failureLineBytes || !strings.Contains(layer.Failed, "more line(s) naming what failed") { - t.Errorf("what failed in a run that fails everywhere is %d bytes, ending %q", len(layer.Failed), - layer.Failed[max(0, len(layer.Failed)-200):]) - } - if !strings.Contains(layer.Summary, "--- FAIL: TestNumber0 (0.00s); and TestNumber1, ") || - !strings.Contains(layer.Summary, "and 2994 more") { - t.Errorf("a run failing 3000 tests is said as %q", layer.Summary) - } - - var out tail - out.Write([]byte(b.String())) - report := withWhatFailed(out.String(), layer) - const maxCheckReport = 60 << 10 // as the controller cuts it, from the front - if len(report) > maxCheckReport { - report = report[len(report)-maxCheckReport:] - } - if !strings.Contains(report, "--- what failed") || !strings.Contains(report, "--- FAIL: TestNumber0 (0.00s)") { - t.Error("what failed did not survive the controller's cut of the report") - } - - // The build's log keeps its first logLines lines, says how many it left out, then what failed. - if len(logged) < logLines+2 || logged[logLines] != "… 1002 more line(s) of its output are not in this log, which keeps its first 8000" || - logged[logLines+1] != "--- what failed, picked from the whole of its output" || - logged[logLines+2] != "--- FAIL: TestNumber0 (0.00s)" { - t.Errorf("the log past its bound says %q", logged[logLines:min(len(logged), logLines+3)]) - } -} // A passing check says nothing of what failed. func TestAPassingCheckReportsAsBefore(t *testing.T) { diff --git a/internal/builder/gosource_integration_test.go b/internal/builder/gosource_integration_test.go new file mode 100644 index 00000000..ae57922d --- /dev/null +++ b/internal/builder/gosource_integration_test.go @@ -0,0 +1,84 @@ +//go:build integration + +package builder + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestAGoProgramsBuildSourceIsItsImportClosure(t *testing.T) { + root := aGoTree(t, aProgram) + got, err := GoBuildSource(root, "cmd/prog") + if err != nil { + t.Fatal(err) + } + for _, c := range []struct { + file string + held bool + why string + }{ + {"cmd/prog/main.go", true, "the program itself"}, + {"cmd/prog/helper.go", true, "a file added to the program's package"}, + {"lib/lib.go", true, "a package it imports"}, + {"emb/emb_amd64.s", true, "assembly beside a package's Go"}, + {"lib/lib_plan9.go", true, "a file one system builds"}, + {"plan9only/p.go", true, "what a file one system builds imports"}, + {"emb/static/index.html", true, "an embedded file"}, + {"emb/static/deep/style.css", true, "an embedded file below the pattern's directory"}, + {"emb/a b.txt", true, "an embed named outright, quoted"}, + {"vendor/example.org/dep/dep.go", true, "a vendored package it imports"}, + {"vendor/example.org/dep/sub/sub.go", true, "what a vendored package imports"}, + {"vendor/example.net/local/pkg/p.go", true, "a replaced module, as vendored"}, + {"third_party/local/pkg/p.go", true, "a replaced module, at its directory"}, + {"third_party/local/go.mod", true, "a replaced module's requirements"}, + {"go.mod", true, "the module's requirements"}, + {"go.sum", true, "the module's sums"}, + {"vendor/modules.txt", true, "the vendored modules"}, + {"go.work", true, "a workspace, were one made"}, + {"lib/lib_test.go", false, "a test is not built"}, + {"testonly/t.go", false, "a package only a test imports"}, + {"lib/sub/sub.go", false, "a package below an imported one, not imported"}, + {"unrelated/u.go", false, "a package nothing imports"}, + {"vendor/example.org/other/other.go", false, "a vendored package nothing imports"}, + {"README.md", false, "a file no build reads"}, + } { + if SourceHolds(got, c.file) != c.held { + t.Errorf("%s (%s): held %v, wanted %v\n %v", c.file, c.why, !c.held, c.held, got) + } + } + + // **Never narrower than go list -deps**: every package go names, and every file it compiles or embeds, + // is held. Where no go command is at hand, said, not passed. + goCmd, err := exec.LookPath("go") + if err != nil { + t.Skip("NOT COMPARED: no go command to list the closure with") + } + list := exec.Command(goCmd, "list", "-deps", "-f", + `{{if not .Standard}}{{$d := .Dir}}{{range .GoFiles}}{{$d}}/{{.}} +{{end}}{{range .SFiles}}{{$d}}/{{.}} +{{end}}{{range .EmbedFiles}}{{$d}}/{{.}} +{{end}}{{end}}`, "./cmd/prog") + list.Dir = root + list.Env = append(os.Environ(), "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOWORK=off", "GOOS=linux", "GOARCH=amd64") + out, err := list.CombinedOutput() + if err != nil { + t.Fatalf("go list: %v\n%s", err, out) + } + real, _ := filepath.EvalSymlinks(root) + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if line == "" { + continue + } + rel, err := filepath.Rel(real, line) + if err != nil || strings.HasPrefix(rel, "..") { + rel, _ = filepath.Rel(root, line) + } + if !SourceHolds(got, filepath.ToSlash(rel)) { + t.Errorf("go list builds %s, and the build source does not hold it", rel) + } + } +} diff --git a/internal/builder/gosource_test.go b/internal/builder/gosource_test.go index fd5aa2e6..80fbea1d 100644 --- a/internal/builder/gosource_test.go +++ b/internal/builder/gosource_test.go @@ -2,7 +2,6 @@ package builder import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -60,79 +59,6 @@ var aProgram = map[string]string{ "README.md": "x", } -func TestAGoProgramsBuildSourceIsItsImportClosure(t *testing.T) { - root := aGoTree(t, aProgram) - got, err := GoBuildSource(root, "cmd/prog") - if err != nil { - t.Fatal(err) - } - for _, c := range []struct { - file string - held bool - why string - }{ - {"cmd/prog/main.go", true, "the program itself"}, - {"cmd/prog/helper.go", true, "a file added to the program's package"}, - {"lib/lib.go", true, "a package it imports"}, - {"emb/emb_amd64.s", true, "assembly beside a package's Go"}, - {"lib/lib_plan9.go", true, "a file one system builds"}, - {"plan9only/p.go", true, "what a file one system builds imports"}, - {"emb/static/index.html", true, "an embedded file"}, - {"emb/static/deep/style.css", true, "an embedded file below the pattern's directory"}, - {"emb/a b.txt", true, "an embed named outright, quoted"}, - {"vendor/example.org/dep/dep.go", true, "a vendored package it imports"}, - {"vendor/example.org/dep/sub/sub.go", true, "what a vendored package imports"}, - {"vendor/example.net/local/pkg/p.go", true, "a replaced module, as vendored"}, - {"third_party/local/pkg/p.go", true, "a replaced module, at its directory"}, - {"third_party/local/go.mod", true, "a replaced module's requirements"}, - {"go.mod", true, "the module's requirements"}, - {"go.sum", true, "the module's sums"}, - {"vendor/modules.txt", true, "the vendored modules"}, - {"go.work", true, "a workspace, were one made"}, - {"lib/lib_test.go", false, "a test is not built"}, - {"testonly/t.go", false, "a package only a test imports"}, - {"lib/sub/sub.go", false, "a package below an imported one, not imported"}, - {"unrelated/u.go", false, "a package nothing imports"}, - {"vendor/example.org/other/other.go", false, "a vendored package nothing imports"}, - {"README.md", false, "a file no build reads"}, - } { - if SourceHolds(got, c.file) != c.held { - t.Errorf("%s (%s): held %v, wanted %v\n %v", c.file, c.why, !c.held, c.held, got) - } - } - - // **Never narrower than go list -deps**: every package go names, and every file it compiles or embeds, - // is held. Where no go command is at hand, said, not passed. - goCmd, err := exec.LookPath("go") - if err != nil { - t.Skip("NOT COMPARED: no go command to list the closure with") - } - list := exec.Command(goCmd, "list", "-deps", "-f", - `{{if not .Standard}}{{$d := .Dir}}{{range .GoFiles}}{{$d}}/{{.}} -{{end}}{{range .SFiles}}{{$d}}/{{.}} -{{end}}{{range .EmbedFiles}}{{$d}}/{{.}} -{{end}}{{end}}`, "./cmd/prog") - list.Dir = root - list.Env = append(os.Environ(), "GOFLAGS=-mod=vendor", "GOPROXY=off", "GOWORK=off", "GOOS=linux", "GOARCH=amd64") - out, err := list.CombinedOutput() - if err != nil { - t.Fatalf("go list: %v\n%s", err, out) - } - real, _ := filepath.EvalSymlinks(root) - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if line == "" { - continue - } - rel, err := filepath.Rel(real, line) - if err != nil || strings.HasPrefix(rel, "..") { - rel, _ = filepath.Rel(root, line) - } - if !SourceHolds(got, filepath.ToSlash(rel)) { - t.Errorf("go list builds %s, and the build source does not hold it", rel) - } - } -} - // A file added to the program's import closure moves its build source with it: the closure read again // grows by the package. func TestTheBuildSourceGrowsWithAnImport(t *testing.T) { diff --git a/internal/builder/release_branch_test.go b/internal/builder/judge_build_test.go similarity index 63% rename from internal/builder/release_branch_test.go rename to internal/builder/judge_build_test.go index a0adb5ae..63a3c896 100644 --- a/internal/builder/release_branch_test.go +++ b/internal/builder/judge_build_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( @@ -10,10 +12,7 @@ import ( "testing" ) -// **A pull request into a release branch is judged by the controller of that branch** (novox/hq ADR 0358, issue -// 704): the build seat's check builds the judge from mesh-controller read at the release branch — with mesh-host's -// validator of that branch when mesh-host has one — reads every repository beside it at that branch where it has one -// and at its trunk otherwise, and names the branch, the judge's commit and each repository read with its commit. +// Helpers of the judge-build tests: repositories with a branch, a fake container runtime, and the planted workspace. const releaseBranch = "feat/57-config-and-provisioning" @@ -193,10 +192,10 @@ func theReleasedMesh(t *testing.T) (map[string]Beside, string, string, string) { aHostSaying("the release branch's validator")) lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{"README": "the lab"}, nil) return map[string]Beside{ - "mesh-controller": {Repository: controller, Ref: releaseBranch}, + "mesh-controller": {Repository: controller, Ref: "main"}, "mesh-controller-main": {Repository: controller, Ref: "main"}, - "mesh-host": {Repository: host, Ref: releaseBranch, Else: "main"}, - "mesh-lab": {Repository: lab, Ref: releaseBranch, Else: "main"}, + "mesh-host": {Repository: host, Ref: "main"}, + "mesh-lab": {Repository: lab, Ref: "main"}, }, controllerRelease, hostRelease, labMain } @@ -210,230 +209,6 @@ func aPullRequestInto(t *testing.T, files map[string]string, base string, change return repo, head } -func TestACataloguePullRequestIntoAReleaseBranchIsJudgedByThatBranchsController(t *testing.T) { - built := aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - beside, controllerRelease, hostRelease, labMain := theReleasedMesh(t) - catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"}) - cache := t.TempDir() - check := func(id string) CheckVerdict { - t.Helper() - v, err := Check(t.Context(), Command, CheckSpec{ID: id, Repository: catalogue, Ref: head, Owner: "novox", - Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, JudgeCache: cache, - Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, Toolchain: "golang"}, - t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - return v - } - v := check("check-704-catalogue") - // The release branch's controller judged, with mesh-host's validator of the release branch. - if v.Gate == nil || v.Gate.Verdict != "pass" || - !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the release branch's validator") { - t.Fatalf("the gate was not judged by the release branch's controller with its node-engine's validator: %+v\n%s", - v.Gate, v.Report) - } - // It names the release branch, the judge's commit, and each repository read beside it at its branch and commit. - mark := "[release " + releaseBranch + ", judged by its controller " + controllerRelease[:12] + "] " - if !strings.HasPrefix(v.Gate.Summary, mark) || !strings.HasPrefix(v.Repo.Summary, mark) { - t.Fatalf("the statuses do not name the release branch and its judge first:\n%s\n%s", v.Gate.Summary, v.Repo.Summary) - } - if v.Release != releaseBranch || v.JudgedBy != controllerRelease[:12] { - t.Fatalf("the verdict names release %q judged by %q, not %s by %s", v.Release, v.JudgedBy, releaseBranch, - controllerRelease[:12]) - } - if v.BesideAt["mesh-host"] != releaseBranch || v.Beside["mesh-host"] != hostRelease[:12] { - t.Fatalf("mesh-host was read at %q (%s), not its release branch at %s", v.BesideAt["mesh-host"], - v.Beside["mesh-host"], hostRelease[:12]) - } - // A repository with no release branch is read at its trunk, and says so. - if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] { - t.Fatalf("the lab, with no release branch, was read at %q (%s), not main at %s", v.BesideAt["mesh-lab"], - v.Beside["mesh-lab"], labMain[:12]) - } - if !strings.Contains(v.Report, "read beside it: ") || !strings.Contains(v.Report, "mesh-host "+releaseBranch+" at "+hostRelease[:12]) { - t.Fatalf("the report does not say what was read beside it:\n%s", v.Report) - } - // It says which validator judged: mesh-host's of the release branch, with its commit. - if want := "mesh-host's validator of " + releaseBranch + " at " + hostRelease[:12]; v.Validator != want || - !strings.Contains(v.Report, want) { - t.Fatalf("the verdict says the validator was %q, not %q:\n%s", v.Validator, want, v.Report) - } - // Each repository read beside it is named with the branch and the commit read. - for dir := range beside { - if v.BesideAt[dir] == "" || v.Beside[dir] == "" || - !strings.Contains(v.Report, dir+" "+v.BesideAt[dir]+" at "+v.Beside[dir]) { - t.Fatalf("%s is not named with its branch and commit: at %q, commit %q\n%s", dir, v.BesideAt[dir], - v.Beside[dir], v.Report) - } - } - if n := builds(t, built); n != 1 { - t.Fatalf("the judge was built %d times for one check", n) - } - // **The judge is built with the judge cache's Go cache** (ADR 0358 §6), which no check's container mounts, and no - // Go environment file. - said, _ := os.ReadFile(built) - if !strings.Contains(string(said), "GOCACHE="+filepath.Join(cache, "go-cache")+" GOENV=off go build") { - t.Fatalf("the release judge was not built with the judge cache's Go cache:\n%s", said) - } - // **Clause 5**: the head was merged onto the release branch's head, which the verdict names as its trunk commit, - // so the release branch's move puts it out of date as a trunk's move does (ADR 0323 §4, the forge's holder). - if releaseHead := gitInRepo(t, catalogue, "rev-parse", releaseBranch); v.Trunk != releaseHead[:12] { - t.Fatalf("the verdict names trunk %q, not the release branch's head %s", v.Trunk, releaseHead[:12]) - } - - // **Built once per release-branch head**: a second check while that head stands takes the judge kept. - v = check("check-704-catalogue-again") - if !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the release branch's validator") { - t.Fatalf("the kept judge did not judge: %+v\n%s", v.Gate, v.Report) - } - if n := builds(t, built); n != 1 { - t.Fatalf("the judge of the same release-branch head was built again: %d builds", n) - } -} - -func TestAControllerPullRequestIntoAReleaseBranchReadsTheNodeEngineAtThatBranch(t *testing.T) { - aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - beside, _, hostRelease, _ := theReleasedMesh(t) - delete(beside, "mesh-controller") - controller, head := aPullRequestInto(t, aJudgeOf("the change's controller"), releaseBranch, - map[string]string{"cmd/x.txt": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-controller", Repository: controller, Ref: head, - Owner: "novox", Repo: "mesh-controller", Number: 315, Base: releaseBranch, Release: releaseBranch, - JudgeCache: t.TempDir(), Paths: []string{"cmd/x.txt"}, Modules: []string{"mesh-controller"}, Judge: "self", - Beside: beside, Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the change's controller with the vendored validator") { - t.Fatalf("a controller change into a release branch did not judge itself: %+v\n%s", v.Gate, v.Report) - } - if v.BesideAt["mesh-host"] != releaseBranch || v.Beside["mesh-host"] != hostRelease[:12] { - t.Fatalf("mesh-host was read beside it at %q (%s), not at the release branch's %s", v.BesideAt["mesh-host"], - v.Beside["mesh-host"], hostRelease[:12]) - } - if v.Release != releaseBranch || v.JudgedBy == "" || v.JudgedBy != v.Merge[:12] { - t.Fatalf("the verdict names release %q judged by %q, not its own merge %s", v.Release, v.JudgedBy, v.Merge) - } -} - -func TestAPullRequestIntoTheTrunkIsJudgedAsBefore(t *testing.T) { - aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("main's controller"), - aJudgeOf("the release branch's controller")) - host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"), - aHostSaying("the release branch's validator")) - catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "true\n"}, "main", map[string]string{"modules/gitea/index.ts": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-trunk", Repository: catalogue, Ref: head, - Owner: "novox", Repo: "mesh-catalog", Number: 1, Base: "main", JudgeCache: t.TempDir(), - Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang", - Beside: map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain}, - "mesh-host": {Repository: host, Ref: hostMain}}}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || v.Gate.Summary != "main's controller with the vendored validator" { - t.Fatalf("a trunk pull request was not judged by the controller beside it as before: %+v\n%s", v.Gate, v.Report) - } - if v.Release != "" || v.JudgedBy != "" || v.BesideAt != nil || strings.Contains(v.Report, "release branch") { - t.Fatalf("a trunk pull request's verdict speaks of a release branch: %q %q %v", v.Release, v.JudgedBy, v.BesideAt) - } - if v.Beside["mesh-host"] != hostMain[:12] { - t.Fatalf("mesh-host was read at %s, not the commit asked %s", v.Beside["mesh-host"], hostMain[:12]) - } -} - -// **Without mesh-host's release branch, the release branch's controller keeps the validator it vendors** (ADR 0358 -// §2), and the verdict says so. -func TestWithoutTheNodeEnginesReleaseBranchTheJudgeKeepsItsVendoredValidator(t *testing.T) { - aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - beside, controllerRelease, _, _ := theReleasedMesh(t) - host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"), nil) - beside["mesh-host"] = Beside{Repository: host, Ref: releaseBranch, Else: "main"} - catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-vendored", Repository: catalogue, Ref: head, - Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, - JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, - Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with the vendored validator") { - t.Fatalf("without mesh-host's release branch the judge did not keep its vendored validator: %+v\n%s", v.Gate, v.Report) - } - if v.JudgedBy != controllerRelease[:12] || v.BesideAt["mesh-host"] != "main" || v.Beside["mesh-host"] != hostMain[:12] { - t.Fatalf("judged by %q, mesh-host read at %q (%s)", v.JudgedBy, v.BesideAt["mesh-host"], v.Beside["mesh-host"]) - } - if !strings.Contains(v.Validator, "controller vendors") || !strings.Contains(v.Report, v.Validator) { - t.Fatalf("the verdict does not say the vendored validator judged: %q\n%s", v.Validator, v.Report) - } -} - -// **Clause 7** (ADR 0358 §7): a release branch's own pull request into its trunk is checked as any pull request into a -// trunk — by the controller beside it as the mesh runs it, the repositories at what a trunk check reads — and its -// verdict names no release branch. -func TestAReleaseBranchsPullRequestIntoItsTrunkIsJudgedByTheControllerTheMeshRuns(t *testing.T) { - aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - controller, controllerMain, _ := aRepositoryWithARelease(t, aJudgeOf("the controller the mesh runs"), - aJudgeOf("the release branch's controller")) - host, hostMain, _ := aRepositoryWithARelease(t, aHostSaying("main's validator"), - aHostSaying("the release branch's validator")) - catalogue, _, releaseHead := aRepositoryWithARelease(t, map[string]string{ - "modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "true\n"}, - map[string]string{"modules/gitea/index.ts": "released"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-release-into-trunk", Repository: catalogue, - Ref: releaseHead, Owner: "novox", Repo: "mesh-catalog", Number: 300, Base: "main", JudgeCache: t.TempDir(), - Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Toolchain: "golang", - Beside: map[string]Beside{"mesh-controller": {Repository: controller, Ref: controllerMain}, - "mesh-host": {Repository: host, Ref: hostMain}}}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.Gate == nil || v.Gate.Summary != "the controller the mesh runs with the vendored validator" { - t.Fatalf("the release branch's pull request into its trunk was not judged by the controller the mesh runs: %+v\n%s", - v.Gate, v.Report) - } - if v.Release != "" || v.JudgedBy != "" || v.BesideAt != nil || v.Validator != "" || - strings.Contains(v.Report, "release branch") || strings.Contains(v.Gate.Summary, "[release") { - t.Fatalf("its verdict names a release branch: %q %q %v %q", v.Release, v.JudgedBy, v.BesideAt, v.Validator) - } - if v.Beside["mesh-host"] != hostMain[:12] || v.Beside["mesh-controller"] != controllerMain[:12] { - t.Fatalf("beside it were read %v, not the trunk's %s and %s", v.Beside, controllerMain[:12], hostMain[:12]) - } -} - -// **mesh-lab is read at its trunk, always** (ADR 0358): its replays run with the container runtime's socket, so a -// branch of the release's name in mesh-lab — which anyone may create — is never what a check reads. -func TestAReleaseBranchCheckReadsMeshLabAtItsTrunkEvenWhenItHasThatBranch(t *testing.T) { - aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - beside, _, _, _ := theReleasedMesh(t) - lab, labMain, labRelease := aRepositoryWithARelease(t, map[string]string{"README": "the lab"}, - map[string]string{"README": "a branch anyone could push"}) - beside["mesh-lab"] = Beside{Repository: lab, Ref: releaseBranch, Else: "main"} - catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-lab", Repository: catalogue, Ref: head, - Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, - JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, - Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] || v.Beside["mesh-lab"] == labRelease[:12] { - t.Fatalf("mesh-lab was read at %q (%s), not its trunk main at %s", v.BesideAt["mesh-lab"], v.Beside["mesh-lab"], - labMain[:12]) - } -} - // aPlantedWorkspace is a workspace a pull request's own script has written into, as it can: a git configuration in its // HOME whose fsmonitor hook says each directory git runs in, a git configuration under XDG_CONFIG_HOME doing the same, // and a Go environment file that would hand the judge's build a tool of its own. It answers the workspace and the @@ -475,8 +250,8 @@ func judgeRanTheHook(t *testing.T, ran string) string { // **Every judge is built where no pull request reaches** (issue 482, ADR 0358 review): a git configuration with a // hook, and a Go environment file, planted in the workspace by an earlier pull request's script, do not run during -// the release judge's build — while the pull request's own script, in the workspace, still meets them. -func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheReleaseJudgesBuild(t *testing.T) { +// the judge's build — while the pull request's own script, in the workspace, still meets them. +func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheJudgesBuild(t *testing.T) { aRuntimeThatBuildsTheJudge(t) homes := filepath.Join(t.TempDir(), "homes") t.Setenv("JUDGE_HOMES", homes) @@ -486,7 +261,7 @@ func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheReleaseJudgesBuild(t *test catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, CheckScript: "git status >/dev/null 2>&1; true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"}) v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-planted", Repository: catalogue, Ref: head, - Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, + Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, Toolchain: "golang"}, workspace, registry, GitCredential{}, nil) if err != nil { @@ -506,7 +281,7 @@ func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheReleaseJudgesBuild(t *test } } -// The same for the judge built with a node-engine change's own validator, into a release branch (ADR 0358 §2). +// The same for the judge built with a node-engine change's own validator. func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheValidatorJudgesBuild(t *testing.T) { aRuntimeThatBuildsTheJudge(t) homes := filepath.Join(t.TempDir(), "homes") @@ -520,14 +295,14 @@ func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheValidatorJudgesBuild(t *te files[CheckScript] = "git status >/dev/null 2>&1; true\n" host, head := aPullRequestInto(t, files, releaseBranch, aHostSaying("this change's validator")) v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-planted-validator", Repository: host, Ref: head, - Owner: "novox", Repo: "mesh-host", Number: 111, Base: releaseBranch, Release: releaseBranch, Judge: "validator", + Owner: "novox", Repo: "mesh-host", Number: 111, Base: releaseBranch, Judge: "validator", JudgeCache: t.TempDir(), Paths: []string{"validate/v.go"}, Modules: []string{"mesh-host"}, Beside: beside, Toolchain: "golang"}, workspace, registry, GitCredential{}, nil) if err != nil { t.Fatal(err) } - if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "the release branch's controller with this change's validator") { - t.Fatalf("a node-engine change into a release branch was not judged by that branch's controller with its own "+ + if v.Gate == nil || !strings.HasSuffix(v.Gate.Summary, "controller with this change's validator") { + t.Fatalf("a node-engine change was not judged by the controller with its own "+ "validator: %+v\n%s", v.Gate, v.Report) } if body, _ := os.ReadFile(ran); !strings.Contains(string(body), "/check/mesh-host") { @@ -541,31 +316,6 @@ func TestAHookPlantedInTheWorkspaceDoesNotRunDuringTheValidatorJudgesBuild(t *te } } -// **mesh-lab is never read at a release branch**, asked with a fallback or without (ADR 0358). -func TestTheBuildSeatNeverReadsMeshLabAtAReleaseBranch(t *testing.T) { - aRuntimeThatBuildsTheJudge(t) - registry := aFactsRegistry(t) - lab, labMain, _ := aRepositoryWithARelease(t, map[string]string{"README": "the lab"}, - map[string]string{"README": "a branch anyone could push"}) - for _, asked := range []Beside{{Repository: lab, Ref: releaseBranch}, {Repository: lab, Ref: releaseBranch, Else: "main"}} { - beside, _, _, _ := theReleasedMesh(t) - beside["mesh-lab"] = asked - catalogue, head := aPullRequestInto(t, map[string]string{"modules/gitea/module.json": `{"module":"gitea"}`, - CheckScript: "true\n"}, releaseBranch, map[string]string{"modules/gitea/index.ts": "x"}) - v, err := Check(t.Context(), Command, CheckSpec{ID: "check-704-lab-asked", Repository: catalogue, Ref: head, - Owner: "novox", Repo: "mesh-catalog", Number: 286, Base: releaseBranch, Release: releaseBranch, - JudgeCache: t.TempDir(), Paths: []string{"modules/gitea/index.ts"}, Modules: []string{"gitea"}, Beside: beside, - Toolchain: "golang"}, t.TempDir(), registry, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if v.BesideAt["mesh-lab"] != "main" || v.Beside["mesh-lab"] != labMain[:12] { - t.Fatalf("mesh-lab asked as %+v was read at %q (%s), not main at %s", asked, v.BesideAt["mesh-lab"], - v.Beside["mesh-lab"], labMain[:12]) - } - } -} - // **Judges are built warm and where no pull request reaches** (ADR 0358 §6): every judge build uses the judge cache's // one Go cache, kept between checks; its home is made fresh under the judge cache and removed after; nothing is left in // a temporary directory of the host; and no container a check runs for the pull request mounts the judge cache. diff --git a/internal/builder/kill_integration_test.go b/internal/builder/kill_integration_test.go new file mode 100644 index 00000000..ee2fd00a --- /dev/null +++ b/internal/builder/kill_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package builder + +import ( + "context" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) { + dir := t.TempDir() + child := filepath.Join(dir, "child") + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + began := time.Now() + go func() { + // A shell that starts a grandchild and waits on it: killing the shell alone would leave the + // grandchild running, holding the output open. + _, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`) + done <- err + }() + var pid int + for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" { + pid, _ = strconv.Atoi(strings.TrimSpace(string(raw))) + break + } + } + if pid == 0 { + t.Fatal("the command never started its child") + } + cancel() + select { + case err := <-done: + if err == nil { + t.Fatal("a killed command reported success") + } + case <-time.After(KillWait + 5*time.Second): + t.Fatal("the killed command never returned") + } + if took := time.Since(began); took > KillWait { + t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took) + } + for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { + if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { + return + } + } + _ = syscall.Kill(pid, syscall.SIGKILL) + t.Fatalf("the grandchild %d outlived the kill", pid) +} diff --git a/internal/builder/kill_test.go b/internal/builder/kill_test.go index 30b63c9b..b3779f65 100644 --- a/internal/builder/kill_test.go +++ b/internal/builder/kill_test.go @@ -2,61 +2,13 @@ package builder import ( "context" - "errors" - "os" - "path/filepath" "reflect" - "strconv" "strings" - "syscall" "testing" - "time" ) // A build killed by hand (novox/hq ADR 0219) ends every command it started: the context's end kills // the command's whole process group, not the one process the context knows about. -func TestAKilledBuildEndsItsWholeProcessGroup(t *testing.T) { - dir := t.TempDir() - child := filepath.Join(dir, "child") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - began := time.Now() - go func() { - // A shell that starts a grandchild and waits on it: killing the shell alone would leave the - // grandchild running, holding the output open. - _, err := Command(ctx, dir, "sh", "-c", `sleep 60 & echo $! > child; wait`) - done <- err - }() - var pid int - for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { - if raw, err := os.ReadFile(child); err == nil && strings.TrimSpace(string(raw)) != "" { - pid, _ = strconv.Atoi(strings.TrimSpace(string(raw))) - break - } - } - if pid == 0 { - t.Fatal("the command never started its child") - } - cancel() - select { - case err := <-done: - if err == nil { - t.Fatal("a killed command reported success") - } - case <-time.After(KillWait + 5*time.Second): - t.Fatal("the killed command never returned") - } - if took := time.Since(began); took > KillWait { - t.Errorf("ending the command took %s: the group was not killed, the wait ran out", took) - } - for deadline := time.Now().Add(2 * time.Second); time.Now().Before(deadline); time.Sleep(20 * time.Millisecond) { - if err := syscall.Kill(pid, 0); errors.Is(err, syscall.ESRCH) { - return - } - } - _ = syscall.Kill(pid, syscall.SIGKILL) - t.Fatalf("the grandchild %d outlived the kill", pid) -} // Every `docker run` a build starts carries its id as a label; nothing else is touched. func TestEveryContainerABuildStartsCarriesItsID(t *testing.T) { diff --git a/internal/builder/lend_integration_test.go b/internal/builder/lend_integration_test.go new file mode 100644 index 00000000..234a8419 --- /dev/null +++ b/internal/builder/lend_integration_test.go @@ -0,0 +1,26 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { + r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = anImage + l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} + if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + want := []string{ + "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), + "push bus/server", + } + if strings.Join(l.order, "\n") != strings.Join(want, "\n") { + t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) + } +} diff --git a/internal/builder/lend_test.go b/internal/builder/lend_test.go index ff92a998..833c29ec 100644 --- a/internal/builder/lend_test.go +++ b/internal/builder/lend_test.go @@ -121,20 +121,3 @@ func (l *lending) PublishImage(ctx context.Context, localTag, repository string) l.order = append(l.order, "push "+repository) return l.mirroring.PublishImage(ctx, localTag, repository) } - -func TestAnImageIsPushedOnlyAfterItsBasesAreLent(t *testing.T) { - r, workspace := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) - r.contents["modules/bus/module.json"] = anImage - l := &lending{mirroring: &mirroring{recorded: r, at: "127.0.0.1:5000"}} - if _, err := Build(context.Background(), r.run, l, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, - nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - want := []string{ - "lend bus/server 127.0.0.1:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64), - "push bus/server", - } - if strings.Join(l.order, "\n") != strings.Join(want, "\n") { - t.Fatalf("the build did:\n%s\nwant:\n%s", strings.Join(l.order, "\n"), strings.Join(want, "\n")) - } -} diff --git a/internal/builder/mirror_shared_integration_test.go b/internal/builder/mirror_shared_integration_test.go new file mode 100644 index 00000000..7b5095a6 --- /dev/null +++ b/internal/builder/mirror_shared_integration_test.go @@ -0,0 +1,40 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) { + // A recipe that reaches for an image nobody declared is refused — after the base was copied. + r, workspace := aRepository(t, anImage, map[string]string{ + "modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"}) + r.contents["modules/bus/module.json"] = anImage + r.tree = "aaaa" + m := &mirroring{recorded: r, at: "registry-a:5000"} + got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a recipe fetching an undeclared image was built") + } + want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64) + if len(got.Mirrored) != 1 || got.Mirrored[0] != want { + t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want) + } + + // And a build that works says it too. + r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r2.contents["modules/bus/module.json"] = anImage + r2.tree = "aaaa" + ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"}, + "https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want { + t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want) + } +} diff --git a/internal/builder/mirror_shared_test.go b/internal/builder/mirror_shared_test.go index 45d21695..bc986db6 100644 --- a/internal/builder/mirror_shared_test.go +++ b/internal/builder/mirror_shared_test.go @@ -219,36 +219,6 @@ func countPrefix(m map[string][]byte, prefix string) int { // What a build copied is said whether it worked or not: the copy is in the store either way, and a // build that failed after copying is the one record that it is there. -func TestABuildSaysWhatItMirroredEvenWhenItFails(t *testing.T) { - // A recipe that reaches for an image nobody declared is refused — after the base was copied. - r, workspace := aRepository(t, anImage, map[string]string{ - "modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}\nCOPY --from=vendor/other:1 /x /x"}) - r.contents["modules/bus/module.json"] = anImage - r.tree = "aaaa" - m := &mirroring{recorded: r, at: "registry-a:5000"} - got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, - nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a recipe fetching an undeclared image was built") - } - want := "registry-a:5000/upstream/docker.io/vendor/server@sha256:" + strings.Repeat("1", 64) - if len(got.Mirrored) != 1 || got.Mirrored[0] != want { - t.Fatalf("a failed build said it mirrored %v, want [%s]", got.Mirrored, want) - } - - // And a build that works says it too. - r2, workspace2 := aRepository(t, anImage, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) - r2.contents["modules/bus/module.json"] = anImage - r2.tree = "aaaa" - ok, err := Build(context.Background(), r2.run, &mirroring{recorded: r2, at: "registry-a:5000"}, - "https://forge.invalid/catalogue.git", "modules/bus", "", workspace2, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if len(ok.Mirrored) != 1 || ok.Mirrored[0] != want { - t.Fatalf("a build said it mirrored %v, want [%s]", ok.Mirrored, want) - } -} // held puts one image in a repository the old way and answers its index digest and the registry. func heldUnderAModule(t *testing.T) (*aRegistryOfRepositories, Registry, string, string, map[string][]byte, func()) { diff --git a/internal/builder/packages_integration_test.go b/internal/builder/packages_integration_test.go new file mode 100644 index 00000000..0933214a --- /dev/null +++ b/internal/builder/packages_integration_test.go @@ -0,0 +1,135 @@ +//go:build integration + +package builder + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + npmrc := filepath.Join(workspace, "source", ".npmrc") + inContext := false + run := func(ctx context.Context, dir, name string, args ...string) (string, error) { + if name == "docker" && len(args) > 0 && args[0] == "build" { + _, err := os.Stat(npmrc) + inContext = err == nil + } + return r.run(ctx, dir, name, args...) + } + if _, err := Build(context.Background(), run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + + var build string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") { + build = line + } + } + if build == "" { + t.Fatal("no docker build ran") + } + if strings.Contains(build, "--secret") { + t.Fatalf("the build used a buildkit secret, which this path avoids: %s", build) + } + if !strings.Contains(build, "--network host") { + t.Fatalf("the build was not given the host network to reach the registry: %s", build) + } + // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and + // removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq + // issue 462). + if !inContext { + t.Fatal("the credential was not in the build context when the image was built") + } + if _, err := os.Stat(npmrc); !os.IsNotExist(err) { + t.Fatalf("the credential outlives the build in the workspace: %v", err) + } +} + +func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) { + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && (strings.Contains(line, "--secret") || strings.Contains(line, "--network host")) { + t.Fatalf("a build with no credential was still given build-network or a secret: %s", line) + } + } + tree := filepath.Join(workspace, "source") + if _, err := os.Stat(filepath.Join(tree, ".npmrc")); err == nil { + t.Fatal("an .npmrc was written into a build that has no credential") + } +} + +func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + got, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, GitCredential{}, nil) + if err != nil { + t.Fatalf("the package did not build: %v", err) + } + if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" { + t.Fatalf("a package is published by name and version, got %+v", got.Built) + } + if len(r.images) != 0 || len(r.archives) != 0 { + t.Fatal("a package was pushed to the artifact store, which is not where packages live") + } + var ran string + for _, line := range r.ran { + if strings.HasPrefix(line, "docker run") { + ran = line + } + } + if ran == "" { + t.Fatal("nothing ran to build the package") + } + if !strings.Contains(ran, "node:22-bookworm-slim") { + t.Fatalf("a package was not built on a public base: %s", ran) + } + if !strings.Contains(ran, ":/root/.npmrc:ro") { + t.Fatalf("the credential was not mounted read-only for the publish: %s", ran) + } +} + +func TestAPackageWithNoRegistryIsRefused(t *testing.T) { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a package built with no registry to publish to, silently") + } +} + +func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) { + // A Dockerfile with no .npmrc reference (like the control plane's) must build clean: no .npmrc + // in its context, no host network — so its image stays deterministic and the credential does not + // leak into a build that never resolves a mesh package. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) + n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { + t.Fatalf("the build failed: %v", err) + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--network host") { + t.Fatalf("a build that does not ask for the credential got the host network: %s", line) + } + } + if _, err := os.Stat(filepath.Join(workspace, "source", ".npmrc")); err == nil { + t.Fatal("an .npmrc was written into a build that does not reference it") + } +} diff --git a/internal/builder/packages_test.go b/internal/builder/packages_test.go index 7cccbf16..23bf50fe 100644 --- a/internal/builder/packages_test.go +++ b/internal/builder/packages_test.go @@ -1,9 +1,6 @@ package builder import ( - "context" - "os" - "path/filepath" "strings" "testing" ) @@ -67,65 +64,6 @@ func TestNpmrcDisabledUntilThereIsARegistry(t *testing.T) { // COPY in a stage it does not publish — and the build runs on the host network so a RUN resolving the // registry reaches it where the binding says (novox/hq ADR 0076). Not a buildkit secret, because this // machine's docker may carry no buildx. -func TestAnImageBuildGetsTheCredentialInTheContextAndHostNetwork(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY .npmrc ./", "files/x": "y"}) - n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - npmrc := filepath.Join(workspace, "source", ".npmrc") - inContext := false - run := func(ctx context.Context, dir, name string, args ...string) (string, error) { - if name == "docker" && len(args) > 0 && args[0] == "build" { - _, err := os.Stat(npmrc) - inContext = err == nil - } - return r.run(ctx, dir, name, args...) - } - if _, err := Build(context.Background(), run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { - t.Fatalf("the build failed: %v", err) - } - - var build string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") { - build = line - } - } - if build == "" { - t.Fatal("no docker build ran") - } - if strings.Contains(build, "--secret") { - t.Fatalf("the build used a buildkit secret, which this path avoids: %s", build) - } - if !strings.Contains(build, "--network host") { - t.Fatalf("the build was not given the host network to reach the registry: %s", build) - } - // The .npmrc is written into the build context (the source tree), where a Dockerfile COPYs it, and - // removed with the tree when the build ends: a later check's container mounts the workspace (novox/hq - // issue 462). - if !inContext { - t.Fatal("the credential was not in the build context when the image was built") - } - if _, err := os.Stat(npmrc); !os.IsNotExist(err) { - t.Fatalf("the credential outlives the build in the workspace: %v", err) - } -} - -func TestAnImageBuildWithoutACredentialGetsNoHostNetwork(t *testing.T) { - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch", "files/x": "y"}) - if _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatalf("the build failed: %v", err) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") && (strings.Contains(line, "--secret") || strings.Contains(line, "--network host")) { - t.Fatalf("a build with no credential was still given build-network or a secret: %s", line) - } - } - tree := filepath.Join(workspace, "source") - if _, err := os.Stat(filepath.Join(tree, ".npmrc")); err == nil { - t.Fatal("an .npmrc was written into a build that has no credential") - } -} const aPackage = `{"module":"mesh-sdk","version":"1", "build":{"artifacts":[{"name":"lib","kind":"package","language":"typescript"}]}, @@ -133,49 +71,6 @@ const aPackage = `{"module":"mesh-sdk","version":"1", // A package is compiled on a public base and published to the mesh's package registry by version, // with nothing pushed to the artifact store and the credential mounted, not baked (novox/hq ADR 0076). -func TestAPackageIsBuiltOnAPublicBaseAndPublishedByVersion(t *testing.T) { - r, workspace := aRepository(t, aPackage, map[string]string{ - "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, - }) - n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - got, err := Build(context.Background(), r.run, r, - "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, n, GitCredential{}, nil) - if err != nil { - t.Fatalf("the package did not build: %v", err) - } - if len(got.Built) != 1 || got.Built[0].Reference != "@novox/mesh-sdk@0.1.0" { - t.Fatalf("a package is published by name and version, got %+v", got.Built) - } - if len(r.images) != 0 || len(r.archives) != 0 { - t.Fatal("a package was pushed to the artifact store, which is not where packages live") - } - var ran string - for _, line := range r.ran { - if strings.HasPrefix(line, "docker run") { - ran = line - } - } - if ran == "" { - t.Fatal("nothing ran to build the package") - } - if !strings.Contains(ran, "node:22-bookworm-slim") { - t.Fatalf("a package was not built on a public base: %s", ran) - } - if !strings.Contains(ran, ":/root/.npmrc:ro") { - t.Fatalf("the credential was not mounted read-only for the publish: %s", ran) - } -} - -func TestAPackageWithNoRegistryIsRefused(t *testing.T) { - r, workspace := aRepository(t, aPackage, map[string]string{ - "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, - }) - _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/mesh-sdk.git", "", "", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a package built with no registry to publish to, silently") - } -} func TestNpmrcRendersBasicAuthWhenGivenAUserAndPassword(t *testing.T) { n := Npmrc{ @@ -210,23 +105,3 @@ func TestNpmrcRefusesWhenGivenNeitherTokenNorPassword(t *testing.T) { t.Fatal("a username with no password rendered an .npmrc") } } - -func TestAnImageThatDoesNotAskForTheCredentialDoesNotGetIt(t *testing.T) { - // A Dockerfile with no .npmrc reference (like the control plane's) must build clean: no .npmrc - // in its context, no host network — so its image stays deterministic and the credential does not - // leak into a build that never resolves a mesh package. - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) - n := Npmrc{Scope: "@novox", Registry: "https://forge.invalid/api/packages/novox/npm/", Token: "t"} - if _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, n, GitCredential{}, nil); err != nil { - t.Fatalf("the build failed: %v", err) - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") && strings.Contains(line, "--network host") { - t.Fatalf("a build that does not ask for the credential got the host network: %s", line) - } - } - if _, err := os.Stat(filepath.Join(workspace, "source", ".npmrc")); err == nil { - t.Fatal("an .npmrc was written into a build that does not reference it") - } -} diff --git a/internal/builder/predefined_test.go b/internal/builder/predefined_test.go index 71ac4fec..b1315afa 100644 --- a/internal/builder/predefined_test.go +++ b/internal/builder/predefined_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/registry_unbound_integration_test.go b/internal/builder/registry_unbound_integration_test.go new file mode 100644 index 00000000..eb4689c0 --- /dev/null +++ b/internal/builder/registry_unbound_integration_test.go @@ -0,0 +1,93 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func TestABuildNeedingTheRegistryIsRefusedOnAnAgentWithoutOne(t *testing.T) { + none := Npmrc{Machine: "workstation"} + cases := map[string]struct { + manifest string + files map[string]string + held map[string]string + }{ + "a package": {aPackage, map[string]string{"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`}, nil}, + "an image naming .npmrc": {withBoth, map[string]string{ + "Dockerfile": "FROM scratch\nCOPY .npmrc /root/.npmrc\n", "files/x": "y"}, nil}, + "a TypeScript bundle depending on the SDK": {aBundle, map[string]string{ + "index.ts": "console.log(1)", "package.json": `{"dependencies":{"@novox/mesh-sdk":"^0.1.0"}}`}, + map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}}, + } + for name, c := range cases { + t.Run(name, func(t *testing.T) { + r, workspace := aRepository(t, c.manifest, c.files) + var said []string + _, err := Build(context.Background(), compiling{r}.run, r, + "https://forge.invalid/m.git", "", "", workspace, c.held, none, GitCredential{}, + func(step, message string) { said = append(said, step+": "+message) }) + if err == nil { + t.Fatal("built with no package registry") + } + for _, want := range []string{"workstation", "npm-package-registry", "unbound", "gitea"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + for _, line := range r.ran { + if strings.HasPrefix(line, "docker build") || strings.HasPrefix(line, "docker run") || + strings.HasPrefix(line, "docker pull") { + t.Errorf("something was built after the refusal: %s", line) + } + } + if len(r.images) != 0 || len(r.archives) != 0 { + t.Errorf("something was published after the refusal: %v %v", r.images, r.archives) + } + if !strings.Contains(strings.Join(said, "\n"), "REFUSED") { + t.Errorf("the build's log does not say it was refused:\n%s", strings.Join(said, "\n")) + } + }) + } + + // A Go artifact builds on the same agent. + files := aSharedRepository("one", "1") + manifest := `{"module":"ctl","version":"1","build":{"artifacts":[ + {"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]}, + "resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}` + r := &recorded{contents: map[string]string{ManifestName: manifest}} + for k, v := range files { + r.contents[k] = v + } + held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)} + if _, err := Build(context.Background(), onTrunk{recorded: r}.run, r, + "https://forge.invalid/ctl.git", "", "", t.TempDir(), held, none, GitCredential{}, nil); err != nil { + t.Fatalf("a Go artifact did not build on an agent without the registry: %v", err) + } + // And an image that does not name .npmrc. + r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) + if _, err := Build(context.Background(), r.run, r, + "https://forge.invalid/meshboard.git", "", "", workspace, nil, none, GitCredential{}, nil); err != nil { + t.Fatalf("an image not naming .npmrc did not build on an agent without the registry: %v", err) + } +} + +func TestABuildSaysWhichRegistryItResolvedFrom(t *testing.T) { + for _, from := range []string{FromBinding, FromGenesis} { + r, workspace := aRepository(t, aPackage, map[string]string{ + "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, + }) + n := Npmrc{Scope: "@novox", Registry: "http://127.0.0.1:3000/api/packages/novox/npm/", + Username: "mesh-builder", Password: "p", From: from, Machine: "anchor"} + var said []string + if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/mesh-sdk.git", "", "", + workspace, nil, n, GitCredential{}, func(step, message string) { said = append(said, step+": "+message) }); err != nil { + t.Fatalf("%s: %v", from, err) + } + if !strings.Contains(strings.Join(said, "\n"), "from "+from) { + t.Errorf("the log does not say the registry came from %s:\n%s", from, strings.Join(said, "\n")) + } + } +} diff --git a/internal/builder/registry_unbound_test.go b/internal/builder/registry_unbound_test.go index d766a72d..c5c1f963 100644 --- a/internal/builder/registry_unbound_test.go +++ b/internal/builder/registry_unbound_test.go @@ -1,10 +1,8 @@ package builder import ( - "context" "os" "path/filepath" - "strings" "testing" "github.com/novox/mesh-controller/internal/beside" @@ -17,90 +15,8 @@ import ( // with an `@novox/` dependency each fail before anything is built or published, naming the machine, // the unbound requirement and the module that provides it. A Go artifact, and an image that does not // name .npmrc, build there as before. -func TestABuildNeedingTheRegistryIsRefusedOnAnAgentWithoutOne(t *testing.T) { - none := Npmrc{Machine: "workstation"} - cases := map[string]struct { - manifest string - files map[string]string - held map[string]string - }{ - "a package": {aPackage, map[string]string{"package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`}, nil}, - "an image naming .npmrc": {withBoth, map[string]string{ - "Dockerfile": "FROM scratch\nCOPY .npmrc /root/.npmrc\n", "files/x": "y"}, nil}, - "a TypeScript bundle depending on the SDK": {aBundle, map[string]string{ - "index.ts": "console.log(1)", "package.json": `{"dependencies":{"@novox/mesh-sdk":"^0.1.0"}}`}, - map[string]string{"mesh-tools/build": "registry.invalid/mesh-tools/build@sha256:" + strings.Repeat("b", 64)}}, - } - for name, c := range cases { - t.Run(name, func(t *testing.T) { - r, workspace := aRepository(t, c.manifest, c.files) - var said []string - _, err := Build(context.Background(), compiling{r}.run, r, - "https://forge.invalid/m.git", "", "", workspace, c.held, none, GitCredential{}, - func(step, message string) { said = append(said, step+": "+message) }) - if err == nil { - t.Fatal("built with no package registry") - } - for _, want := range []string{"workstation", "npm-package-registry", "unbound", "gitea"} { - if !strings.Contains(err.Error(), want) { - t.Errorf("the refusal does not say %q: %v", want, err) - } - } - for _, line := range r.ran { - if strings.HasPrefix(line, "docker build") || strings.HasPrefix(line, "docker run") || - strings.HasPrefix(line, "docker pull") { - t.Errorf("something was built after the refusal: %s", line) - } - } - if len(r.images) != 0 || len(r.archives) != 0 { - t.Errorf("something was published after the refusal: %v %v", r.images, r.archives) - } - if !strings.Contains(strings.Join(said, "\n"), "REFUSED") { - t.Errorf("the build's log does not say it was refused:\n%s", strings.Join(said, "\n")) - } - }) - } - - // A Go artifact builds on the same agent. - files := aSharedRepository("one", "1") - manifest := `{"module":"ctl","version":"1","build":{"artifacts":[ - {"name":"controller","kind":"bundle","language":"go","system":"arch","from":"cmd/ctl","binary":"ctl"}]}, - "resources":[{"id":"controller","type":"process","name":"ctl","artifact":"controller","run":["./ctl"]}]}` - r := &recorded{contents: map[string]string{ManifestName: manifest}} - for k, v := range files { - r.contents[k] = v - } - held := map[string]string{"mesh-tools-go/build": "registry.invalid/mesh-tools-go/build@sha256:" + strings.Repeat("b", 64)} - if _, err := Build(context.Background(), onTrunk{recorded: r}.run, r, - "https://forge.invalid/ctl.git", "", "", t.TempDir(), held, none, GitCredential{}, nil); err != nil { - t.Fatalf("a Go artifact did not build on an agent without the registry: %v", err) - } - // And an image that does not name .npmrc. - r, workspace := aRepository(t, withBoth, map[string]string{"Dockerfile": "FROM scratch\nCOPY . .", "files/x": "y"}) - if _, err := Build(context.Background(), r.run, r, - "https://forge.invalid/meshboard.git", "", "", workspace, nil, none, GitCredential{}, nil); err != nil { - t.Fatalf("an image not naming .npmrc did not build on an agent without the registry: %v", err) - } -} // **The build's log says which registry it resolved from** (novox/hq ADR 0336 §4). -func TestABuildSaysWhichRegistryItResolvedFrom(t *testing.T) { - for _, from := range []string{FromBinding, FromGenesis} { - r, workspace := aRepository(t, aPackage, map[string]string{ - "package.json": `{"name":"@novox/mesh-sdk","version":"0.1.0"}`, - }) - n := Npmrc{Scope: "@novox", Registry: "http://127.0.0.1:3000/api/packages/novox/npm/", - Username: "mesh-builder", Password: "p", From: from, Machine: "anchor"} - var said []string - if _, err := Build(context.Background(), r.run, r, "https://forge.invalid/mesh-sdk.git", "", "", - workspace, nil, n, GitCredential{}, func(step, message string) { said = append(said, step+": "+message) }); err != nil { - t.Fatalf("%s: %v", from, err) - } - if !strings.Contains(strings.Join(said, "\n"), "from "+from) { - t.Errorf("the log does not say the registry came from %s:\n%s", from, strings.Join(said, "\n")) - } - } -} // The refusal names the module that provides the registry; it is the catalogue's provider. func TestTheRegistryABuildIsRefusedForNamesItsProvider(t *testing.T) { diff --git a/internal/builder/registry_wait_test.go b/internal/builder/registry_wait_test.go index 8bb9be85..c529f716 100644 --- a/internal/builder/registry_wait_test.go +++ b/internal/builder/registry_wait_test.go @@ -76,13 +76,13 @@ func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) { waitingFor(t, 5*time.Second) said := saying(t) f := &fakeRegistry{} - r := Registry{Address: heldStill(t, f, 400*time.Millisecond)} + r := Registry{Address: heldStill(t, f, 100*time.Millisecond)} body := []byte("a theme") sum := sha256.Sum256(body) digest := "sha256:" + hex.EncodeToString(sum[:]) if _, err := r.PublishArchive(context.Background(), "shell/config", body, digest); err != nil { - t.Fatalf("a registry refusing for 400ms failed the build: %v", err) + t.Fatalf("a registry refusing for 100ms failed the build: %v", err) } if string(f.blobs[digest]) != "a theme" { t.Fatalf("the registry holds %q", f.blobs[digest]) @@ -97,7 +97,7 @@ func TestABuildWaitsForARegistryHeldStillAndGoesOn(t *testing.T) { } func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) { - waitingFor(t, 300*time.Millisecond) + waitingFor(t, 100*time.Millisecond) said := saying(t) f := &fakeRegistry{} r := Registry{Address: heldStill(t, f, -1)} @@ -113,7 +113,7 @@ func TestABuildFailsLoudlyOnARegistryRefusingPastTheBound(t *testing.T) { if !strings.Contains(err.Error(), "refused every connection for") || !strings.Contains(err.Error(), "connection refused") { t.Errorf("the failure does not say it waited and was refused throughout: %v", err) } - if waited := time.Since(started); waited < 300*time.Millisecond { + if waited := time.Since(started); waited < 100*time.Millisecond { t.Errorf("failed after %s, before the bound", waited) } if log := strings.Join(said(), "\n"); !strings.Contains(log, "waits for the registry") { diff --git a/internal/builder/replay310_test.go b/internal/builder/replay310_test.go index 1680588c..4eb05a6e 100644 --- a/internal/builder/replay310_test.go +++ b/internal/builder/replay310_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay341_test.go b/internal/builder/replay341_test.go index 031a903d..1d28498b 100644 --- a/internal/builder/replay341_test.go +++ b/internal/builder/replay341_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay363_test.go b/internal/builder/replay363_test.go index 07271d28..7715e4ea 100644 --- a/internal/builder/replay363_test.go +++ b/internal/builder/replay363_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay713_test.go b/internal/builder/replay713_test.go index 214ae2a2..76566b9e 100644 --- a/internal/builder/replay713_test.go +++ b/internal/builder/replay713_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/replay716_test.go b/internal/builder/replay716_test.go index 306c3e92..d1991c1a 100644 --- a/internal/builder/replay716_test.go +++ b/internal/builder/replay716_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/reproducible_integration_test.go b/internal/builder/reproducible_integration_test.go new file mode 100644 index 00000000..7f3c5ac5 --- /dev/null +++ b/internal/builder/reproducible_integration_test.go @@ -0,0 +1,91 @@ +//go:build integration + +package builder + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestAGoBundleBuiltTwiceFromOneBuildSourceIsOneDigest(t *testing.T) { + for _, tool := range []string{"go", "git"} { + if _, err := exec.LookPath(tool); err != nil { + t.Skipf("%s is not on this machine: %v", tool, err) + } + } + chain, err := ToolchainFor("go") + if err != nil { + t.Fatal(err) + } + root := t.TempDir() + repo := filepath.Join(root, "origin") + files := map[string]string{ + "go.mod": "module example.com/ctl\n\ngo 1.22\n", + "cmd/ctl/main.go": "package main\n\nfunc main() { println(\"ctl\") }\n", + "README.md": "one\n", + } + for name, body := range files { + if err := os.MkdirAll(filepath.Dir(filepath.Join(repo, name)), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(repo, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + git := func(dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.invalid", + "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...) + cmd.Dir = dir + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) + } + git(repo, "init", "-q") + git(repo, "add", ".") + git(repo, "commit", "-q", "-m", "one") + first := git(repo, "rev-parse", "HEAD") + if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("two\n"), 0o644); err != nil { + t.Fatal(err) + } + git(repo, "commit", "-q", "-am", "only the README") + second := git(repo, "rev-parse", "HEAD") + + build := func(commit, at string) string { + t.Helper() + clone := filepath.Join(root, at) + git(root, "clone", "-q", repo, clone) + git(clone, "checkout", "-q", commit) + out := filepath.Join(clone, Out("server"), "ctl") + line := append([]string(nil), chain.Compile...) + line = append(line, "-ldflags", strings.Join(chain.LinkerFlags, " "), chain.OutputFlag, out, "./cmd/ctl") + cmd := exec.Command(line[0], line[1:]...) + cmd.Dir = clone + cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off") + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s: %v\n%s", strings.Join(line, " "), err, b) + } + body, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(body, []byte(commit)) { + t.Errorf("the binary built at %s carries the commit it was built at", commit[:12]) + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]) + } + a, b := build(first, "a"), build(second, "elsewhere/b") + if a != b { + t.Fatalf("one build source, two digests: %s at %s, %s at %s (compiled with %q)", + a[:12], first[:12], b[:12], second[:12], strings.Join(chain.Compile, " ")) + } +} diff --git a/internal/builder/reproducible_test.go b/internal/builder/reproducible_test.go index 345ce055..137d1a03 100644 --- a/internal/builder/reproducible_test.go +++ b/internal/builder/reproducible_test.go @@ -1,11 +1,7 @@ package builder import ( - "bytes" - "crypto/sha256" - "encoding/hex" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -19,82 +15,6 @@ import ( // // The toolchain's own compile line is run here, as the build seat runs it inside the toolchain image: in // two clones at two paths, of two commits that differ outside the program. One digest is wanted. -func TestAGoBundleBuiltTwiceFromOneBuildSourceIsOneDigest(t *testing.T) { - for _, tool := range []string{"go", "git"} { - if _, err := exec.LookPath(tool); err != nil { - t.Skipf("%s is not on this machine: %v", tool, err) - } - } - chain, err := ToolchainFor("go") - if err != nil { - t.Fatal(err) - } - root := t.TempDir() - repo := filepath.Join(root, "origin") - files := map[string]string{ - "go.mod": "module example.com/ctl\n\ngo 1.22\n", - "cmd/ctl/main.go": "package main\n\nfunc main() { println(\"ctl\") }\n", - "README.md": "one\n", - } - for name, body := range files { - if err := os.MkdirAll(filepath.Dir(filepath.Join(repo, name)), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(repo, name), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - git := func(dir string, args ...string) string { - t.Helper() - cmd := exec.Command("git", append([]string{"-c", "user.name=t", "-c", "user.email=t@example.invalid", - "-c", "commit.gpgsign=false", "-c", "init.defaultBranch=main"}, args...)...) - cmd.Dir = dir - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) - } - return strings.TrimSpace(string(out)) - } - git(repo, "init", "-q") - git(repo, "add", ".") - git(repo, "commit", "-q", "-m", "one") - first := git(repo, "rev-parse", "HEAD") - if err := os.WriteFile(filepath.Join(repo, "README.md"), []byte("two\n"), 0o644); err != nil { - t.Fatal(err) - } - git(repo, "commit", "-q", "-am", "only the README") - second := git(repo, "rev-parse", "HEAD") - - build := func(commit, at string) string { - t.Helper() - clone := filepath.Join(root, at) - git(root, "clone", "-q", repo, clone) - git(clone, "checkout", "-q", commit) - out := filepath.Join(clone, Out("server"), "ctl") - line := append([]string(nil), chain.Compile...) - line = append(line, "-ldflags", strings.Join(chain.LinkerFlags, " "), chain.OutputFlag, out, "./cmd/ctl") - cmd := exec.Command(line[0], line[1:]...) - cmd.Dir = clone - cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local", "GOPROXY=off", "GOWORK=off") - if b, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("%s: %v\n%s", strings.Join(line, " "), err, b) - } - body, err := os.ReadFile(out) - if err != nil { - t.Fatal(err) - } - if bytes.Contains(body, []byte(commit)) { - t.Errorf("the binary built at %s carries the commit it was built at", commit[:12]) - } - sum := sha256.Sum256(body) - return hex.EncodeToString(sum[:]) - } - a, b := build(first, "a"), build(second, "elsewhere/b") - if a != b { - t.Fatalf("one build source, two digests: %s at %s, %s at %s (compiled with %q)", - a[:12], first[:12], b[:12], second[:12], strings.Join(chain.Compile, " ")) - } -} // The toolchain says so itself, so a reader of the compile line sees it without running a build. func TestTheGoToolchainStampsNoVersionControl(t *testing.T) { diff --git a/internal/builder/reuse_integration_test.go b/internal/builder/reuse_integration_test.go new file mode 100644 index 00000000..c7ab6afb --- /dev/null +++ b/internal/builder/reuse_integration_test.go @@ -0,0 +1,336 @@ +//go:build integration + +package builder + +import ( + "reflect" + "strings" + "testing" +) + +func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { + t.Helper() + r, run, workspace := aModuleAt(t, tree) + return aCheckBuiltBy(t, scratch, r, run, workspace) +} + +func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { + t.Helper() + spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, + Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", + Ref: "main"}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "", + CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil)) + if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { + t.Fatalf("the check did not build the module for its merge: %+v", built) + } + if !dockerBuilt(r) { + t.Fatal("the check built no image") + } + for repository := range r.images { + if !strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the check published %s outside the scratch namespace", repository) + } + } + for repository := range r.archives { + if !strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the check published %s outside the scratch namespace", repository) + } + } + return built[0] +} + +func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { + t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives) + } + if got.Reused == nil || got.Why != "" { + t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why) + } + if got.ReuseKey != checked.Key { + t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key) + } + if len(scratch.took) != 2 { + t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took) + } + for _, made := range checked.Made { + registered := strings.Replace(made.Reference, ScratchPrefix, "", 1) + found := false + for _, b := range got.Built { + found = found || (b.Reference == registered && b.Digest == made.Digest) + } + if !found { + t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference) + } + } + if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) { + t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image) + } + words := got.Reused.Words() + if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") || + !strings.Contains(words, "sha256:") { + t.Fatalf("the walk says %q", words) + } +} + +func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + for name, leave := range map[string][]NamedBuild{ + "no verdict": nil, + "another digest": func() []NamedBuild { + n := named(checked) + n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) + return n + }(), + "another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(), + } { + t.Run(name, func(t *testing.T) { + scratch.took = nil + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) { + t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took) + } + if !strings.Contains(got.Why, "no check verdict of the walk's pull requests at the head the forge merged names it") || + !strings.Contains(got.Why, "at head abcdefabcdef") { + t.Fatalf("the walk says %q", got.Why) + } + }) + } +} + +func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-two") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || got.Reused != nil { + t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused) + } + if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) { + t.Fatalf("the walk says %q, not which part of the key differs", got.Why) + } + + // And a builder of another version builds too, saying so. + withVersion(t, "builder-2") + r, run, workspace = aModuleAt(t, "tree-one") + got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) { + t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why) + } + + // No check of the walk's pull requests built it: said. + r, run, workspace = aModuleAt(t, "tree-one") + got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") { + t.Fatalf("the walk says %q", got.Why) + } +} + +func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + scratch.failTake = true + + r, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a failed copy failed the walk: %v", err) + } + if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 { + t.Fatal("a failed copy did not fall back to a build") + } + if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") { + t.Fatalf("the walk says %q", got.Why) + } + for repository := range r.images { + if strings.HasPrefix(repository, ScratchPrefix) { + t.Fatalf("the walk's own build went into the scratch namespace: %s", repository) + } + } +} + +func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + checked := aCheckBuilt(t, scratch, "tree-one") + + r, run, workspace := aModuleAt(t, "tree-one") + if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, + Npmrc{}, GitCredential{}, nil); err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || len(scratch.took) > 0 { + t.Fatal("a build not asked to reuse took a check's build") + } + + r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it + got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git", + "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") { + t.Fatalf("a commit off the trunk took a check's build: %q", got.Why) + } +} + +func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) { + if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" { + t.Fatal("a check whose repository layer failed builds for its merge") + } + if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" { + t.Fatal("a check that passed with a warning builds nothing for its merge") + } + r, run, workspace := aModuleAt(t, "tree-one") + if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()}, + workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 { + t.Fatalf("a check moving no module built %v, ran %v", built, r.ran) + } +} + +func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) { + withVersion(t, "builder-1") + r, run, workspace := aModuleAt(t, "tree-one") + r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[ + {"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}` + spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{ + {Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0}, + {Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "", + CheckVerdict{}, GitCredential{}, nil, logging(nil)) + if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") || + !strings.Contains(built[1].NotBuilt, "stands after meshboard") { + t.Fatalf("built %+v", built) + } + if len(r.images) > 0 { + t.Fatalf("a module without a key was published: %v", r.images) + } +} + +func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { + t.Helper() + return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave) +} + +func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { + t.Helper() + r, run, workspace := aModuleAt(t, tree) + spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number, + Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", + Ref: "main"}}} + built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave}, + workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil)) + if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { + t.Fatalf("the second check did not answer the module for its merge: %+v", built) + } + return built[0], r +} + +func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + + again, r := aCheckAgain(t, scratch, "tree-one", named(first)) + if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { + t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives) + } + if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) { + t.Fatalf("the second check names %+v, not the first check's build %+v", again, first) + } + // The walk takes it by what the second verdict names. + leave := named(again) + leave[0].Check = again.Kept + w, run, workspace := aModuleAt(t, "tree-one") + got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil || got.Reused == nil || dockerBuilt(w) { + t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why) + } +} + +func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + + again, r := aCheckAgain(t, scratch, "tree-two", named(first)) + if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key { + t.Fatalf("a check whose key changed did not build: %+v", again) + } +} + +func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) { + withVersion(t, "builder-1") + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + for name, leave := range map[string][]NamedBuild{ + "no verdict": nil, + "another digest": func() []NamedBuild { + n := named(first) + n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) + return n + }(), + "another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(), + } { + t.Run(name, func(t *testing.T) { + again, r := aCheckAgain(t, scratch, "tree-one", leave) + if !dockerBuilt(r) || again.Kept != "" { + t.Fatalf("a record no verdict names was kept: %+v", again) + } + }) + } +} + +func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) { + withVersion(t, "builder-1") + other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8} + for name, c := range map[string]struct { + pull PullRef + head string + }{ + "another head": {aPull, "999999999999"}, + "another pull request": {other, "abcdefabcdef"}, + } { + t.Run(name, func(t *testing.T) { + scratch := newScratch() + first := aCheckBuilt(t, scratch, "tree-one") + leave := named(first) + leave[0].Pull = c.pull // as if its own verdict named it + again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave) + if !dockerBuilt(r) || again.Kept != "" { + t.Fatalf("a build of %s was kept: %+v", name, again) + } + }) + } +} diff --git a/internal/builder/reuse_test.go b/internal/builder/reuse_test.go index c9cbb229..2f2c5277 100644 --- a/internal/builder/reuse_test.go +++ b/internal/builder/reuse_test.go @@ -6,7 +6,6 @@ import ( "errors" "net/http" "net/http/httptest" - "reflect" "slices" "strconv" "strings" @@ -104,38 +103,8 @@ func dockerBuilt(r *recorded) bool { var aPull = PullRef{Owner: "novox", Repo: "meshboard", Number: 7} // aCheckBuilt is the check of aPull building the module from a tree, into the scratch namespace. -func aCheckBuilt(t *testing.T, scratch *aScratch, tree string) CheckBuilt { - t.Helper() - r, run, workspace := aModuleAt(t, tree) - return aCheckBuiltBy(t, scratch, r, run, workspace) -} // aCheckBuiltBy is aCheckBuilt of the repository given. -func aCheckBuiltBy(t *testing.T, scratch *aScratch, r *recorded, run Runner, workspace string) CheckBuilt { - t.Helper() - spec := CheckSpec{ID: "build-1", Owner: aPull.Owner, Repo: aPull.Repo, Number: aPull.Number, - Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", - Ref: "main"}}} - built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch}, workspace, "", "", - CheckVerdict{Head: "abcdefabcdef", Trunk: "123456123456"}, GitCredential{}, nil, logging(nil)) - if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { - t.Fatalf("the check did not build the module for its merge: %+v", built) - } - if !dockerBuilt(r) { - t.Fatal("the check built no image") - } - for repository := range r.images { - if !strings.HasPrefix(repository, ScratchPrefix) { - t.Fatalf("the check published %s outside the scratch namespace", repository) - } - } - for repository := range r.archives { - if !strings.HasPrefix(repository, ScratchPrefix) { - t.Fatalf("the check published %s outside the scratch namespace", repository) - } - } - return built[0] -} // named is the walk's leave naming what a check's verdict says it built, as the controller reads it. func named(c CheckBuilt) []NamedBuild { @@ -159,211 +128,21 @@ func withVersion(t *testing.T, v string) { // A merge whose reuse key equals its check's builds nothing again: the walk copies the check's build by digest // and registers that digest. -func TestAWalkWhoseKeyIsItsChecksBuildsNothingAndRegistersTheChecksDigest(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - - r, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { - t.Fatalf("the walk built again: ran %v, published %v %v", r.ran, r.images, r.archives) - } - if got.Reused == nil || got.Why != "" { - t.Fatalf("the walk did not say it reused the check's build: reused %+v, why %q", got.Reused, got.Why) - } - if got.ReuseKey != checked.Key { - t.Fatalf("the walk's key %s is not the check's %s", got.ReuseKey, checked.Key) - } - if len(scratch.took) != 2 { - t.Fatalf("the walk copied %v, not the check's two artifacts", scratch.took) - } - for _, made := range checked.Made { - registered := strings.Replace(made.Reference, ScratchPrefix, "", 1) - found := false - for _, b := range got.Built { - found = found || (b.Reference == registered && b.Digest == made.Digest) - } - if !found { - t.Fatalf("the walk registers %+v, not the check's %s copied", got.Built, made.Reference) - } - } - if image, _ := got.Manifest.Resources[0]["image"].(string); image != "registry.invalid/meshboard/server@sha256:"+strings.Repeat("a", 64) { - t.Fatalf("the manifest pins %q, not the check's digest in the module's namespace", image) - } - words := got.Reused.Words() - if !strings.HasPrefix(words, "reused from the check of novox/meshboard#7 at head abcdefabcdef (merged into trunk 123456123456)") || - !strings.Contains(words, "sha256:") { - t.Fatalf("the walk says %q", words) - } -} // A record no verdict names is refused and built instead, and the walk says so: the store takes writes from anything // on its network, so a record planted under another merge's key is never registered. -func TestARecordNoVerdictNamesIsRefusedAndBuilt(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - for name, leave := range map[string][]NamedBuild{ - "no verdict": nil, - "another digest": func() []NamedBuild { - n := named(checked) - n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) - return n - }(), - "another check": func() []NamedBuild { n := named(checked); n[0].Check = "build-9"; return n }(), - } { - t.Run(name, func(t *testing.T) { - scratch.took = nil - r, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Reused != nil || len(scratch.took) > 0 || !dockerBuilt(r) { - t.Fatalf("a record no verdict names was taken: %+v, copied %v", got.Reused, scratch.took) - } - if !strings.Contains(got.Why, "no check verdict of the walk's pull requests at the head the forge merged names it") || - !strings.Contains(got.Why, "at head abcdefabcdef") { - t.Fatalf("the walk says %q", got.Why) - } - }) - } -} // A differing key builds, and says which part differs. -func TestAWalkWhoseKeyDiffersBuildsAndSaysWhy(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - - r, run, workspace := aModuleAt(t, "tree-two") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || got.Reused != nil { - t.Fatalf("a walk whose source differs did not build: reused %+v", got.Reused) - } - if !strings.Contains(got.Why, "the key differs") || !strings.Contains(got.Why, partTree) { - t.Fatalf("the walk says %q, not which part of the key differs", got.Why) - } - - // And a builder of another version builds too, saying so. - withVersion(t, "builder-2") - r, run, workspace = aModuleAt(t, "tree-one") - got, err = BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || !strings.Contains(got.Why, partBuilder) { - t.Fatalf("a newer builder took the older one's build, or did not say why not: %q", got.Why) - } - - // No check of the walk's pull requests built it: said. - r, run, workspace = aModuleAt(t, "tree-one") - got, _ = BuildReusing(t.Context(), run, r, Reuse{Scratch: newScratch(), Pulls: []PullRef{aPull}}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if !dockerBuilt(r) || !strings.Contains(got.Why, "no check of novox/meshboard#7 built it") { - t.Fatalf("the walk says %q", got.Why) - } -} // A copy that fails is said and falls back to a build; it never fails the walk. -func TestAFailedCopyOfAChecksBuildFallsBackToBuilding(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - scratch.failTake = true - - r, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a failed copy failed the walk: %v", err) - } - if !dockerBuilt(r) || got.Reused != nil || len(r.images) == 0 { - t.Fatal("a failed copy did not fall back to a build") - } - if !strings.Contains(got.Why, "copying meshboard's") || !strings.Contains(got.Why, "refused the mount") { - t.Fatalf("the walk says %q", got.Why) - } - for repository := range r.images { - if strings.HasPrefix(repository, ScratchPrefix) { - t.Fatalf("the walk's own build went into the scratch namespace: %s", repository) - } - } -} // Only a walk takes a check's build: a plain build — a check's, a dry run's, `build --ref`'s — builds, and a commit // off the trunk never takes one. -func TestOnlyAWalkOfATrunkCommitTakesAChecksBuild(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - checked := aCheckBuilt(t, scratch, "tree-one") - - r, run, workspace := aModuleAt(t, "tree-one") - if _, err := Build(t.Context(), run, r, "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, - Npmrc{}, GitCredential{}, nil); err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || len(scratch.took) > 0 { - t.Fatal("a build not asked to reuse took a check's build") - } - - r, _, workspace = aModuleAt(t, "tree-one") // no trunk said: off it - got, err := BuildReusing(t.Context(), r.run, r, Reuse{Scratch: scratch, Named: named(checked)}, "https://forge.invalid/meshboard.git", - "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if !dockerBuilt(r) || got.Reused != nil || !strings.Contains(got.Why, "not on its trunk") { - t.Fatalf("a commit off the trunk took a check's build: %q", got.Why) - } -} // A check that did not pass builds nothing for its merge, and one asked to build nothing builds nothing. -func TestACheckThatFailedBuildsNothingForItsMerge(t *testing.T) { - if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "fail"}}) == "" { - t.Fatal("a check whose repository layer failed builds for its merge") - } - if skipsTheBuilds(CheckVerdict{Gate: &Layer{Verdict: "pass"}, Repo: &Layer{Verdict: "warning"}}) != "" { - t.Fatal("a check that passed with a warning builds nothing for its merge") - } - r, run, workspace := aModuleAt(t, "tree-one") - if built := buildForTheMerge(t.Context(), run, CheckSpec{ID: "build-1"}, CheckBuilding{Publish: r, Scratch: newScratch()}, - workspace, "", "", CheckVerdict{}, GitCredential{}, nil, logging(nil)); built != nil || len(r.ran) > 0 { - t.Fatalf("a check moving no module built %v, ran %v", built, r.ran) - } -} // A module whose source does not pin its build has no key and is not built by the check; what stands after it // is not built either. -func TestACheckBuildsNoModuleWithoutAKeyNorWhatStandsAfterIt(t *testing.T) { - withVersion(t, "builder-1") - r, run, workspace := aModuleAt(t, "tree-one") - r.contents["mod/"+ManifestName] = `{"module":"meshboard","version":"1","build":{"artifacts":[ - {"name":"mirror","kind":"upstream","from":"docker.io/library/alpine:3"}]}}` - spec := CheckSpec{ID: "build-1", Owner: "novox", Repo: "meshboard", Number: 7, Builds: []CheckBuildSpec{ - {Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", Tier: 0}, - {Module: "after", Repository: "https://forge.invalid/after.git", Tier: 1}}} - built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: newScratch()}, workspace, "", "", - CheckVerdict{}, GitCredential{}, nil, logging(nil)) - if len(built) != 2 || !strings.Contains(built[0].NotBuilt, "named by tag") || - !strings.Contains(built[1].NotBuilt, "stands after meshboard") { - t.Fatalf("built %+v", built) - } - if len(r.images) > 0 { - t.Fatalf("a module without a key was published: %v", r.images) - } -} // What the key names: every input a fresh build could differ by. func TestTheReuseKeyNamesEveryInputOfTheBuild(t *testing.T) { @@ -642,86 +421,16 @@ func TestABuildTwoChecksShareGoesWithTheLastOfThem(t *testing.T) { // aCheckAgain is a later check (build-2) of aPull building the module from a tree, with the builds the build seat's // verdicts name for it (novox/hq issue 703). -func aCheckAgain(t *testing.T, scratch *aScratch, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { - t.Helper() - return aCheckOf(t, scratch, aPull, "abcdefabcdef", tree, leave) -} // aCheckOf is a later check (build-2) of a pull request at a head, the trunk moved since build-1. -func aCheckOf(t *testing.T, scratch *aScratch, pull PullRef, head, tree string, leave []NamedBuild) (CheckBuilt, *recorded) { - t.Helper() - r, run, workspace := aModuleAt(t, tree) - spec := CheckSpec{ID: "build-2", Owner: pull.Owner, Repo: pull.Repo, Number: pull.Number, - Builds: []CheckBuildSpec{{Module: "meshboard", Repository: "https://forge.invalid/meshboard.git", Path: "mod", - Ref: "main"}}} - built := buildForTheMerge(t.Context(), run, spec, CheckBuilding{Publish: r, Scratch: scratch, Named: leave}, - workspace, "", "", CheckVerdict{Head: head, Trunk: "654321654321"}, GitCredential{}, nil, logging(nil)) - if len(built) != 1 || built[0].NotBuilt != "" || built[0].Key == "" { - t.Fatalf("the second check did not answer the module for its merge: %+v", built) - } - return built[0], r -} // A check asked again when only the trunk moved, whose key is unchanged, builds nothing: it keeps the build an // earlier verdict names and names it by key and digest, so the walk can take it (novox/hq issue 703). -func TestASecondCheckWhoseKeyIsUnchangedBuildsNothingAndNamesTheEarlierBuild(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - - again, r := aCheckAgain(t, scratch, "tree-one", named(first)) - if dockerBuilt(r) || len(r.images) > 0 || len(r.archives) > 0 { - t.Fatalf("the second check built again: ran %v, published %v %v", r.ran, r.images, r.archives) - } - if again.Key != first.Key || again.Kept != "build-1" || !reflect.DeepEqual(again.Made, first.Made) { - t.Fatalf("the second check names %+v, not the first check's build %+v", again, first) - } - // The walk takes it by what the second verdict names. - leave := named(again) - leave[0].Check = again.Kept - w, run, workspace := aModuleAt(t, "tree-one") - got, err := BuildReusing(t.Context(), run, w, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: leave}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil || got.Reused == nil || dockerBuilt(w) { - t.Fatalf("the walk did not take the kept build: %v, why %q", err, got.Why) - } -} // A check whose key changed builds, as before. -func TestASecondCheckWhoseKeyChangedBuilds(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - - again, r := aCheckAgain(t, scratch, "tree-two", named(first)) - if !dockerBuilt(r) || again.Kept != "" || again.Key == first.Key { - t.Fatalf("a check whose key changed did not build: %+v", again) - } -} // A scratch record no verdict names is never kept: the store takes writes from anything on its network, so a // check builds rather than name a planted record (novox/hq issue 703, ADR 0350 §2). -func TestASecondCheckKeepsNoRecordAVerdictDoesNotName(t *testing.T) { - withVersion(t, "builder-1") - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - for name, leave := range map[string][]NamedBuild{ - "no verdict": nil, - "another digest": func() []NamedBuild { - n := named(first) - n[0].Digests["server"] = "sha256:" + strings.Repeat("e", 64) - return n - }(), - "another key": func() []NamedBuild { n := named(first); n[0].Key = "r1:" + strings.Repeat("f", 64); return n }(), - } { - t.Run(name, func(t *testing.T) { - again, r := aCheckAgain(t, scratch, "tree-one", leave) - if !dockerBuilt(r) || again.Kept != "" { - t.Fatalf("a record no verdict names was kept: %+v", again) - } - }) - } -} // A build a later check kept stays when the check that built it goes: its artifacts and its record, which the later // check's verdict names (novox/hq issue 703, ADR 0350 §5). @@ -767,25 +476,3 @@ func TestABuildALaterCheckKeptStaysWhenTheCheckThatBuiltItGoes(t *testing.T) { // Only an earlier check of the same pull request at the same head is kept from: another head of it, or another pull // request whose verdict names the same key and digests, builds (novox/hq issue 703). -func TestACheckKeepsNoBuildOfAnotherHeadOrPullRequest(t *testing.T) { - withVersion(t, "builder-1") - other := PullRef{Owner: "novox", Repo: "meshboard", Number: 8} - for name, c := range map[string]struct { - pull PullRef - head string - }{ - "another head": {aPull, "999999999999"}, - "another pull request": {other, "abcdefabcdef"}, - } { - t.Run(name, func(t *testing.T) { - scratch := newScratch() - first := aCheckBuilt(t, scratch, "tree-one") - leave := named(first) - leave[0].Pull = c.pull // as if its own verdict named it - again, r := aCheckOf(t, scratch, c.pull, c.head, "tree-one", leave) - if !dockerBuilt(r) || again.Kept != "" { - t.Fatalf("a build of %s was kept: %+v", name, again) - } - }) - } -} diff --git a/internal/builder/siblings_test.go b/internal/builder/siblings_test.go index 98dc8951..5fd1fd2c 100644 --- a/internal/builder/siblings_test.go +++ b/internal/builder/siblings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package builder import ( diff --git a/internal/builder/source_integration_test.go b/internal/builder/source_integration_test.go new file mode 100644 index 00000000..b047ae72 --- /dev/null +++ b/internal/builder/source_integration_test.go @@ -0,0 +1,44 @@ +//go:build integration + +package builder + +import ( + "context" + "strings" + "testing" +) + +func fingerprintOf(t *testing.T, manifest, tree string, mirrored string) string { + t.Helper() + r, workspace := aRepository(t, manifest, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) + r.contents["modules/bus/module.json"] = manifest + r.tree = tree + m := &mirroring{recorded: r, at: mirrored} + got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, + nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + return got.Source +} + +func TestASourceFingerprintNamesWhatABuildWasMadeFrom(t *testing.T) { + one := fingerprintOf(t, anImage, "aaaa", "registry-a:5000") + if !strings.HasPrefix(one, sourcePrefix) { + t.Fatalf("no fingerprint: %q", one) + } + // The same tree and base, the base copied to another registry address: one source. + if again := fingerprintOf(t, anImage, "aaaa", "registry-b:5000"); again != one { + t.Fatalf("one source has two fingerprints: %s %s", one, again) + } + // The module's tree changed: another source. + if changed := fingerprintOf(t, anImage, "bbbb", "registry-a:5000"); changed == one { + t.Fatal("a changed tree kept its fingerprint") + } + // The base moved: another source. + moved := strings.Replace(anImage, "1111111111111111111111111111111111111111111111111111111111111111", + "2222222222222222222222222222222222222222222222222222222222222222", 1) + if changed := fingerprintOf(t, moved, "aaaa", "registry-a:5000"); changed == one { + t.Fatal("a moved base kept its fingerprint") + } +} diff --git a/internal/builder/source_test.go b/internal/builder/source_test.go index 39e97255..ce1e6541 100644 --- a/internal/builder/source_test.go +++ b/internal/builder/source_test.go @@ -14,20 +14,6 @@ const anImage = `{"module":"bus","version":"1", "artifacts":[{"name":"server","kind":"image","from":"Dockerfile"}]}, "resources":[{"id":"svc","type":"container","name":"bus","artifact":"server"}]}` -func fingerprintOf(t *testing.T, manifest, tree string, mirrored string) string { - t.Helper() - r, workspace := aRepository(t, manifest, map[string]string{"modules/bus/Dockerfile": "ARG BASE\nFROM ${BASE}"}) - r.contents["modules/bus/module.json"] = manifest - r.tree = tree - m := &mirroring{recorded: r, at: mirrored} - got, err := Build(context.Background(), r.run, m, "https://forge.invalid/catalogue.git", "modules/bus", "", workspace, - nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - return got.Source -} - // mirroring is a store that copies a vendor's image into the mesh's registry, at an address a test says. type mirroring struct { *recorded @@ -39,27 +25,6 @@ func (m *mirroring) MirrorImage(_ context.Context, from, repository string) (str return m.at + "/" + repository + "@" + digest, nil } -func TestASourceFingerprintNamesWhatABuildWasMadeFrom(t *testing.T) { - one := fingerprintOf(t, anImage, "aaaa", "registry-a:5000") - if !strings.HasPrefix(one, sourcePrefix) { - t.Fatalf("no fingerprint: %q", one) - } - // The same tree and base, the base copied to another registry address: one source. - if again := fingerprintOf(t, anImage, "aaaa", "registry-b:5000"); again != one { - t.Fatalf("one source has two fingerprints: %s %s", one, again) - } - // The module's tree changed: another source. - if changed := fingerprintOf(t, anImage, "bbbb", "registry-a:5000"); changed == one { - t.Fatal("a changed tree kept its fingerprint") - } - // The base moved: another source. - moved := strings.Replace(anImage, "1111111111111111111111111111111111111111111111111111111111111111", - "2222222222222222222222222222222222222222222222222222222222222222", 1) - if changed := fingerprintOf(t, moved, "aaaa", "registry-a:5000"); changed == one { - t.Fatal("a moved base kept its fingerprint") - } -} - func TestABuildTheRegistryDecidesHasNoFingerprint(t *testing.T) { s := newSourceInputs("app") s.tree = "aaaa" diff --git a/internal/builder/template_integration_test.go b/internal/builder/template_integration_test.go new file mode 100644 index 00000000..a8880e35 --- /dev/null +++ b/internal/builder/template_integration_test.go @@ -0,0 +1,103 @@ +//go:build integration + +package builder + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { + r, workspace := aRepository(t, templatedModule, templatedFiles) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if r.archives["meshboard/files"] == "" { + t.Fatalf("the templates archive was not published: %v", r.archives) + } + if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { + t.Fatal(problem) + } +} + +func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { + files := map[string]string{} + for k, v := range templatedFiles { + files[k] = v + } + delete(files, "files/a/deeper.conf") + r, workspace := aRepository(t, templatedModule, files) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { + t.Fatalf("a build whose template is missing did not fail naming it: %v", err) + } +} + +func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { + withVersion(t, "builder-1") + contents := map[string]string{"mod/" + ManifestName: templatedModule} + for k, v := range templatedFiles { + contents["mod/"+k] = v + } + scratch := newScratch() + r, run, workspace := aModuleWith(t, "tree-one", contents) + checked := aCheckBuiltBy(t, scratch, r, run, workspace) + + r, run, workspace = aModuleWith(t, "tree-one", contents) + got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, + "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if got.Reused == nil || len(r.archives) > 0 { + t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) + } + if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } +} + +func TestABuildReadsEachTemplateBackFromTheTemplatesDirectoryItsManifestNames(t *testing.T) { + files := map[string]string{"files/bin/meshboard-hook": "#!/bin/sh\n", "templates/meshboard.conf": "listen = 8080\n"} + r, workspace := aRepository(t, unpackingModule, files) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + if text := contentOf(t, got.Manifest, "conf"); text != files["templates/meshboard.conf"] { + t.Fatalf("the template's text is %q, not the file's", text) + } + if got.Manifest.Templates != "" { + t.Fatalf("the built manifest still names its templates directory: %q", got.Manifest.Templates) + } + // files/ packs exactly as it would with no templates beside it. + alone, err := pack(writeTree(t, t.TempDir(), map[string]string{"bin/meshboard-hook": "#!/bin/sh\n"})) + if err != nil { + t.Fatal(err) + } + sum := sha256.Sum256(alone) + packed := "" + for _, b := range got.Built { + if b.Name == "scripts" { + packed = b.Digest + } + } + if packed != "sha256:"+hex.EncodeToString(sum[:]) { + t.Fatalf("files/ packed to %q, not to what it packs to alone", packed) + } +} diff --git a/internal/builder/template_test.go b/internal/builder/template_test.go index c8c792b2..2bbf8979 100644 --- a/internal/builder/template_test.go +++ b/internal/builder/template_test.go @@ -1,9 +1,6 @@ package builder import ( - "context" - "crypto/sha256" - "encoding/hex" "os" "path/filepath" "strings" @@ -48,66 +45,8 @@ func contentOf(t *testing.T, m catalogue.Manifest, id string) string { return "" } -func TestABuildReadsEachTemplateBackFromItsArchive(t *testing.T) { - r, workspace := aRepository(t, templatedModule, templatedFiles) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } - if text := contentOf(t, got.Manifest, "deeper"); text != templatedFiles["files/a/deeper.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } - if r.archives["meshboard/files"] == "" { - t.Fatalf("the templates archive was not published: %v", r.archives) - } - if problem := catalogue.HeldTemplateProblem(got.Manifest); problem != "" { - t.Fatal(problem) - } -} - -func TestABuildWhoseTemplateIsMissingFailsNamingIt(t *testing.T) { - files := map[string]string{} - for k, v := range templatedFiles { - files[k] = v - } - delete(files, "files/a/deeper.conf") - r, workspace := aRepository(t, templatedModule, files) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err == nil || !strings.Contains(err.Error(), "files/a/deeper.conf") { - t.Fatalf("a build whose template is missing did not fail naming it: %v", err) - } -} - // A walk that takes its check's build copies the archive by digest and builds nothing: the templates are read back // from the tree packed again, which is the archive only when it packs to the digest taken. -func TestAWalkTakingAChecksBuildReadsTheTemplatesBackToo(t *testing.T) { - withVersion(t, "builder-1") - contents := map[string]string{"mod/" + ManifestName: templatedModule} - for k, v := range templatedFiles { - contents["mod/"+k] = v - } - scratch := newScratch() - r, run, workspace := aModuleWith(t, "tree-one", contents) - checked := aCheckBuiltBy(t, scratch, r, run, workspace) - - r, run, workspace = aModuleWith(t, "tree-one", contents) - got, err := BuildReusing(t.Context(), run, r, Reuse{Scratch: scratch, Pulls: []PullRef{aPull}, Named: named(checked)}, - "https://forge.invalid/meshboard.git", "mod", "main", workspace, nil, Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if got.Reused == nil || len(r.archives) > 0 { - t.Fatalf("the walk built again: reused %+v, why %q, published %v", got.Reused, got.Why, r.archives) - } - if text := contentOf(t, got.Manifest, "conf"); text != templatedFiles["files/meshboard.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } -} // The tree packed again must be the archive taken; text read from anything else is not the archive's. func TestTemplateTextsAreRefusedWhenTheTreeDoesNotPackToTheDigestTaken(t *testing.T) { @@ -148,34 +87,3 @@ const unpackingModule = `{"module":"meshboard","version":"1", "resources":[ {"id":"scripts","type":"archive","path":"/usr/local/lib/meshboard","artifact":"scripts"}, {"id":"conf","type":"file","path":"/etc/meshboard/meshboard.conf","template":"templates/meshboard.conf"}]}` - -func TestABuildReadsEachTemplateBackFromTheTemplatesDirectoryItsManifestNames(t *testing.T) { - files := map[string]string{"files/bin/meshboard-hook": "#!/bin/sh\n", "templates/meshboard.conf": "listen = 8080\n"} - r, workspace := aRepository(t, unpackingModule, files) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/meshboard.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - if text := contentOf(t, got.Manifest, "conf"); text != files["templates/meshboard.conf"] { - t.Fatalf("the template's text is %q, not the file's", text) - } - if got.Manifest.Templates != "" { - t.Fatalf("the built manifest still names its templates directory: %q", got.Manifest.Templates) - } - // files/ packs exactly as it would with no templates beside it. - alone, err := pack(writeTree(t, t.TempDir(), map[string]string{"bin/meshboard-hook": "#!/bin/sh\n"})) - if err != nil { - t.Fatal(err) - } - sum := sha256.Sum256(alone) - packed := "" - for _, b := range got.Built { - if b.Name == "scripts" { - packed = b.Digest - } - } - if packed != "sha256:"+hex.EncodeToString(sum[:]) { - t.Fatalf("files/ packed to %q, not to what it packs to alone", packed) - } -} diff --git a/internal/builder/upstream_file.go b/internal/builder/upstream_file.go index c4d3a730..5b3a0ebf 100644 --- a/internal/builder/upstream_file.go +++ b/internal/builder/upstream_file.go @@ -41,7 +41,8 @@ const maxUpstream = 512 << 20 // maxEntries is the most entries an upstream archive may hold: millions of empty ones would fill the build // seat's inodes and the node's. Betterbird's release holds 64. -const maxEntries = 100_000 +// It is a variable so that a test can lower it instead of writing a hundred thousand headers. +var maxEntries = 100_000 // noDowngrade refuses a redirect off https: the digest still guards the bytes, and a fetch in the clear says // to whoever is on the path what the build seat fetches. diff --git a/internal/builder/upstream_file_integration_test.go b/internal/builder/upstream_file_integration_test.go new file mode 100644 index 00000000..31a1dfad --- /dev/null +++ b/internal/builder/upstream_file_integration_test.go @@ -0,0 +1,180 @@ +//go:build integration + +package builder + +import ( + "archive/tar" + "bytes" + "context" + "os/exec" + "path/filepath" + "sort" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) { + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + digest, _ := got.Manifest.Resources[0]["digest"].(string) + if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest { + t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v", + got.Manifest.Resources[0], r.archives) + } + + packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"}, + func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + if "sha256:"+sumHex(packed) != digest { + t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed)) + } + entries := entriesOf(t, packed) + want := map[string]string{ + "application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--", + "betterbird": "#!/bin/sh\n -rwxr-xr-x", + "defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--", + } + if len(entries) != len(want) { + t.Fatalf("the archive holds %v, want %v", entries, want) + } + for name, w := range want { + if entries[name] != w { + t.Errorf("%s is %q in the archive, want %q", name, entries[name], w) + } + } +} + +func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) { + // The vendor of an add-on serves only its newest version at one address: a pin that was fetched once + // must still build after the address moved on. + body := release(t) + v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) + first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatal(err) + } + v.files["/release.tar.gz"] = []byte("the vendor's next release") + second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err != nil { + t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err) + } + if v.asked != 1 { + t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked) + } + if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] { + t.Fatal("two builds of one pin gave two digests") + } +} + +func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) { + pinned := release(t) + served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}})) + v := &vendor{files: map[string][]byte{"/release.tar.gz": served}} + base := serveVendor(t, v) + r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil) + _, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, + Npmrc{}, GitCredential{}, nil) + if err == nil { + t.Fatal("a download that is not the pinned file was built") + } + for _, digest := range []string{sumHex(pinned), sumHex(served)} { + if !strings.Contains(err.Error(), "sha256:"+digest) { + t.Errorf("the refusal does not name sha256:%s: %v", digest, err) + } + } + if len(r.archives) != 0 { + t.Fatalf("something was published: %v", r.archives) + } + kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*")) + if len(kept) != 0 { + t.Fatalf("the refused download was kept: %v", kept) + } +} + +func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) { + // Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it. + if _, err := exec.LookPath("xz"); err != nil { + t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)") + } + plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}}) + cmd := exec.Command("xz", "-zc") + cmd.Stdin = bytes.NewReader(plain) + compressed, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", + URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + var names []string + for n := range entriesOf(t, packed) { + names = append(names, n) + } + sort.Strings(names) + if strings.Join(names, ",") != "betterbird" { + t.Fatalf("the xz release unpacked to %v", names) + } +} + +func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) { + // git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be + // followed by bytes after the tar's end, which must be read before xz is waited for. + var raw bytes.Buffer + w := tar.NewWriter(&raw) + w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}}) + w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755}) + w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755}) + w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}) + w.Write([]byte("x")) + w.Close() + body := gzipped(t, raw.Bytes()) + v := &vendor{files: map[string][]byte{"/r.tar.gz": body}} + base := serveVendor(t, v) + packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", + URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {}) + if err != nil { + t.Fatal(err) + } + if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" { + t.Fatalf("%v", e) + } + if _, err := exec.LookPath("xz"); err != nil { + t.Skip("NOT TESTED here: no xz") + } + // A megabyte after the tar's end: more than a pipe holds. + trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...) + cmd := exec.Command("xz", "-zc") + cmd.Stdin = bytes.NewReader(trailing) + compressed, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + v.files["/r.tar.xz"] = compressed + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz", + SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil { + t.Fatal(err) + } +} diff --git a/internal/builder/upstream_file_test.go b/internal/builder/upstream_file_test.go index 34e82f1e..4cb07a49 100644 --- a/internal/builder/upstream_file_test.go +++ b/internal/builder/upstream_file_test.go @@ -15,13 +15,10 @@ import ( "net/http" "net/http/httptest" "os" - "os/exec" "path/filepath" "runtime" - "sort" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/catalogue" ) @@ -153,98 +150,6 @@ func pinnedManifest(base, sum string) string { "resources":[{"id":"installation","type":"archive","path":"/opt/betterbird","artifact":"archive"}]}` } -func TestAnArchivePinnedByURLIsFetchedCheckedAndPackedWithoutItsOwnDirectory(t *testing.T) { - body := release(t) - v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} - base := serveVendor(t, v) - r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) - got, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - digest, _ := got.Manifest.Resources[0]["digest"].(string) - if !strings.HasPrefix(digest, "sha256:") || r.archives["betterbird/archive"] != digest { - t.Fatalf("the release was not published and pinned as the module's archive: %v, published %v", - got.Manifest.Resources[0], r.archives) - } - - packed, err := upstreamArchive(context.Background(), workspace, catalogue.Artifact{Name: "archive", Kind: "archive", - URL: base + "/release.tar.gz", SHA256: sumHex(body), Version: "153.4.0esr-bb10", Within: "betterbird"}, - func(string, string, ...any) {}) - if err != nil { - t.Fatal(err) - } - if "sha256:"+sumHex(packed) != digest { - t.Fatalf("a second packing of one pin gave another digest: %s and %s", digest, sumHex(packed)) - } - entries := entriesOf(t, packed) - want := map[string]string{ - "application.ini": "[App]\nVersion=153.4.0\n -rw-r--r--", - "betterbird": "#!/bin/sh\n -rwxr-xr-x", - "defaults/pref/channel-prefs.js": "pref(\"app.update.channel\", \"default\");\n -rw-r--r--", - } - if len(entries) != len(want) { - t.Fatalf("the archive holds %v, want %v", entries, want) - } - for name, w := range want { - if entries[name] != w { - t.Errorf("%s is %q in the archive, want %q", name, entries[name], w) - } - } -} - -func TestAnUpstreamFileIsFetchedOnceAndKeptByItsDigest(t *testing.T) { - // The vendor of an add-on serves only its newest version at one address: a pin that was fetched once - // must still build after the address moved on. - body := release(t) - v := &vendor{files: map[string][]byte{"/release.tar.gz": body}} - base := serveVendor(t, v) - r, workspace := aRepository(t, pinnedManifest(base, sumHex(body)), nil) - first, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatal(err) - } - v.files["/release.tar.gz"] = []byte("the vendor's next release") - second, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err != nil { - t.Fatalf("a pin fetched before failed to build once the address served something else: %v", err) - } - if v.asked != 1 { - t.Fatalf("the vendor was asked %d times for one pin, want once", v.asked) - } - if first.Manifest.Resources[0]["digest"] != second.Manifest.Resources[0]["digest"] { - t.Fatal("two builds of one pin gave two digests") - } -} - -func TestAnUpstreamFileWhoseDownloadDoesNotMatchItsPinFailsAndNamesBothDigests(t *testing.T) { - pinned := release(t) - served := gzipped(t, tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "not what was reviewed", mode: 0o755}})) - v := &vendor{files: map[string][]byte{"/release.tar.gz": served}} - base := serveVendor(t, v) - r, workspace := aRepository(t, pinnedManifest(base, sumHex(pinned)), nil) - _, err := Build(context.Background(), r.run, r, "https://forge.invalid/catalogue.git", "", "", workspace, nil, - Npmrc{}, GitCredential{}, nil) - if err == nil { - t.Fatal("a download that is not the pinned file was built") - } - for _, digest := range []string{sumHex(pinned), sumHex(served)} { - if !strings.Contains(err.Error(), "sha256:"+digest) { - t.Errorf("the refusal does not name sha256:%s: %v", digest, err) - } - } - if len(r.archives) != 0 { - t.Fatalf("something was published: %v", r.archives) - } - kept, _ := filepath.Glob(filepath.Join(upstreamDir(workspace), "*")) - if len(kept) != 0 { - t.Fatalf("the refused download was kept: %v", kept) - } -} - func TestAnUpstreamFileThatIsNotAnArchiveIsKeptWholeUnderItsName(t *testing.T) { xpi := []byte("PK\x03\x04 an add-on") v := &vendor{files: map[string][]byte{"/owl.xpi": xpi}} @@ -282,78 +187,10 @@ func TestAnUpstreamArchiveHoldingALinkOrAnEntryOutsideItsDirectoryIsRefused(t *t } } -func TestAnUpstreamArchiveCompressedWithXzIsRead(t *testing.T) { - // Betterbird publishes its Linux release as .tar.xz; the build seat's image carries xz for it. - if _, err := exec.LookPath("xz"); err != nil { - t.Skip("NOT TESTED here: no xz on this machine (the build seat's image carries it)") - } - plain := tarOf(t, []tarEntry{{name: "betterbird/betterbird", body: "#!/bin/sh\n", mode: 0o755}}) - cmd := exec.Command("xz", "-zc") - cmd.Stdin = bytes.NewReader(plain) - compressed, err := cmd.Output() - if err != nil { - t.Fatal(err) - } - v := &vendor{files: map[string][]byte{"/r.tar.xz": compressed}} - base := serveVendor(t, v) - packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "archive", Kind: "archive", - URL: base + "/r.tar.xz", SHA256: sumHex(compressed), Version: "1", Within: "betterbird"}, func(string, string, ...any) {}) - if err != nil { - t.Fatal(err) - } - var names []string - for n := range entriesOf(t, packed) { - names = append(names, n) - } - sort.Strings(names) - if strings.Join(names, ",") != "betterbird" { - t.Fatalf("the xz release unpacked to %v", names) - } -} - -func TestAnUpstreamTarWithAGlobalHeaderNestedWithinAndTrailingDataIsRead(t *testing.T) { - // git archive writes a pax global header; a release may unpack under a/b with a/ listed first; xz may be - // followed by bytes after the tar's end, which must be read before xz is waited for. - var raw bytes.Buffer - w := tar.NewWriter(&raw) - w.WriteHeader(&tar.Header{Typeflag: tar.TypeXGlobalHeader, Name: "pax_global_header", PAXRecords: map[string]string{"comment": "abc"}}) - w.WriteHeader(&tar.Header{Name: "a/", Typeflag: tar.TypeDir, Mode: 0o755}) - w.WriteHeader(&tar.Header{Name: "a/b/", Typeflag: tar.TypeDir, Mode: 0o755}) - w.WriteHeader(&tar.Header{Name: "a/b/f", Typeflag: tar.TypeReg, Mode: 0o644, Size: 1}) - w.Write([]byte("x")) - w.Close() - body := gzipped(t, raw.Bytes()) - v := &vendor{files: map[string][]byte{"/r.tar.gz": body}} - base := serveVendor(t, v) - packed, err := upstreamArchive(context.Background(), t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", - URL: base + "/r.tar.gz", SHA256: sumHex(body), Version: "1", Within: "a/b"}, func(string, string, ...any) {}) - if err != nil { - t.Fatal(err) - } - if e := entriesOf(t, packed); len(e) != 1 || e["f"] == "" { - t.Fatalf("%v", e) - } - if _, err := exec.LookPath("xz"); err != nil { - t.Skip("NOT TESTED here: no xz") - } - // A megabyte after the tar's end: more than a pipe holds. - trailing := append(append([]byte{}, raw.Bytes()...), bytes.Repeat([]byte{0}, 1<<20)...) - cmd := exec.Command("xz", "-zc") - cmd.Stdin = bytes.NewReader(trailing) - compressed, err := cmd.Output() - if err != nil { - t.Fatal(err) - } - v.files["/r.tar.xz"] = compressed - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - if _, err := upstreamArchive(ctx, t.TempDir(), catalogue.Artifact{Name: "a", Kind: "archive", URL: base + "/r.tar.xz", - SHA256: sumHex(compressed), Version: "1", Within: "a/b"}, func(string, string, ...any) {}); err != nil { - t.Fatal(err) - } -} - func TestAnUpstreamArchiveWithTooManyEntriesIsRefused(t *testing.T) { + before := maxEntries + maxEntries = 100 + t.Cleanup(func() { maxEntries = before }) var raw bytes.Buffer w := tar.NewWriter(&raw) for i := 0; i <= maxEntries; i++ { diff --git a/internal/builder/xz_framing_integration_test.go b/internal/builder/xz_framing_integration_test.go new file mode 100644 index 00000000..8192e3e9 --- /dev/null +++ b/internal/builder/xz_framing_integration_test.go @@ -0,0 +1,40 @@ +//go:build integration + +package builder + +import ( + "bytes" + "context" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) { + // The streams this test builds are ones the reader decodes, so what the walk refuses below is the + // same framing the reader follows. + body := bytes.Repeat([]byte("betterbird "), 300) + whole := tarHolding(t, body) + half := len(whole) / 2 + for name, file := range map[string][]byte{ + "one stream of delta blocks": xzStream(whole, 512, true, -1), + "two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0), + xzStream(whole[half:], 100, false, -1)...), + } { + path := writeTemp(t, file) + if xzPath, err := exec.LookPath("xz"); err == nil { + if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil { + t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out) + } + } + into := t.TempDir() + if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil { + t.Fatalf("%s: %v", name, err) + } + got, err := os.ReadFile(filepath.Join(into, "data")) + if err != nil || !bytes.Equal(got, body) { + t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err) + } + } +} diff --git a/internal/builder/xz_framing_test.go b/internal/builder/xz_framing_test.go index a672cb80..558ded0f 100644 --- a/internal/builder/xz_framing_test.go +++ b/internal/builder/xz_framing_test.go @@ -9,7 +9,6 @@ import ( "errors" "hash/crc32" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -125,34 +124,6 @@ func writeTemp(t *testing.T, body []byte) string { return file } -func TestAnXzFileOfAFewDeltaBlocksAndTwoStreamsIsRead(t *testing.T) { - // The streams this test builds are ones the reader decodes, so what the walk refuses below is the - // same framing the reader follows. - body := bytes.Repeat([]byte("betterbird "), 300) - whole := tarHolding(t, body) - half := len(whole) / 2 - for name, file := range map[string][]byte{ - "one stream of delta blocks": xzStream(whole, 512, true, -1), - "two streams with padding": append(append(xzStream(whole[:half], 700, true, -1), 0, 0, 0, 0), - xzStream(whole[half:], 100, false, -1)...), - } { - path := writeTemp(t, file) - if xzPath, err := exec.LookPath("xz"); err == nil { - if out, err := exec.Command(xzPath, "-t", path).CombinedOutput(); err != nil { - t.Fatalf("%s: xz itself refuses the built stream: %v %s", name, err, out) - } - } - into := t.TempDir() - if err := unpackUpstream(context.Background(), path, into, "betterbird"); err != nil { - t.Fatalf("%s: %v", name, err) - } - got, err := os.ReadFile(filepath.Join(into, "data")) - if err != nil || !bytes.Equal(got, body) { - t.Fatalf("%s: unpacked %d bytes, want %d: %v", name, len(got), len(body), err) - } - } -} - func TestAnXzFileOfManyDeltaBlocksIsRefusedBeforeItIsDecoded(t *testing.T) { // Review 68 on pull request 264: `xz --block-size=1 --delta` of a 200 kB tar kept about 67 MiB live in // the reader, one delta decoder per block. Its index here lists one block, as a file crafted to pass a diff --git a/internal/catalogue/delivery_seat.go b/internal/catalogue/delivery_seat.go index 19a2d51b..dbc80faf 100644 --- a/internal/catalogue/delivery_seat.go +++ b/internal/catalogue/delivery_seat.go @@ -91,7 +91,9 @@ func deliveryVerbs() []Verb { Input: schema(map[string]string{"id": "the delivery's id (owner/repository@commit): the head reviewed", "reviewed-by": "who reviewed it, as who@node", "why": "the review's verdict in a line"}, []string{"id", "reviewed-by", "why"}), - // Added after the seat's holder shipped (novox/hq issue 913): optional until mesh-delivery serves it. + // Kept one more release, optional (novox/hq issue 1022): mesh-delivery no longer serves it, but the manifest + // it still has on the mesh does until its catalogue member merges, and a seat refuses a holder that serves + // a verb it does not promise. The next release removes this entry. Optional: true}, } } diff --git a/internal/catalogue/delivery_seat_test.go b/internal/catalogue/delivery_seat_test.go index 783ec06e..7d3a6193 100644 --- a/internal/catalogue/delivery_seat_test.go +++ b/internal/catalogue/delivery_seat_test.go @@ -134,7 +134,7 @@ func TestTheDeliverySeatPromisesTimesOptionally(t *testing.T) { } } -// The seat marks a reviewed head ready (novox/hq issue 913): `mark-ready`, with the delivery's id, who reviewed it +// The seat still promises `mark-ready`, optionally, for one more release (issue 1022); it was added by novox/hq issue 913: `mark-ready`, with the delivery's id, who reviewed it // and why, so a reviewer marks a pull request through one audited verb, label and comment in the order the forge // counts, instead of two raw forge calls. Added after the holder shipped, it is optional. func TestTheDeliverySeatPromisesMarkReadyOptionally(t *testing.T) { diff --git a/internal/catalogue/environment_into_integration_test.go b/internal/catalogue/environment_into_integration_test.go new file mode 100644 index 00000000..b41864ec --- /dev/null +++ b/internal/catalogue/environment_into_integration_test.go @@ -0,0 +1,91 @@ +//go:build integration + +package catalogue + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("no sh on this machine") + } + script := "PATH=/usr/bin:/bin\n" + composedPOSIX + "once=$PATH\n" + composedPOSIX + + `[ "$PATH" = "$once" ] || { echo "changed: $once -> $PATH"; exit 1; }` + "\n" + + `echo "$PATH"; echo "$GOPATH"` + out, err := exec.Command(sh, "-c", script).CombinedOutput() + if err != nil { + t.Fatalf("sourcing twice: %v\n%s", err, out) + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + if lines[0] != want { + t.Fatalf("PATH is %s, not %s", lines[0], want) + } + if lines[1] != "/home/op/go" { + t.Fatalf("GOPATH was not exported: %q", lines[1]) + } + // And an entry the account already has stays where it is, and once. + out, err = exec.Command(sh, "-c", "PATH=/opt/scripts:/usr/bin\n"+composedPOSIX+`echo "$PATH"`).CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + if got := strings.TrimSpace(string(out)); got != + "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/opt/scripts:/usr/bin:/opt/agent/bin" { + t.Fatalf("an entry already on PATH was added again or moved: %s", got) + } +} + +func TestTheServiceManagerReadsTheSystemdRenderingAsMeant(t *testing.T) { + generator := "/usr/lib/systemd/user-environment-generators/30-systemd-environment-d-generator" + if _, err := os.Stat(generator); err != nil { + t.Skip("NOT JUDGED: no environment.d generator on this machine, so the service manager's reading " + + "of the rendering is judged only where systemd runs; the rendering itself is held byte for byte " + + "by TestTheEnvironmentRendersForTheServiceManagerByteForByte") + } + const base = "/usr/bin:/bin" + read := func(conf string) map[string]string { + t.Helper() + config := t.TempDir() + if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil { + t.Fatal(err) + } + if conf != "" { + if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(conf), 0o644); err != nil { + t.Fatal(err) + } + } + cmd := exec.Command(generator) + cmd.Env = []string{"PATH=" + base, "HOME=" + config, "XDG_CONFIG_HOME=" + config} + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("%v\n%s", err, out) + } + vars := map[string]string{} + for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + vars[k] = v + } + } + return vars + } + machine, read50 := read(""), read(composedSystemd) + + // What the machine's own files do to PATH: nothing, or append to it. One that replaces or prepends + // leaves nothing this test can say about the mesh's file, and says so rather than guess. + added, ok := strings.CutPrefix(machine["PATH"], base) + if machine["PATH"] != "" && !ok { + t.Skipf("NOT JUDGED: this machine's own environment.d sets PATH to %s, not %s with something after it, so "+ + "what the mesh's file adds cannot be told apart from it", machine["PATH"], base) + } + want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + added + if read50["PATH"] != want || read50["GOPATH"] != "/home/op/go" { + t.Fatalf("the service manager read PATH=%s GOPATH=%s, not PATH=%s GOPATH=/home/op/go (the machine's own "+ + "files add %q to PATH)", read50["PATH"], read50["GOPATH"], want, added) + } +} diff --git a/internal/catalogue/environment_into_test.go b/internal/catalogue/environment_into_test.go index 1d428f5c..ca112e9b 100644 --- a/internal/catalogue/environment_into_test.go +++ b/internal/catalogue/environment_into_test.go @@ -1,9 +1,6 @@ package catalogue import ( - "os" - "os/exec" - "path/filepath" "strings" "testing" ) @@ -101,36 +98,6 @@ func TestTheEnvironmentRendersForTheServiceManagerByteForByte(t *testing.T) { // Sourcing twice changes nothing (ADR 0203 §3): a login shell that starts another reads the file // again, and a PATH that grew each time would be the symptom. Run by a real `sh`, because the claim // is about what a shell does with the file, not about what the file looks like. -func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) { - sh, err := exec.LookPath("sh") - if err != nil { - t.Skip("no sh on this machine") - } - script := "PATH=/usr/bin:/bin\n" + composedPOSIX + "once=$PATH\n" + composedPOSIX + - `[ "$PATH" = "$once" ] || { echo "changed: $once -> $PATH"; exit 1; }` + "\n" + - `echo "$PATH"; echo "$GOPATH"` - out, err := exec.Command(sh, "-c", script).CombinedOutput() - if err != nil { - t.Fatalf("sourcing twice: %v\n%s", err, out) - } - lines := strings.Split(strings.TrimSpace(string(out)), "\n") - want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" - if lines[0] != want { - t.Fatalf("PATH is %s, not %s", lines[0], want) - } - if lines[1] != "/home/op/go" { - t.Fatalf("GOPATH was not exported: %q", lines[1]) - } - // And an entry the account already has stays where it is, and once. - out, err = exec.Command(sh, "-c", "PATH=/opt/scripts:/usr/bin\n"+composedPOSIX+`echo "$PATH"`).CombinedOutput() - if err != nil { - t.Fatalf("%v\n%s", err, out) - } - if got := strings.TrimSpace(string(out)); got != - "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/opt/scripts:/usr/bin:/opt/agent/bin" { - t.Fatalf("an entry already on PATH was added again or moved: %s", got) - } -} // The environment.d rendering, read by the service manager's own generator where this machine has // one — the same reader an account's user manager runs, so the PATH it composes is the one asserted. @@ -143,54 +110,6 @@ func TestThePOSIXEnvironmentSourcedTwiceLeavesPATHAsOnce(t *testing.T) { // A machine without the generator — the build seat's toolchain image holds no systemd — cannot judge // this and says so: there the rendering is held byte for byte by // TestTheEnvironmentRendersForTheServiceManagerByteForByte, and this reading only where systemd runs. -func TestTheServiceManagerReadsTheSystemdRenderingAsMeant(t *testing.T) { - generator := "/usr/lib/systemd/user-environment-generators/30-systemd-environment-d-generator" - if _, err := os.Stat(generator); err != nil { - t.Skip("NOT JUDGED: no environment.d generator on this machine, so the service manager's reading " + - "of the rendering is judged only where systemd runs; the rendering itself is held byte for byte " + - "by TestTheEnvironmentRendersForTheServiceManagerByteForByte") - } - const base = "/usr/bin:/bin" - read := func(conf string) map[string]string { - t.Helper() - config := t.TempDir() - if err := os.MkdirAll(filepath.Join(config, "environment.d"), 0o755); err != nil { - t.Fatal(err) - } - if conf != "" { - if err := os.WriteFile(filepath.Join(config, "environment.d", "50-mesh.conf"), []byte(conf), 0o644); err != nil { - t.Fatal(err) - } - } - cmd := exec.Command(generator) - cmd.Env = []string{"PATH=" + base, "HOME=" + config, "XDG_CONFIG_HOME=" + config} - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("%v\n%s", err, out) - } - vars := map[string]string{} - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - if k, v, ok := strings.Cut(line, "="); ok { - vars[k] = v - } - } - return vars - } - machine, read50 := read(""), read(composedSystemd) - - // What the machine's own files do to PATH: nothing, or append to it. One that replaces or prepends - // leaves nothing this test can say about the mesh's file, and says so rather than guess. - added, ok := strings.CutPrefix(machine["PATH"], base) - if machine["PATH"] != "" && !ok { - t.Skipf("NOT JUDGED: this machine's own environment.d sets PATH to %s, not %s with something after it, so "+ - "what the mesh's file adds cannot be told apart from it", machine["PATH"], base) - } - want := "/home/op/go/bin:/usr/local/go/bin:/home/op/.local/bin:/home/op/bin:/usr/bin:/bin:/opt/agent/bin:/opt/scripts" + added - if read50["PATH"] != want || read50["GOPATH"] != "/home/op/go" { - t.Fatalf("the service manager read PATH=%s GOPATH=%s, not PATH=%s GOPATH=/home/op/go (the machine's own "+ - "files add %q to PATH)", read50["PATH"], read50["GOPATH"], want, added) - } -} // Nothing contributed renders nothing, in both formats — not an empty `export PATH`. func TestNoContributionsRenderNothing(t *testing.T) { diff --git a/internal/catalogue/mounts_test.go b/internal/catalogue/mounts_test.go index d0e94a84..f929a5d3 100644 --- a/internal/catalogue/mounts_test.go +++ b/internal/catalogue/mounts_test.go @@ -75,8 +75,9 @@ func TestEveryCatalogueManifestDeclaresWhatItMounts(t *testing.T) { if len(files) == 0 { t.Fatal("no manifests in the catalogue, so this proved nothing") } + root := beside.Dir(t, "mesh-catalog") // once: it copies the captured tree for _, file := range files { - if _, err := ReadTreeManifest(beside.Dir(t, "mesh-catalog"), filepath.Dir(file)); err != nil { + if _, err := ReadTreeManifest(root, filepath.Dir(file)); err != nil { t.Errorf("%s: %v", filepath.Base(filepath.Dir(file)), err) } } diff --git a/internal/catalogue/verbs.go b/internal/catalogue/verbs.go index 48b62765..a9c1b774 100644 --- a/internal/catalogue/verbs.go +++ b/internal/catalogue/verbs.go @@ -149,12 +149,12 @@ var ControllerVerbs = []Verb{ "paths": "the files it changes, comma-separated, from the repository's root", "module-dirs": "the directories holding a module.json at the head, comma-separated, as the forge says them", "removed": "the files among paths it deletes, comma-separated"}, []string{"repository", "paths"})}, - {Name: "delivery-order", Description: "A delivery group's order (novox/hq ADR 0239): its members in the order " + - "they are delivered, every pair and why — declared, built by, version skew, engine before controller — and " + - "the rules each pair won over by precedence (ADR 0249); the cycle, and each contradiction no precedence " + - "resolves with how to declare the order, when the pairs contradict each other.", + {Name: "delivery-order", Description: "A delivery group's order (novox/hq ADR 0419): its members in the order " + + "they are delivered, every pair and why — built by, verb before caller, version skew, engine before " + + "controller — read from the module graph and the manifests' change alone; no `after:` line is read. The " + + "cycle, with every module and edge of it, is the one refusal.", Input: schema(map[string]string{"members": "the members, as JSON: [{id, repository, base, head, number, paths, " + - "module_dirs, module_dirs_said, removed, after}]"}, []string{"members"})}, + "module_dirs, module_dirs_said, removed}]"}, []string{"members"})}, {Name: "delivery-check", Description: "Ask the build seat for a delivery group's composed check (novox/hq ADR " + "0239): every member's head laid over the mesh in turn and judged as one future state. Answers the ask's id; " + "the verdict is said as `checked` with the group's id. With one member and no group, that head is checked " + diff --git a/internal/conditions/bus_test.go b/internal/conditions/bus_test.go index 7f945c83..4526dd5b 100644 --- a/internal/conditions/bus_test.go +++ b/internal/conditions/bus_test.go @@ -1,3 +1,5 @@ +//go:build integration + package conditions import ( diff --git a/internal/conditions/store.go b/internal/conditions/store.go index 0311d60d..c9ba4fd9 100644 --- a/internal/conditions/store.go +++ b/internal/conditions/store.go @@ -53,14 +53,15 @@ const Seat = "mesh-controller" // Keeper raises, observes, silences and clears conditions, and says each transition. type Keeper struct { - store Backend - looks LookStore - history History - teller Teller - now func() time.Time - say func(format string, args ...any) - changed func() - epoch func() (uint64, error) + store Backend + looks LookStore + history History + teller Teller + now func() time.Time + say func(format string, args ...any) + changed func() + epoch func() (uint64, error) + tellBackoff time.Duration mu sync.Mutex // cleared is when each recently cleared condition cleared and how often it had been raised, so @@ -108,6 +109,9 @@ type Options struct { // Epoch is the controller lease this keeper writes under (to-be 45 §6): every write carries its // epoch, and none is made while it is not held. Nil writes with no epoch (a test, the memory store). Epoch func() (uint64, error) + // TellBackoff is the first wait before a transition the bus would not take is offered again; it doubles each + // time. Zero is 200 ms; a test sets it short so that it need not wait. + TellBackoff time.Duration } // TellFor is how long one transition is offered to the bus before it is said lost. @@ -117,7 +121,7 @@ var TellFor = 10 * time.Minute // that cleared just before this controller started and is raised again now is a reopening. func NewKeeper(ctx context.Context, o Options) *Keeper { k := &Keeper{store: o.Store, looks: o.Looks, history: o.History, teller: o.Teller, now: o.Now, say: o.Say, changed: o.Changed, - epoch: o.Epoch, + epoch: o.Epoch, tellBackoff: o.TellBackoff, cleared: map[string]clearing{}, reworded: map[string]bool{}, out: make(chan Event, 1024), drained: make(chan struct{})} if k.now == nil { k.now = time.Now @@ -713,7 +717,10 @@ func (k *Keeper) telling() { continue } deadline := time.Now().Add(TellFor) - wait := 200 * time.Millisecond + wait := k.tellBackoff + if wait <= 0 { + wait = 200 * time.Millisecond + } kept, said := k.history == nil, k.teller == nil for { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/internal/conditions/store_test.go b/internal/conditions/store_test.go index f1d49a28..a1f58869 100644 --- a/internal/conditions/store_test.go +++ b/internal/conditions/store_test.go @@ -370,12 +370,12 @@ func TestTheEventShapeIsTheContract(t *testing.T) { func TestATransitionIsOfferedAgainWhileTheBusIsAway(t *testing.T) { store, told, c := NewInMemory(), &Told{}, newClock() told.SetFail(errors.New("no responders")) - k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now}) + k := NewKeeper(t.Context(), Options{Store: store, History: store, Teller: told, Now: c.now, TellBackoff: 5 * time.Millisecond}) defer k.Close(context.Background()) if _, err := k.Observe(t.Context(), silent("ace")); err != nil { t.Fatal(err) } - time.Sleep(300 * time.Millisecond) + time.Sleep(30 * time.Millisecond) told.SetFail(nil) said := settled(t, told, 1) if said[0].Key != "machine.ace.silent" || k.Unsaid() != 0 { diff --git a/internal/identity/authority_test.go b/internal/identity/authority_test.go index d14cae06..c674316f 100644 --- a/internal/identity/authority_test.go +++ b/internal/identity/authority_test.go @@ -1,3 +1,5 @@ +//go:build integration + package identity import ( diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go index 7aa15825..bacf1d62 100644 --- a/internal/identity/identity_test.go +++ b/internal/identity/identity_test.go @@ -1,3 +1,5 @@ +//go:build integration + package identity import ( diff --git a/internal/inventory/adoption_test.go b/internal/inventory/adoption_test.go index c779d322..b2f83741 100644 --- a/internal/inventory/adoption_test.go +++ b/internal/inventory/adoption_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/agent_account_test.go b/internal/inventory/agent_account_test.go index 44139bb8..f7ccab1a 100644 --- a/internal/inventory/agent_account_test.go +++ b/internal/inventory/agent_account_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/artifacts_test.go b/internal/inventory/artifacts_test.go index 5deef9cc..3160866e 100644 --- a/internal/inventory/artifacts_test.go +++ b/internal/inventory/artifacts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/bindings_test.go b/internal/inventory/bindings_test.go index 4767cbf1..9e1e778f 100644 --- a/internal/inventory/bindings_test.go +++ b/internal/inventory/bindings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/builds_test.go b/internal/inventory/builds_test.go index 8608ff0b..fbdfd08a 100644 --- a/internal/inventory/builds_test.go +++ b/internal/inventory/builds_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/busrecords_integration_test.go b/internal/inventory/busrecords_integration_test.go new file mode 100644 index 00000000..02fad1cf --- /dev/null +++ b/internal/inventory/busrecords_integration_test.go @@ -0,0 +1,316 @@ +//go:build integration + +package inventory + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/catalogue" + "golang.org/x/crypto/bcrypt" +) + +func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) { + t.Helper() + inv := ForTest(t) + ctx := context.Background() + for _, m := range manifests { + if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil { + t.Fatal(err) + } + } + return inv, ctx +} + +func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) { + // It declares where its account is delivered: a module with no own secret named broker can never + // be issued one, and is no user at all (novox/hq issue 195) — the case asserted below. + shop := catalogue.Manifest{ + Module: "shop", Version: "1", + Emits: []string{"order.placed"}, Tools: []string{"price"}, + Uses: []string{"telegram-sender"}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, + } + quiet := catalogue.Manifest{Module: "quiet", Version: "1", Emits: []string{"thing.happened"}} + inv, ctx := aMeshWith(t, theSeatDeclarer(), shop, quiet) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + for _, module := range []string{"shop", "quiet"} { + if _, err := inv.Assign(ctx, "one", module); err != nil { + t.Fatal(err) + } + } + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + var on []broker.Declared + for _, d := range records.Assigned["one"] { + if d.Module == "shop" { + on = append(on, d) + } + } + if len(on) != 1 || on[0].NoAccount { + t.Fatalf("the machine's modules read as %+v", records.Assigned["one"]) + } + if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" { + t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+ + "seat it was assigned to send to", on[0].Uses) + } + if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" { + t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+ + "serves nothing", on[0].Serves) + } + + // And it derives into a user the server would accept. + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var found bool + for _, u := range users { + if u.Username() != "one.shop" { + continue + } + found = true + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + // Its tools are every one under its own name — the list in the manifest is a person's + // vocabulary for asking, not the module's permission to answer. + if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") || + !granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") || + !granted(perms.Subscribe, "mesh.mod.shop.tool.>") { + t.Fatalf("one.shop's authority is not what it declared: %+v", perms) + } + } + if !found { + t.Fatal("no user was derived for the assigned module") + } + for _, u := range users { + if u.Username() == "one.quiet" { + t.Fatal("a module with nowhere to read an account was made a user (novox/hq issue 195)") + } + } +} + +func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) { + inv, ctx := aMeshWith(t) + for _, name := range []string{"live", "expired", "none"} { + if _, err := inv.AddNode(ctx, name); err != nil { + t.Fatal(err) + } + } + if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil { + t.Fatal(err) + } + // Briefly, then waited out: a token with no lifetime is refused at issue, which is the right + // refusal and leaves this as the way to have an expired one. + if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + if strings.Join(records.Enrolling, ",") != "live" { + t.Fatalf("machines with a live token read as %v", records.Enrolling) + } +} + +func TestIssuingATokenRecordsTheAccountItIsThePasswordOf(t *testing.T) { + inv, ctx := aMeshWith(t) + if _, err := inv.AddNode(ctx, "joiner"); err != nil { + t.Fatal(err) + } + issued, err := inv.IssueToken(ctx, "joiner", time.Hour) + if err != nil { + t.Fatal(err) + } + + name := broker.Principal{Kind: broker.KindEnrolment, Node: "joiner"}.Username() + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + user, has := users[name] + if !has { + t.Fatalf("no bus account for %q; the composer would leave the enrolment out and the "+ + "machine would be refused before the mesh heard of it: %v", name, users) + } + if user.Kind != BusEnrolment || user.Node != "joiner" { + t.Errorf("the account is %+v, not this node's enrolment", user) + } + if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(issued.Secret)); err != nil { + t.Error("the account does not accept the secret the token carries, so presenting the " + + "token would be refused by the server") + } + + // And the composition contains it, which is the thing the server reads. + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + derived, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + hashes := map[string]string{} + for n, u := range users { + hashes[n] = u.PasswordHash + } + _, missing := broker.WithPasswords(derived, hashes) + for _, m := range missing { + if m == name { + t.Fatal("the enrolment user is composed without a password, which is a user nobody can be") + } + } +} + +func TestTheBussOwnModuleBecomesTheSnapshotUser(t *testing.T) { + bus := catalogue.Manifest{ + Module: "nats", Version: "1", + Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}}, + Claims: []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}, + Tools: []string{"nats_streams"}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, + } + shop := catalogue.Manifest{Module: "shop", Version: "1", Emits: []string{"order.placed"}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}} + inv, ctx := aMeshWith(t, bus, shop) + if _, err := inv.AddNode(ctx, "one"); err != nil { + t.Fatal(err) + } + for _, module := range []string{"nats", "shop"} { + if _, err := inv.Assign(ctx, "one", module); err != nil { + t.Fatal(err) + } + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + seen := map[string]bool{} + for _, u := range users { + perms, err := broker.PermissionsFor(u) + if err != nil { + t.Fatal(err) + } + snapshots := granted(perms.Publish, "$JS.API.STREAM.SNAPSHOT.*") + switch u.Username() { + case "one.nats": + seen[u.Username()] = true + if !snapshots || len(perms.Publish) != len(broker.BusSnapshotGrants.Publish) { + t.Fatalf("the bus's own user is granted %v, not the snapshot API alone", perms.Publish) + } + case "one.shop": + seen[u.Username()] = true + if snapshots { + t.Fatal("a module that is not the bus may snapshot it") + } + } + } + if !seen["one.nats"] || !seen["one.shop"] { + t.Fatalf("users derived: %v", seen) + } +} + +func TestABarIsGrantedTheStateItsMachinesModulesOfferIt(t *testing.T) { + bar := catalogue.Manifest{Module: "a-bar", Version: "1", + Claims: []catalogue.Claim{{Name: catalogue.BarSeat, Scope: catalogue.ScopeNode}}} + power := catalogue.Manifest{Module: "power", Version: "1", + State: []catalogue.StateDeclaration{{Name: "draw"}}, + Contributions: []catalogue.SeatContribution{{Seat: catalogue.BarSeat, Kind: catalogue.BarKindBlock, + Data: map[string]any{"bar": "bottom", "place": "status", "shows": "state", + "options": map[string]any{"state": "power.draw"}}}}} + inv, ctx := aMeshWith(t, bar, power) + for node, modules := range map[string][]string{"laptop": {"a-bar", "power"}, "desk": {"a-bar"}} { + if _, err := inv.AddNode(ctx, node); err != nil { + t.Fatal(err) + } + for _, module := range modules { + if _, err := inv.Assign(ctx, node, module); err != nil { + t.Fatal(err) + } + } + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + reads := func(node, module string) []broker.KeyedRead { + for _, d := range records.Assigned[node] { + if d.Module == module { + if len(d.Reads) != 0 { + t.Fatalf("%s on %s reads whole buckets: %v", module, node, d.Reads) + } + return d.KeyedReads + } + } + t.Fatalf("%s is not on %s", module, node) + return nil + } + // The laptop's own key, and no other (novox/hq ADR 0260). + if got := reads("laptop", "a-bar"); len(got) != 1 || got[0] != (broker.KeyedRead{Read: "power.draw", Key: "laptop"}) { + t.Fatalf("the laptop's bar reads %v", got) + } + if got := reads("desk", "a-bar"); len(got) != 0 { + t.Fatalf("a bar with no power beside it reads %v", got) + } + if got := reads("laptop", "power"); len(got) != 0 { + t.Fatalf("power reads %v", got) + } +} + +func TestTheProvisionerClientsMachineHasAnAccountOfItsOwn(t *testing.T) { + provisioner := catalogue.Manifest{ + Module: "mesh-provisioner", Version: "1", + DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode, + Serves: []catalogue.Verb{{Name: "get"}}}}, + Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}}, + OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}, + broker.ProvisionerSecret: {Path: "/run/provisioner-broker"}}, + } + unread := provisioner + unread.Module = "provisioner-unread" + unread.DefinesSeats = nil + unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} + inv, ctx := aMeshWith(t, provisioner, unread) + for node, module := range map[string]string{"one": "mesh-provisioner", "two": "provisioner-unread"} { + if _, err := inv.AddNode(ctx, node); err != nil { + t.Fatal(err) + } + if _, err := inv.Assign(ctx, node, module); err != nil { + t.Fatal(err) + } + } + records, err := inv.BusRecords(ctx) + if err != nil { + t.Fatal(err) + } + users, err := broker.Users(records) + if err != nil { + t.Fatal(err) + } + var clients []string + for _, u := range users { + if u.Kind == broker.KindProvisioner { + clients = append(clients, u.Username()+" for "+u.Module) + } + } + if strings.Join(clients, ",") != "provisioner.one for mesh-provisioner" { + t.Fatalf("the provisioner clients' accounts are %v, want one for the machine whose module reads it", clients) + } +} diff --git a/internal/inventory/busrecords_test.go b/internal/inventory/busrecords_test.go index 4f1b3f93..3dd19b98 100644 --- a/internal/inventory/busrecords_test.go +++ b/internal/inventory/busrecords_test.go @@ -1,14 +1,11 @@ package inventory import ( - "context" "strings" "testing" - "time" "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/catalogue" - "golang.org/x/crypto/bcrypt" ) // Reading the bus's user list out of the mesh's records, against a real store. @@ -17,18 +14,6 @@ import ( // the server reads whatever file it is given, and a module whose user is absent fails on its first // publish with an authorisation error that says nothing about a missing assignment. -func aMeshWith(t *testing.T, manifests ...catalogue.Manifest) (*Inventory, context.Context) { - t.Helper() - inv := ForTest(t) - ctx := context.Background() - for _, m := range manifests { - if err := inv.RegisterModule(ctx, m, Source{Repository: "/r"}); err != nil { - t.Fatal(err) - } - } - return inv, ctx -} - func theSeatDeclarer() catalogue.Manifest { return catalogue.Manifest{ Module: "telegram", Version: "1", @@ -41,108 +26,9 @@ func theSeatDeclarer() catalogue.Manifest { // A module assigned to a machine becomes a user with the authority its manifest declared — and the // protocol of a seat declared by a *different* module, which is the whole reason a seat exists. -func TestAnAssignedModuleBecomesAUserWithWhatItDeclared(t *testing.T) { - // It declares where its account is delivered: a module with no own secret named broker can never - // be issued one, and is no user at all (novox/hq issue 195) — the case asserted below. - shop := catalogue.Manifest{ - Module: "shop", Version: "1", - Emits: []string{"order.placed"}, Tools: []string{"price"}, - Uses: []string{"telegram-sender"}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, - } - quiet := catalogue.Manifest{Module: "quiet", Version: "1", Emits: []string{"thing.happened"}} - inv, ctx := aMeshWith(t, theSeatDeclarer(), shop, quiet) - if _, err := inv.AddNode(ctx, "one"); err != nil { - t.Fatal(err) - } - for _, module := range []string{"shop", "quiet"} { - if _, err := inv.Assign(ctx, "one", module); err != nil { - t.Fatal(err) - } - } - - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - var on []broker.Declared - for _, d := range records.Assigned["one"] { - if d.Module == "shop" { - on = append(on, d) - } - } - if len(on) != 1 || on[0].NoAccount { - t.Fatalf("the machine's modules read as %+v", records.Assigned["one"]) - } - if len(on[0].Uses) != 1 || on[0].Uses[0].Accepts[0] != "send" { - t.Fatalf("the seat it uses carries no protocol: %+v — so it would be granted nothing on a "+ - "seat it was assigned to send to", on[0].Uses) - } - if len(on[0].Serves) != 1 || on[0].Serves[0] != "price" { - t.Fatalf("its tools read as %v, and a module that cannot subscribe its own tool subject "+ - "serves nothing", on[0].Serves) - } - - // And it derives into a user the server would accept. - users, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - var found bool - for _, u := range users { - if u.Username() != "one.shop" { - continue - } - found = true - perms, err := broker.PermissionsFor(u) - if err != nil { - t.Fatal(err) - } - // Its tools are every one under its own name — the list in the manifest is a person's - // vocabulary for asking, not the module's permission to answer. - if !granted(perms.Publish, "mesh.mod.shop.event.order.placed") || - !granted(perms.Publish, "mesh.seat.telegram-sender.accept.send") || - !granted(perms.Subscribe, "mesh.mod.shop.tool.>") { - t.Fatalf("one.shop's authority is not what it declared: %+v", perms) - } - } - if !found { - t.Fatal("no user was derived for the assigned module") - } - for _, u := range users { - if u.Username() == "one.quiet" { - t.Fatal("a module with nowhere to read an account was made a user (novox/hq issue 195)") - } - } -} // A machine holding a live token gets an enrolment user; one whose token is spent or expired does // not. **An enrolment user outliving its token is a right to join that nobody issued.** -func TestOnlyAMachineWithALiveTokenHasAnEnrolmentUser(t *testing.T) { - inv, ctx := aMeshWith(t) - for _, name := range []string{"live", "expired", "none"} { - if _, err := inv.AddNode(ctx, name); err != nil { - t.Fatal(err) - } - } - if _, err := inv.IssueToken(ctx, "live", time.Hour); err != nil { - t.Fatal(err) - } - // Briefly, then waited out: a token with no lifetime is refused at issue, which is the right - // refusal and leaves this as the way to have an expired one. - if _, err := inv.IssueToken(ctx, "expired", 10*time.Millisecond); err != nil { - t.Fatal(err) - } - time.Sleep(50 * time.Millisecond) - - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - if strings.Join(records.Enrolling, ",") != "live" { - t.Fatalf("machines with a live token read as %v", records.Enrolling) - } -} // A module assigned and absent from the catalogue is refused rather than composed with no authority. // @@ -193,199 +79,14 @@ func granted(all []string, one string) bool { // exactly that string. So this checks the two halves that make the account usable — that a row // exists under the name the composer asks for, and that the secret handed out is what that row // accepts. -func TestIssuingATokenRecordsTheAccountItIsThePasswordOf(t *testing.T) { - inv, ctx := aMeshWith(t) - if _, err := inv.AddNode(ctx, "joiner"); err != nil { - t.Fatal(err) - } - issued, err := inv.IssueToken(ctx, "joiner", time.Hour) - if err != nil { - t.Fatal(err) - } - - name := broker.Principal{Kind: broker.KindEnrolment, Node: "joiner"}.Username() - users, err := inv.BusUsers(ctx) - if err != nil { - t.Fatal(err) - } - user, has := users[name] - if !has { - t.Fatalf("no bus account for %q; the composer would leave the enrolment out and the "+ - "machine would be refused before the mesh heard of it: %v", name, users) - } - if user.Kind != BusEnrolment || user.Node != "joiner" { - t.Errorf("the account is %+v, not this node's enrolment", user) - } - if err := bcrypt.CompareHashAndPassword([]byte(user.PasswordHash), []byte(issued.Secret)); err != nil { - t.Error("the account does not accept the secret the token carries, so presenting the " + - "token would be refused by the server") - } - - // And the composition contains it, which is the thing the server reads. - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - derived, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - hashes := map[string]string{} - for n, u := range users { - hashes[n] = u.PasswordHash - } - _, missing := broker.WithPasswords(derived, hashes) - for _, m := range missing { - if m == name { - t.Fatal("the enrolment user is composed without a password, which is a user nobody can be") - } - } -} // The module holding mesh-broker is the bus, and its user is the bus's own: granted the snapshot API // for the night's backup and nothing else, whatever tools it declares (novox/hq ADR 0235). Another // module on the same machine is granted none of it. -func TestTheBussOwnModuleBecomesTheSnapshotUser(t *testing.T) { - bus := catalogue.Manifest{ - Module: "nats", Version: "1", - Provides: []catalogue.Offer{{Name: "mesh-bus", Scope: catalogue.ScopeMesh}}, - Claims: []catalogue.Claim{{Name: catalogue.BrokerSeat, Scope: catalogue.ScopeMesh}}, - Tools: []string{"nats_streams"}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}, - } - shop := catalogue.Manifest{Module: "shop", Version: "1", Emits: []string{"order.placed"}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}}} - inv, ctx := aMeshWith(t, bus, shop) - if _, err := inv.AddNode(ctx, "one"); err != nil { - t.Fatal(err) - } - for _, module := range []string{"nats", "shop"} { - if _, err := inv.Assign(ctx, "one", module); err != nil { - t.Fatal(err) - } - } - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - users, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - seen := map[string]bool{} - for _, u := range users { - perms, err := broker.PermissionsFor(u) - if err != nil { - t.Fatal(err) - } - snapshots := granted(perms.Publish, "$JS.API.STREAM.SNAPSHOT.*") - switch u.Username() { - case "one.nats": - seen[u.Username()] = true - if !snapshots || len(perms.Publish) != len(broker.BusSnapshotGrants.Publish) { - t.Fatalf("the bus's own user is granted %v, not the snapshot API alone", perms.Publish) - } - case "one.shop": - seen[u.Username()] = true - if snapshots { - t.Fatal("a module that is not the bus may snapshot it") - } - } - } - if !seen["one.nats"] || !seen["one.shop"] { - t.Fatalf("users derived: %v", seen) - } -} // The bar's holder reads the state the modules beside it offer in its blocks (novox/hq ADR 0255), // without naming them: granted on the machine where both are, and nowhere else. -func TestABarIsGrantedTheStateItsMachinesModulesOfferIt(t *testing.T) { - bar := catalogue.Manifest{Module: "a-bar", Version: "1", - Claims: []catalogue.Claim{{Name: catalogue.BarSeat, Scope: catalogue.ScopeNode}}} - power := catalogue.Manifest{Module: "power", Version: "1", - State: []catalogue.StateDeclaration{{Name: "draw"}}, - Contributions: []catalogue.SeatContribution{{Seat: catalogue.BarSeat, Kind: catalogue.BarKindBlock, - Data: map[string]any{"bar": "bottom", "place": "status", "shows": "state", - "options": map[string]any{"state": "power.draw"}}}}} - inv, ctx := aMeshWith(t, bar, power) - for node, modules := range map[string][]string{"laptop": {"a-bar", "power"}, "desk": {"a-bar"}} { - if _, err := inv.AddNode(ctx, node); err != nil { - t.Fatal(err) - } - for _, module := range modules { - if _, err := inv.Assign(ctx, node, module); err != nil { - t.Fatal(err) - } - } - } - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - reads := func(node, module string) []broker.KeyedRead { - for _, d := range records.Assigned[node] { - if d.Module == module { - if len(d.Reads) != 0 { - t.Fatalf("%s on %s reads whole buckets: %v", module, node, d.Reads) - } - return d.KeyedReads - } - } - t.Fatalf("%s is not on %s", module, node) - return nil - } - // The laptop's own key, and no other (novox/hq ADR 0260). - if got := reads("laptop", "a-bar"); len(got) != 1 || got[0] != (broker.KeyedRead{Read: "power.draw", Key: "laptop"}) { - t.Fatalf("the laptop's bar reads %v", got) - } - if got := reads("desk", "a-bar"); len(got) != 0 { - t.Fatalf("a bar with no power beside it reads %v", got) - } - if got := reads("laptop", "power"); len(got) != 0 { - t.Fatalf("power reads %v", got) - } -} // A module running the machine's provisioner client gives that machine a client account of its own, apart from the // module's and the tool runner's (novox/hq ADR 0349 clause 11, design 57 P3); one with nowhere to read the account // gives none. -func TestTheProvisionerClientsMachineHasAnAccountOfItsOwn(t *testing.T) { - provisioner := catalogue.Manifest{ - Module: "mesh-provisioner", Version: "1", - DefinesSeats: []catalogue.SeatDeclaration{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode, - Serves: []catalogue.Verb{{Name: "get"}}}}, - Claims: []catalogue.Claim{{Name: broker.NodeProvisionerSeat, Scope: catalogue.ScopeNode}}, - OwnSecrets: catalogue.OwnSecrets{"broker": {Path: "/run/broker"}, - broker.ProvisionerSecret: {Path: "/run/provisioner-broker"}}, - } - unread := provisioner - unread.Module = "provisioner-unread" - unread.DefinesSeats = nil - unread.OwnSecrets = catalogue.OwnSecrets{"broker": {Path: "/run/broker"}} - inv, ctx := aMeshWith(t, provisioner, unread) - for node, module := range map[string]string{"one": "mesh-provisioner", "two": "provisioner-unread"} { - if _, err := inv.AddNode(ctx, node); err != nil { - t.Fatal(err) - } - if _, err := inv.Assign(ctx, node, module); err != nil { - t.Fatal(err) - } - } - records, err := inv.BusRecords(ctx) - if err != nil { - t.Fatal(err) - } - users, err := broker.Users(records) - if err != nil { - t.Fatal(err) - } - var clients []string - for _, u := range users { - if u.Kind == broker.KindProvisioner { - clients = append(clients, u.Username()+" for "+u.Module) - } - } - if strings.Join(clients, ",") != "provisioner.one for mesh-provisioner" { - t.Fatalf("the provisioner clients' accounts are %v, want one for the machine whose module reads it", clients) - } -} diff --git a/internal/inventory/bususers_integration_test.go b/internal/inventory/bususers_integration_test.go new file mode 100644 index 00000000..390a8ac4 --- /dev/null +++ b/internal/inventory/bususers_integration_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package inventory + +import ( + "context" + "testing" + + "golang.org/x/crypto/bcrypt" +) + +func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + password, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if password == "" { + t.Fatal("no password came back, so nothing can be sealed to the module") + } + + hash, known, err := inv.BusUserHash(ctx, "one.shop") + if err != nil || !known { + t.Fatalf("the user was not recorded: %v %v", known, err) + } + if hash == password { + t.Fatal("the store holds the password itself") + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil { + t.Fatalf("the recorded hash does not verify the password it was made from: %v", err) + } +} + +func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + first, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + second, err := inv.MintBusPassword(ctx, aBusUser("shop")) + if err != nil { + t.Fatal(err) + } + if first == second { + t.Fatal("minting twice produced the same password") + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if len(users) != 1 { + t.Fatalf("%d users after two mints for one name", len(users)) + } + hash := users["one.shop"].PasswordHash + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil { + t.Fatal("the kept hash is not the newest password's") + } + if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil { + t.Fatal("the previous password still verifies, so a rotation revoked nothing") + } +} + +func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + for _, u := range []BusUser{ + {Username: "node.one", Kind: BusNode, Node: "one"}, + aBusUser("shop"), + {Username: "node.two", Kind: BusNode, Node: "two"}, + {Username: "controller", Kind: BusController}, + } { + if _, err := inv.MintBusPassword(ctx, u); err != nil { + t.Fatal(err) + } + } + if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil { + t.Fatal(err) + } + users, err := inv.BusUsers(ctx) + if err != nil { + t.Fatal(err) + } + if _, still := users["node.one"]; still { + t.Fatal("a forgotten node's host credential still works") + } + if _, still := users["one.shop"]; still { + t.Fatal("a module on a forgotten node still has a credential") + } + // And nothing else went with it: the controller has no node, and another machine's user is + // another machine's. + for _, kept := range []string{"node.two", "controller"} { + if _, ok := users[kept]; !ok { + t.Fatalf("%s was removed with another node's users", kept) + } + } +} + +func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { + inv := ForTest(t) + if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil { + t.Fatal("forgetting the bus users of no node was allowed") + } +} + +func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) { + inv := ForTest(t) + ctx := context.Background() + + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, known, err := inv.BusUserHash(ctx, "controller") + if err != nil || !known { + t.Fatalf("the credential was not recorded: %v %v", known, err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { + t.Fatalf("what was recorded does not verify the credential given: %v", err) + } + + // Minted since, then seeded again — which is what a restart does. The rotation must stand, or + // every restart would put the bootstrap password back over it. + minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController}) + if err != nil { + t.Fatal(err) + } + if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, + "bootstrap"); err != nil { + t.Fatal(err) + } + hash, _, err = inv.BusUserHash(ctx, "controller") + if err != nil { + t.Fatal(err) + } + if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil { + t.Fatal("a restart put the bootstrap credential back over a rotated one") + } +} diff --git a/internal/inventory/bususers_test.go b/internal/inventory/bususers_test.go index 401c6220..1bbb5aee 100644 --- a/internal/inventory/bususers_test.go +++ b/internal/inventory/bususers_test.go @@ -1,7 +1,6 @@ package inventory import ( - "context" "testing" "golang.org/x/crypto/bcrypt" @@ -17,147 +16,19 @@ func aBusUser(module string) BusUser { // The plaintext comes back once and the store keeps only a hash that verifies against it. **A // credential recoverable from the mesh's store is one whose blast radius is the store's**, so what // is asserted is that the password is not in there. -func TestABusPasswordIsReturnedOnceAndOnlyItsHashIsKept(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - password, err := inv.MintBusPassword(ctx, aBusUser("shop")) - if err != nil { - t.Fatal(err) - } - if password == "" { - t.Fatal("no password came back, so nothing can be sealed to the module") - } - - hash, known, err := inv.BusUserHash(ctx, "one.shop") - if err != nil || !known { - t.Fatalf("the user was not recorded: %v %v", known, err) - } - if hash == password { - t.Fatal("the store holds the password itself") - } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(password)); err != nil { - t.Fatalf("the recorded hash does not verify the password it was made from: %v", err) - } -} // Minting again replaces what was there rather than failing or adding a second row: that is a // rotation, and the old credential stops working at the next composition. -func TestMintingAgainRotatesRatherThanAddsAUser(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - first, err := inv.MintBusPassword(ctx, aBusUser("shop")) - if err != nil { - t.Fatal(err) - } - second, err := inv.MintBusPassword(ctx, aBusUser("shop")) - if err != nil { - t.Fatal(err) - } - if first == second { - t.Fatal("minting twice produced the same password") - } - users, err := inv.BusUsers(ctx) - if err != nil { - t.Fatal(err) - } - if len(users) != 1 { - t.Fatalf("%d users after two mints for one name", len(users)) - } - hash := users["one.shop"].PasswordHash - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(second)); err != nil { - t.Fatal("the kept hash is not the newest password's") - } - if bcrypt.CompareHashAndPassword([]byte(hash), []byte(first)) == nil { - t.Fatal("the previous password still verifies, so a rotation revoked nothing") - } -} // Forgetting a node takes every credential that belonged to it — its host's and every module // assigned to it. What a forgotten node leaves behind otherwise is a working set of credentials for // a machine the mesh no longer knows. -func TestForgettingANodeTakesItsBusUsersWithIt(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - for _, u := range []BusUser{ - {Username: "node.one", Kind: BusNode, Node: "one"}, - aBusUser("shop"), - {Username: "node.two", Kind: BusNode, Node: "two"}, - {Username: "controller", Kind: BusController}, - } { - if _, err := inv.MintBusPassword(ctx, u); err != nil { - t.Fatal(err) - } - } - if err := inv.ForgetBusUsersOf(ctx, "one"); err != nil { - t.Fatal(err) - } - users, err := inv.BusUsers(ctx) - if err != nil { - t.Fatal(err) - } - if _, still := users["node.one"]; still { - t.Fatal("a forgotten node's host credential still works") - } - if _, still := users["one.shop"]; still { - t.Fatal("a module on a forgotten node still has a credential") - } - // And nothing else went with it: the controller has no node, and another machine's user is - // another machine's. - for _, kept := range []string{"node.two", "controller"} { - if _, ok := users[kept]; !ok { - t.Fatalf("%s was removed with another node's users", kept) - } - } -} // Forgetting the users of no node would forget every user that has none — the controller and every // person — so it is refused rather than run. -func TestForgettingTheUsersOfNoNodeIsRefused(t *testing.T) { - inv := ForTest(t) - if err := inv.ForgetBusUsersOf(context.Background(), ""); err == nil { - t.Fatal("forgetting the bus users of no node was allowed") - } -} // The controller's own user is created by the installer, so the mesh has to be able to record a // credential it did not mint — or the first composition leaves the writer out of the file it writes. -func TestACredentialTheMeshDidNotMintIsRecordedOnceAndNotOverwritten(t *testing.T) { - inv := ForTest(t) - ctx := context.Background() - - if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, - "bootstrap"); err != nil { - t.Fatal(err) - } - hash, known, err := inv.BusUserHash(ctx, "controller") - if err != nil || !known { - t.Fatalf("the credential was not recorded: %v %v", known, err) - } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte("bootstrap")); err != nil { - t.Fatalf("what was recorded does not verify the credential given: %v", err) - } - - // Minted since, then seeded again — which is what a restart does. The rotation must stand, or - // every restart would put the bootstrap password back over it. - minted, err := inv.MintBusPassword(ctx, BusUser{Username: "controller", Kind: BusController}) - if err != nil { - t.Fatal(err) - } - if err := inv.SeedBusUser(ctx, BusUser{Username: "controller", Kind: BusController}, - "bootstrap"); err != nil { - t.Fatal(err) - } - hash, _, err = inv.BusUserHash(ctx, "controller") - if err != nil { - t.Fatal(err) - } - if err := bcrypt.CompareHashAndPassword([]byte(hash), []byte(minted)); err != nil { - t.Fatal("a restart put the bootstrap credential back over a rotated one") - } -} // Outside a test a bus password is hashed at bcrypt's default cost: only ForTest lowers it (novox/hq issue 591). func TestABusPasswordIsHashedAtTheDefaultCostOutsideATest(t *testing.T) { diff --git a/internal/inventory/busy_integration_test.go b/internal/inventory/busy_integration_test.go new file mode 100644 index 00000000..61283905 --- /dev/null +++ b/internal/inventory/busy_integration_test.go @@ -0,0 +1,66 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { + inv, node := aNodeWithModules(t) + ctx := t.Context() + record, err := inv.NodeByName(ctx, node) + if err != nil { + t.Fatal(err) + } + since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) + waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, + Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} + keep := func(busy []BusyWait) bool { + t.Helper() + news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, + Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) + if err != nil { + t.Fatal(err) + } + return news + } + read := func() []BusyWait { + t.Helper() + reports, err := inv.LastReports(ctx) + if err != nil { + t.Fatal(err) + } + for _, r := range reports { + if r.Node == node { + return r.Busy + } + } + t.Fatal("the machine's report is not read back") + return nil + } + keep(nil) + if got := read(); len(got) != 0 { + t.Fatalf("a report with nothing busy reads back %+v", got) + } + if !keep(waiting) { + t.Error("a container starting to wait is not news") + } + got := read() + if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { + t.Fatalf("the wait is not read back: %+v", got) + } + again := []BusyWait{waiting[0]} + again[0].Said = "busy: a check, a minute on" + if keep(again) { + t.Error("a wait whose output changed is news") + } + if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", + Since: since, Until: since.Add(15 * time.Minute)}}) { + t.Error("a container replaced after it waited is not news") + } + if _, ok := WaitingFor(read(), "build-agent"); ok { + t.Error("a container replaced still reads as waiting") + } +} diff --git a/internal/inventory/busy_test.go b/internal/inventory/busy_test.go index bd184a3e..7170119d 100644 --- a/internal/inventory/busy_test.go +++ b/internal/inventory/busy_test.go @@ -83,63 +83,6 @@ func TestABusyContainerIsItsModules(t *testing.T) { // A machine's report of a busy container is kept with its last account and read back beside it, and its // starting and its end are news; its output alone is not (novox/hq ADR 0335). Needs the store. -func TestAReportOfABusyContainerIsKeptAndReadBack(t *testing.T) { - inv, node := aNodeWithModules(t) - ctx := t.Context() - record, err := inv.NodeByName(ctx, node) - if err != nil { - t.Fatal(err) - } - since := time.Date(2026, 10, 13, 1, 0, 0, 0, time.UTC) - waiting := []BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: BusyWaiting, - Since: since, Until: since.Add(15 * time.Minute), Said: "busy: a check"}} - keep := func(busy []BusyWait) bool { - t.Helper() - news, err := inv.RecordOrderedDoing(ctx, record.ID, Doing{Outcome: OutcomeApplied, Declared: "d1", Applied: 1, - Busy: busy}, ReportOrder{Epoch: 1, Sequence: 1}, func(ReportOrder) bool { return false }) - if err != nil { - t.Fatal(err) - } - return news - } - read := func() []BusyWait { - t.Helper() - reports, err := inv.LastReports(ctx) - if err != nil { - t.Fatal(err) - } - for _, r := range reports { - if r.Node == node { - return r.Busy - } - } - t.Fatal("the machine's report is not read back") - return nil - } - keep(nil) - if got := read(); len(got) != 0 { - t.Fatalf("a report with nothing busy reads back %+v", got) - } - if !keep(waiting) { - t.Error("a container starting to wait is not news") - } - got := read() - if b, ok := WaitingFor(got, "build-agent"); !ok || !b.Since.Equal(since) || b.Said != "busy: a check" { - t.Fatalf("the wait is not read back: %+v", got) - } - again := []BusyWait{waiting[0]} - again[0].Said = "busy: a check, a minute on" - if keep(again) { - t.Error("a wait whose output changed is news") - } - if !keep([]BusyWait{{ID: "build-agent.server", Container: "mesh-build-agent", State: "replaced-when-idle", - Since: since, Until: since.Add(15 * time.Minute)}}) { - t.Error("a container replaced after it waited is not news") - } - if _, ok := WaitingFor(read(), "build-agent"); ok { - t.Error("a container replaced still reads as waiting") - } -} // A machine of the rest that answered and waits for a busy container is never read as silent, its wait is kept // out of its apply time, and once replaced its report is the end of the wait (novox/hq ADR 0335). diff --git a/internal/inventory/catalogue_test.go b/internal/inventory/catalogue_test.go index 39f66cc6..f956ba31 100644 --- a/internal/inventory/catalogue_test.go +++ b/internal/inventory/catalogue_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/collection_test.go b/internal/inventory/collection_test.go index e05a2a3f..b608facf 100644 --- a/internal/inventory/collection_test.go +++ b/internal/inventory/collection_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/config_move_test.go b/internal/inventory/config_move_test.go index 1e321bee..a4912522 100644 --- a/internal/inventory/config_move_test.go +++ b/internal/inventory/config_move_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/core_box_test.go b/internal/inventory/core_box_test.go index 5a07a755..8d08c990 100644 --- a/internal/inventory/core_box_test.go +++ b/internal/inventory/core_box_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/data_test.go b/internal/inventory/data_test.go index 9d23e49e..22cc398f 100644 --- a/internal/inventory/data_test.go +++ b/internal/inventory/data_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/dependencies.go b/internal/inventory/dependencies.go index d3042d8e..678575bb 100644 --- a/internal/inventory/dependencies.go +++ b/internal/inventory/dependencies.go @@ -38,6 +38,9 @@ type Edge struct { From string `json:"from"` To string `json:"to"` Kind string `json:"kind"` + // Why is the rule and the fact that drew an edge of a plan's order, in the words the plan shows (novox/hq ADR + // 0419 clause 5): "verb before caller: .". Empty on the catalogue's own edges, which Kind names. + Why string `json:"why,omitempty"` } // Dependencies is the catalogue's dependency relation, whole: every module the mesh holds, with diff --git a/internal/inventory/dependencies_test.go b/internal/inventory/dependencies_test.go index 689e10ff..566e99f5 100644 --- a/internal/inventory/dependencies_test.go +++ b/internal/inventory/dependencies_test.go @@ -42,12 +42,12 @@ func TestDependenciesAreOneRelationWithTheirKinds(t *testing.T) { return false } for _, want := range []Edge{ - {"shop", "mesh-tools", EdgeStandsOn}, - {"builder", "mesh-tools", EdgeStandsOn}, - {"shop-plugin", "shop", EdgeDeclared}, - {"shop", "builder", EdgeBuiltBy}, - {"mesh-controller", "builder", EdgeBuiltBy}, - {"mesh-tools", "builder", EdgeBuiltBy}, + {From: "shop", To: "mesh-tools", Kind: EdgeStandsOn}, + {From: "builder", To: "mesh-tools", Kind: EdgeStandsOn}, + {From: "shop-plugin", To: "shop", Kind: EdgeDeclared}, + {From: "shop", To: "builder", Kind: EdgeBuiltBy}, + {From: "mesh-controller", To: "builder", Kind: EdgeBuiltBy}, + {From: "mesh-tools", To: "builder", Kind: EdgeBuiltBy}, } { if !has(want.From, want.To, want.Kind) { t.Errorf("missing %+v in %+v", want, got) diff --git a/internal/inventory/doing_test.go b/internal/inventory/doing_test.go index 76d6a203..b56c8239 100644 --- a/internal/inventory/doing_test.go +++ b/internal/inventory/doing_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/durations_test.go b/internal/inventory/durations_test.go index f3c84776..356618f5 100644 --- a/internal/inventory/durations_test.go +++ b/internal/inventory/durations_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/forget_test.go b/internal/inventory/forget_test.go index b28e317c..ff79f139 100644 --- a/internal/inventory/forget_test.go +++ b/internal/inventory/forget_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/given_test.go b/internal/inventory/given_test.go index c6544297..a79a7877 100644 --- a/internal/inventory/given_test.go +++ b/internal/inventory/given_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/heals_test.go b/internal/inventory/heals_test.go index 8069c580..4a3a479c 100644 --- a/internal/inventory/heals_test.go +++ b/internal/inventory/heals_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/hold_plans_test.go b/internal/inventory/hold_plans_test.go index 854451a8..daa4744a 100644 --- a/internal/inventory/hold_plans_test.go +++ b/internal/inventory/hold_plans_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/hold_test.go b/internal/inventory/hold_test.go index 50c187c9..19cc4132 100644 --- a/internal/inventory/hold_test.go +++ b/internal/inventory/hold_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/holdings_test.go b/internal/inventory/holdings_test.go index 2a32a2e9..b298a59d 100644 --- a/internal/inventory/holdings_test.go +++ b/internal/inventory/holdings_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/inventory_test.go b/internal/inventory/inventory_test.go index daabd806..178851e1 100644 --- a/internal/inventory/inventory_test.go +++ b/internal/inventory/inventory_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/issued_test.go b/internal/inventory/issued_test.go index f8725f20..65c40254 100644 --- a/internal/inventory/issued_test.go +++ b/internal/inventory/issued_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/looks_test.go b/internal/inventory/looks_test.go index 9b41fdda..9d4e91a4 100644 --- a/internal/inventory/looks_test.go +++ b/internal/inventory/looks_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/mirrored_test.go b/internal/inventory/mirrored_test.go index 3fb13464..e58bf53d 100644 --- a/internal/inventory/mirrored_test.go +++ b/internal/inventory/mirrored_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/named_peer_test.go b/internal/inventory/named_peer_test.go index 52c669ed..81ccc2e7 100644 --- a/internal/inventory/named_peer_test.go +++ b/internal/inventory/named_peer_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory // A carried peer is nameable (novox/hq issue 112): the operator states which machine a carried diff --git a/internal/inventory/older_build_test.go b/internal/inventory/older_build_test.go index a0d3c7a1..5c3f861f 100644 --- a/internal/inventory/older_build_test.go +++ b/internal/inventory/older_build_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/operator_test.go b/internal/inventory/operator_test.go index bfa71e77..9b1a9b13 100644 --- a/internal/inventory/operator_test.go +++ b/internal/inventory/operator_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/order_test.go b/internal/inventory/order_test.go index bd09d191..910d783a 100644 --- a/internal/inventory/order_test.go +++ b/internal/inventory/order_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/outside_test.go b/internal/inventory/outside_test.go index b45ad0fd..70019a76 100644 --- a/internal/inventory/outside_test.go +++ b/internal/inventory/outside_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/pending_test.go b/internal/inventory/pending_test.go index 1ae3b8dd..a3a25572 100644 --- a/internal/inventory/pending_test.go +++ b/internal/inventory/pending_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/people_test.go b/internal/inventory/people_test.go index 5919607e..86351b5e 100644 --- a/internal/inventory/people_test.go +++ b/internal/inventory/people_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/pin_completed_test.go b/internal/inventory/pin_completed_test.go index e090993b..9fac8387 100644 --- a/internal/inventory/pin_completed_test.go +++ b/internal/inventory/pin_completed_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/placement_test.go b/internal/inventory/placement_test.go index 0483fe7b..0acda5c2 100644 --- a/internal/inventory/placement_test.go +++ b/internal/inventory/placement_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/placement_words_test.go b/internal/inventory/placement_words_test.go index d4645ffb..e3a8ee7f 100644 --- a/internal/inventory/placement_words_test.go +++ b/internal/inventory/placement_words_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/plan_id_test.go b/internal/inventory/plan_id_test.go index 76323226..88cd0021 100644 --- a/internal/inventory/plan_id_test.go +++ b/internal/inventory/plan_id_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/plans.go b/internal/inventory/plans.go index dd6b2d1b..c2e2bed4 100644 --- a/internal/inventory/plans.go +++ b/internal/inventory/plans.go @@ -139,6 +139,12 @@ type PlanBatch struct { // Waits says why a batch made for one merge waits for the window: its delivery group has a member still to // merge, or mesh-delivery could not say whether it has (novox/hq ADR 0306 decision 2). Waits string `json:"waits,omitempty"` + // Holds are the groups whose last member is still to merge, by head branch name, each with why in words + // (novox/hq ADR 0418 clause 3): the batch does not close by the merge window while one is held, however long the + // member takes. Cleared when every member is merged or closed; the window then runs again from that moment. + Holds map[string]string `json:"holds,omitempty"` + // HeldAt is when the last hold was cleared: the maximum is counted from it, not from the batch's making. + HeldAt time.Time `json:"held_at,omitempty"` } // OwnPath says the record is a batch of merges on the controller's own path. @@ -199,6 +205,10 @@ func (p Plan) Batch() bool { return p.State == PlanAssembling || p.State == Plan // PlanDelivery is what a walk waits for and what came of the wait (novox/hq ADR 0239). type PlanDelivery struct { + // Order are the edges the plan's own order was drawn with (novox/hq ADR 0419): verb before caller, version skew, + // engine before controller. Kept on the plan because the catalogue's graph does not hold them, and the plan's gates + // (a later tier waits for the earlier to be applied) are read from them. + Order []Edge `json:"order,omitempty"` // Awaits is who must say the walk may start: the seat whose holder owns the delivery. Awaits string `json:"awaits"` // Go is when it was let go, By by whom (the seat's holder, or a person's name) and Why. @@ -292,6 +302,9 @@ type PlanModule struct { FirstAt *time.Time `json:"first_at,omitempty"` Commit string `json:"commit,omitempty"` Why string `json:"why,omitempty"` + // Placed is the edge that put the module after another, and the rule that drew it (novox/hq ADR 0419 clause 5): + // "after (verb before caller: .)". Empty for a module placed first by nothing. + Placed string `json:"placed,omitempty"` // Made is what the walk's build of it did (novox/hq ADR 0350 §4): "reused from ", naming the pull // request's head, the trunk commit and the digests, or "built: ". Said in `plan-moved` and kept here. Made string `json:"made,omitempty"` diff --git a/internal/inventory/plans_test.go b/internal/inventory/plans_test.go index 027600eb..831a63b5 100644 --- a/internal/inventory/plans_test.go +++ b/internal/inventory/plans_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/ports_test.go b/internal/inventory/ports_test.go index 60f0665b..eab01635 100644 --- a/internal/inventory/ports_test.go +++ b/internal/inventory/ports_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/predefined_test.go b/internal/inventory/predefined_test.go index d2ab52d2..14d4fa5e 100644 --- a/internal/inventory/predefined_test.go +++ b/internal/inventory/predefined_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/public_domain_test.go b/internal/inventory/public_domain_test.go index 19190b46..eb81eec7 100644 --- a/internal/inventory/public_domain_test.go +++ b/internal/inventory/public_domain_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/reachable_integration_test.go b/internal/inventory/reachable_integration_test.go new file mode 100644 index 00000000..8b222092 --- /dev/null +++ b/internal/inventory/reachable_integration_test.go @@ -0,0 +1,33 @@ +//go:build integration + +package inventory + +import ( + "context" + "os" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgconn" +) + +func TestAWrongPasswordIsAnAnswerNotAnOutage(t *testing.T) { + dsn := os.Getenv("MESH_TEST_POSTGRES") + if dsn == "" { + t.Skip("MESH_TEST_POSTGRES is not set") + } + wrong := strings.Replace(dsn, "postgres:check@", "postgres:not-the-password@", 1) + if wrong == dsn { + t.Skip("the test store's address does not carry the expected credential") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _, err := pgconn.Connect(ctx, wrong) + if err == nil { + t.Fatal("a wrong password connected") + } + if Unreachable(err) { + t.Fatalf("a wrong password was taken for an outage and would be retried for ever: %v", err) + } +} diff --git a/internal/inventory/reachable_test.go b/internal/inventory/reachable_test.go index 04543587..701d9a9f 100644 --- a/internal/inventory/reachable_test.go +++ b/internal/inventory/reachable_test.go @@ -5,8 +5,6 @@ import ( "errors" "fmt" "io" - "os" - "strings" "testing" "time" @@ -55,22 +53,3 @@ func TestAStoreThatRefusesTheConnectionIsUnreachable(t *testing.T) { // A connection the store refused on a wrong password is an answer, even though it arrives as a // failed connection — asked again for ever, it would hold every message behind it. -func TestAWrongPasswordIsAnAnswerNotAnOutage(t *testing.T) { - dsn := os.Getenv("MESH_TEST_POSTGRES") - if dsn == "" { - t.Skip("MESH_TEST_POSTGRES is not set") - } - wrong := strings.Replace(dsn, "postgres:check@", "postgres:not-the-password@", 1) - if wrong == dsn { - t.Skip("the test store's address does not carry the expected credential") - } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - _, err := pgconn.Connect(ctx, wrong) - if err == nil { - t.Fatal("a wrong password connected") - } - if Unreachable(err) { - t.Fatalf("a wrong password was taken for an outage and would be retried for ever: %v", err) - } -} diff --git a/internal/inventory/record_mirrors_integration_test.go b/internal/inventory/record_mirrors_integration_test.go new file mode 100644 index 00000000..b2dc6685 --- /dev/null +++ b/internal/inventory/record_mirrors_integration_test.go @@ -0,0 +1,37 @@ +//go:build integration + +package inventory + +import ( + "encoding/json" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration(t *testing.T) { + inv := fresh(t) + var m catalogue.Manifest + if err := json.Unmarshal([]byte(`{"module": "proxy", "version": "1", "config": {"port": {"type": "number", + "description": "the port it listens on, a TCP port number", "default": 8080, "minimum": 1, + "maximum": 65535, "enum": [8080, 8443], "predefined": "http.port"}}}`), &m); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { + t.Fatal(err) + } + values, err := inv.DeclaredConfigMirror(t.Context()) + if err != nil { + t.Fatal(err) + } + var got DeclaredConfigMirrored + if err := json.Unmarshal(values["proxy"], &got); err != nil { + t.Fatal(err) + } + port, has := got.Settings["port"] + if !has || port.Type != catalogue.ConfigNumber || port.Description == "" || string(port.Default) != "8080" || + port.Minimum == nil || *port.Minimum != 1 || port.Maximum == nil || *port.Maximum != 65535 || + string(port.Enum) != "[8080,8443]" || port.Predefined != "http.port" { + t.Errorf("a module in the config form is mirrored as %s", values["proxy"]) + } +} diff --git a/internal/inventory/record_mirrors_test.go b/internal/inventory/record_mirrors_test.go index cb329ae1..013d94e5 100644 --- a/internal/inventory/record_mirrors_test.go +++ b/internal/inventory/record_mirrors_test.go @@ -85,29 +85,3 @@ func TestADeclarationUsingEveryFieldRoundTripsThroughTheMirror(t *testing.T) { // **A module moved to the config form is mirrored from it** (ADR 0348 §1): the store keeps its manifest in the // form it was written in, under `config`, and the declared-config mirror reads its declarations from there with // the config form's fields. -func TestAModuleInTheConfigFormIsMirroredWithItsWholeDeclaration(t *testing.T) { - inv := fresh(t) - var m catalogue.Manifest - if err := json.Unmarshal([]byte(`{"module": "proxy", "version": "1", "config": {"port": {"type": "number", - "description": "the port it listens on, a TCP port number", "default": 8080, "minimum": 1, - "maximum": 65535, "enum": [8080, 8443], "predefined": "http.port"}}}`), &m); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(t.Context(), m, Source{}); err != nil { - t.Fatal(err) - } - values, err := inv.DeclaredConfigMirror(t.Context()) - if err != nil { - t.Fatal(err) - } - var got DeclaredConfigMirrored - if err := json.Unmarshal(values["proxy"], &got); err != nil { - t.Fatal(err) - } - port, has := got.Settings["port"] - if !has || port.Type != catalogue.ConfigNumber || port.Description == "" || string(port.Default) != "8080" || - port.Minimum == nil || *port.Minimum != 1 || port.Maximum == nil || *port.Maximum != 65535 || - string(port.Enum) != "[8080,8443]" || port.Predefined != "http.port" { - t.Errorf("a module in the config form is mirrored as %s", values["proxy"]) - } -} diff --git a/internal/inventory/registered_held_integration_test.go b/internal/inventory/registered_held_integration_test.go new file mode 100644 index 00000000..cf299498 --- /dev/null +++ b/internal/inventory/registered_held_integration_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package inventory + +import ( + "encoding/json" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + manifestOf := func(image string) catalogue.Manifest { + return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ + {"id": "app", "type": "container", "name": "one", "image": image}}} + } + record := func(id, image string, asked time.Time) Build { + t.Helper() + raw, _ := json.Marshal(manifestOf(image)) + b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw, + Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}} + if err := inv.RecordBuild(ctx, b); err != nil { + t.Fatal(err) + } + if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit, + Asked: asked}); err != nil { + t.Fatal(err) + } + return b + } + x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111" + y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" + before := record("x", x, time.Now().Add(-2*time.Hour)) + failed := record("y", y, time.Now().Add(-time.Minute)) + if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit, + Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil { + t.Fatal(err) + } + if err := inv.RestoreModule(ctx, before); err != nil { + t.Fatal(err) + } + held, err := inv.Held(ctx) + if err != nil { + t.Fatal(err) + } + running, err := inv.RegisteredHeld(ctx) + if err != nil { + t.Fatal(err) + } + if running["one/one"] == y { + t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"]) + } + if got := running["one/one"]; got != "" && got != x { + t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x) + } +} diff --git a/internal/inventory/registered_held_test.go b/internal/inventory/registered_held_test.go index ecb86043..a471b959 100644 --- a/internal/inventory/registered_held_test.go +++ b/internal/inventory/registered_held_test.go @@ -1,63 +1,12 @@ package inventory import ( - "encoding/json" "testing" - "time" - - "github.com/novox/mesh-controller/internal/catalogue" ) // TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack is novox/hq issue 721's review: the newest build of a module // failed its gate and the mesh put the build before it back. Held names the newest; what runs, and what a merge's build // is compared with, is the build put back — so a merge whose bytes equal the failed build is never said unchanged. -func TestWhatRunsIsTheRegisteredBuildNotANewerOnePutBack(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - manifestOf := func(image string) catalogue.Manifest { - return catalogue.Manifest{Module: "one", Version: "1", Resources: []map[string]any{ - {"id": "app", "type": "container", "name": "one", "image": image}}} - } - record := func(id, image string, asked time.Time) Build { - t.Helper() - raw, _ := json.Marshal(manifestOf(image)) - b := Build{ID: id, Module: "one", Commit: id + "aaaaaaaa", Repository: "novox/one", Manifest: raw, - Asked: asked, At: asked, Made: []Artifact{{Name: "one", Kind: "image", Reference: image}}} - if err := inv.RecordBuild(ctx, b); err != nil { - t.Fatal(err) - } - if err := inv.RegisterModule(ctx, manifestOf(image), Source{Repository: "novox/one", BuiltFrom: b.Commit, - Asked: asked}); err != nil { - t.Fatal(err) - } - return b - } - x := "store/one@sha256:" + "1111111111111111111111111111111111111111111111111111111111111111" - y := "store/one@sha256:" + "2222222222222222222222222222222222222222222222222222222222222222" - before := record("x", x, time.Now().Add(-2*time.Hour)) - failed := record("y", y, time.Now().Add(-time.Minute)) - if err := inv.RecordGate(ctx, GateVerdict{Build: failed.ID, Module: "one", Commit: failed.Commit, - Machines: []string{"anchor"}, Verdict: GateFailed}); err != nil { - t.Fatal(err) - } - if err := inv.RestoreModule(ctx, before); err != nil { - t.Fatal(err) - } - held, err := inv.Held(ctx) - if err != nil { - t.Fatal(err) - } - running, err := inv.RegisteredHeld(ctx) - if err != nil { - t.Fatal(err) - } - if running["one/one"] == y { - t.Fatalf("what runs is read as the build whose gate failed (%s); Held says %s", y, held["one/one"]) - } - if got := running["one/one"]; got != "" && got != x { - t.Fatalf("what runs is read as %s, want the build put back %s or nothing", got, x) - } -} // TestOnlyArtifactsTheRegisteredManifestNamesAreRunning: registeredOnly keeps a held artifact only when its digest is // in its module's registered manifest. diff --git a/internal/inventory/retire_test.go b/internal/inventory/retire_test.go index 1956982c..e90b9b77 100644 --- a/internal/inventory/retire_test.go +++ b/internal/inventory/retire_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/seats_test.go b/internal/inventory/seats_test.go index 1dfac4b2..ff22f6a2 100644 --- a/internal/inventory/seats_test.go +++ b/internal/inventory/seats_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/secret_ask_test.go b/internal/inventory/secret_ask_test.go index 51a21af5..6227c8c9 100644 --- a/internal/inventory/secret_ask_test.go +++ b/internal/inventory/secret_ask_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/secret_family_integration_test.go b/internal/inventory/secret_family_integration_test.go new file mode 100644 index 00000000..36f93ed3 --- /dev/null +++ b/internal/inventory/secret_family_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package inventory + +import ( + "strings" + "testing" + + "github.com/novox/mesh-controller/internal/catalogue" +) + +func TestAMemberGivenIsReadAsGivenAndNothingElseIs(t *testing.T) { + inv := fresh(t) + ctx := t.Context() + node, err := inv.AddNode(ctx, "workstation") + if err != nil { + t.Fatal(err) + } + key, _ := aSealingKey(t) + if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { + t.Fatal(err) + } + m := catalogue.Manifest{Module: "mounts", Version: "1", OwnSecrets: catalogue.OwnSecrets{ + "smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside}, + "token": {Path: "/s/token", IssuedBy: catalogue.IssuedOutside}, + }} + if err := inv.RegisterModule(ctx, m, Source{}); err != nil { + t.Fatal(err) + } + if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-password-games", "hunter2"); err != nil { + t.Fatalf("a member was refused: %v", err) + } + if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-pass", "x"); err == nil { + t.Fatal("a name of no family was accepted") + } + members, err := inv.GivenMembers(ctx, "workstation", "mounts", "smb-password-*") + if err != nil { + t.Fatal(err) + } + if len(members) != 1 || members[0].Name != "smb-password-games" || !members[0].Current || members[0].Sealed == "" || + strings.Contains(members[0].Sealed, "hunter2") { + t.Fatalf("members: %+v", members) + } + given, err := inv.GivenOwnSecrets(ctx, "workstation", "mounts") + if err != nil { + t.Fatal(err) + } + if _, was := given["smb-password-games"]; !was || len(given) != 1 { + t.Fatalf("given: %v", given) + } + // A value the mesh made is not one a person gave. + if _, err := inv.SecretForModule(ctx, "workstation", "mounts", "token"); err != nil { + t.Fatal(err) + } + if given, _ := inv.GivenOwnSecrets(ctx, "workstation", "mounts"); len(given) != 1 { + t.Fatalf("a value the mesh made counted as given: %v", given) + } +} diff --git a/internal/inventory/secret_family_test.go b/internal/inventory/secret_family_test.go index 1021dc63..85756a9b 100644 --- a/internal/inventory/secret_family_test.go +++ b/internal/inventory/secret_family_test.go @@ -27,50 +27,3 @@ func TestAMemberIsGivableAtTheDeskAndANameOutsideTheFamilyIsNot(t *testing.T) { } // A member given is read back as given, sealed to the machine's key; the mesh makes none, so nothing else is. -func TestAMemberGivenIsReadAsGivenAndNothingElseIs(t *testing.T) { - inv := fresh(t) - ctx := t.Context() - node, err := inv.AddNode(ctx, "workstation") - if err != nil { - t.Fatal(err) - } - key, _ := aSealingKey(t) - if err := inv.RecordSealingKey(ctx, node.ID, key); err != nil { - t.Fatal(err) - } - m := catalogue.Manifest{Module: "mounts", Version: "1", OwnSecrets: catalogue.OwnSecrets{ - "smb-password-*": {Path: "/s/smb-password-*.secret", IssuedBy: catalogue.IssuedOutside}, - "token": {Path: "/s/token", IssuedBy: catalogue.IssuedOutside}, - }} - if err := inv.RegisterModule(ctx, m, Source{}); err != nil { - t.Fatal(err) - } - if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-password-games", "hunter2"); err != nil { - t.Fatalf("a member was refused: %v", err) - } - if err := inv.AcceptSecretForModule(ctx, "workstation", "mounts", "smb-pass", "x"); err == nil { - t.Fatal("a name of no family was accepted") - } - members, err := inv.GivenMembers(ctx, "workstation", "mounts", "smb-password-*") - if err != nil { - t.Fatal(err) - } - if len(members) != 1 || members[0].Name != "smb-password-games" || !members[0].Current || members[0].Sealed == "" || - strings.Contains(members[0].Sealed, "hunter2") { - t.Fatalf("members: %+v", members) - } - given, err := inv.GivenOwnSecrets(ctx, "workstation", "mounts") - if err != nil { - t.Fatal(err) - } - if _, was := given["smb-password-games"]; !was || len(given) != 1 { - t.Fatalf("given: %v", given) - } - // A value the mesh made is not one a person gave. - if _, err := inv.SecretForModule(ctx, "workstation", "mounts", "token"); err != nil { - t.Fatal(err) - } - if given, _ := inv.GivenOwnSecrets(ctx, "workstation", "mounts"); len(given) != 1 { - t.Fatalf("a value the mesh made counted as given: %v", given) - } -} diff --git a/internal/inventory/secret_move_test.go b/internal/inventory/secret_move_test.go index 3f964db3..9050ad15 100644 --- a/internal/inventory/secret_move_test.go +++ b/internal/inventory/secret_move_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/secrets_test.go b/internal/inventory/secrets_test.go index 760db472..69abd1fa 100644 --- a/internal/inventory/secrets_test.go +++ b/internal/inventory/secrets_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( @@ -6,12 +8,13 @@ import ( "crypto/rand" "encoding/base64" "errors" - "github.com/novox/mesh-controller/internal/catalogue" - "github.com/novox/mesh-controller/internal/secrets" "reflect" "strings" "testing" + "github.com/novox/mesh-controller/internal/catalogue" + "github.com/novox/mesh-controller/internal/secrets" + "golang.org/x/crypto/nacl/box" ) diff --git a/internal/inventory/sends_test.go b/internal/inventory/sends_test.go index 37aaafd6..b08ab198 100644 --- a/internal/inventory/sends_test.go +++ b/internal/inventory/sends_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/sent_builds_test.go b/internal/inventory/sent_builds_test.go index 44015d8e..f28ab693 100644 --- a/internal/inventory/sent_builds_test.go +++ b/internal/inventory/sent_builds_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/sent_bus_users_test.go b/internal/inventory/sent_bus_users_test.go index 57175714..33439454 100644 --- a/internal/inventory/sent_bus_users_test.go +++ b/internal/inventory/sent_bus_users_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import "testing" diff --git a/internal/inventory/sent_plans_test.go b/internal/inventory/sent_plans_test.go index b87f81be..63a82deb 100644 --- a/internal/inventory/sent_plans_test.go +++ b/internal/inventory/sent_plans_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/settings_judged_test.go b/internal/inventory/settings_judged_test.go index 1563fa17..ca1b8488 100644 --- a/internal/inventory/settings_judged_test.go +++ b/internal/inventory/settings_judged_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/store_as_run_test.go b/internal/inventory/store_as_run_test.go index 9cb37f6c..59172256 100644 --- a/internal/inventory/store_as_run_test.go +++ b/internal/inventory/store_as_run_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/stored_silent_file_test.go b/internal/inventory/stored_silent_file_test.go index d5029b9e..a110b235 100644 --- a/internal/inventory/stored_silent_file_test.go +++ b/internal/inventory/stored_silent_file_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/template_test.go b/internal/inventory/template_test.go index b9f58cc8..75c8c12e 100644 --- a/internal/inventory/template_test.go +++ b/internal/inventory/template_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/tunnel_test.go b/internal/inventory/tunnel_test.go index 9498ad4b..d6e7980a 100644 --- a/internal/inventory/tunnel_test.go +++ b/internal/inventory/tunnel_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/unknown_field_test.go b/internal/inventory/unknown_field_test.go index fd163f86..0995f182 100644 --- a/internal/inventory/unknown_field_test.go +++ b/internal/inventory/unknown_field_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/unseen_test.go b/internal/inventory/unseen_test.go index 73a98031..e19b0752 100644 --- a/internal/inventory/unseen_test.go +++ b/internal/inventory/unseen_test.go @@ -1,3 +1,5 @@ +//go:build integration + package inventory import ( diff --git a/internal/inventory/walkphases_582_integration_test.go b/internal/inventory/walkphases_582_integration_test.go new file mode 100644 index 00000000..643d2510 --- /dev/null +++ b/internal/inventory/walkphases_582_integration_test.go @@ -0,0 +1,49 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + record, err := inv.AddNode(ctx, "g14") + if err != nil { + t.Fatal(err) + } + if _, err := inv.NextSequence(ctx, record.ID); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil { + t.Fatal(err) + } + sent, found, err := inv.SentTo(ctx, "g14") + if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" { + t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err) + } + if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok { + t.Fatalf("no report yet: %v %v", ok, err) + } + if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil { + t.Fatal(err) + } + // The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send. + r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) + if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) { + t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err) + } + // A later send, reported: never this walk's. + if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil { + t.Fatal(err) + } + if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil { + t.Fatal(err) + } + again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) + if !again.At.Equal(r.At) { + t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r) + } +} diff --git a/internal/inventory/walkphases_582_test.go b/internal/inventory/walkphases_582_test.go index 98d8cab2..0b363f87 100644 --- a/internal/inventory/walkphases_582_test.go +++ b/internal/inventory/walkphases_582_test.go @@ -146,46 +146,6 @@ func TestAMachineSentTwiceByOneWalkIsHeardByItsLaterSend(t *testing.T) { // The store: a machine's own send time is read with what it was sent, and its `apply` duration is found by it — // the send and the duration start at one moment, the node's sent_at. -func TestAWalksRestIsReadFromEachMachinesOwnSend(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - record, err := inv.AddNode(ctx, "g14") - if err != nil { - t.Fatal(err) - } - if _, err := inv.NextSequence(ctx, record.ID); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, record.ID, "d-582", map[string]string{"build-agent": "c1"}); err != nil { - t.Fatal(err) - } - sent, found, err := inv.SentTo(ctx, "g14") - if err != nil || !found || sent.At.IsZero() || sent.Digest != "d-582" { - t.Fatalf("the machine's own send time: %+v %v %v", sent, found, err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-582", OutcomeApplied); err != nil { - t.Fatal(err) - } - // The walk's record of its send, 20 s after the machine's own: still found, by the machine's own send. - r, ok, err := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if err != nil || !ok || r.Outcome != OutcomeApplied || r.At.Before(sent.At) { - t.Fatalf("the machine's own report of its send: %+v %v %v", r, ok, err) - } - // A later send, reported: never this walk's. - if err := inv.RecordSent(ctx, record.ID, "d-next", map[string]string{"build-agent": "c2"}); err != nil { - t.Fatal(err) - } - if err := inv.RecordApplyDuration(ctx, "g14", "d-next", OutcomeApplied); err != nil { - t.Fatal(err) - } - again, _, _ := inv.FirstAppliedAfter(ctx, "g14", sent, sent.At.Add(20*time.Second)) - if !again.At.Equal(r.At) { - t.Fatalf("a later send's report read as this walk's: %+v, was %+v", again, r) - } -} // Issue 840 (ADR 0362): the walk's report of a machine is asked for with only the provisioned modules the walk sent // there, never every module the machine's declaration hands to the provisioner. diff --git a/internal/inventory/walkphases_integration_test.go b/internal/inventory/walkphases_integration_test.go new file mode 100644 index 00000000..f7cbb108 --- /dev/null +++ b/internal/inventory/walkphases_integration_test.go @@ -0,0 +1,77 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + p := walkOf(t, recordedWalk) + p.Revision, p.Epoch = 0, 0 + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + back, err := inv.PlanByID(ctx, p.ID) + if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil || + !back.Times.WindowClosed.Equal(*p.Times.WindowClosed) { + t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times) + } + if back.Delivery.Merges[0].Merged.IsZero() { + t.Fatal("a merge's time was not kept") + } + sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) + if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok { + t.Fatalf("no report yet: %v %v", ok, err) + } + for _, d := range []Duration{ + {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied}, + {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied}, + } { + if err := inv.RecordDuration(ctx, d); err != nil { + t.Fatal(err) + } + } + r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent) + if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) { + t.Fatalf("the report after the send: %+v %v %v", r, ok, err) + } + if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total { + t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err) + } + if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total", + Started: sent, Took: time.Minute}); err != nil { + t.Fatal(err) + } + if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total { + t.Fatalf("the total kept: %v %v %v", anyKept, total, err) + } +} + +func TestALateReportIsSilentNotTheEnd(t *testing.T) { + sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) + p := walkOf(t, recordedWalk) + p.Modules["mesh-delivery"].SentAt = &sent + p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}} + w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { + if node == "server" { + return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true + } + return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true + }) + if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) { + t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End) + } + inv := ForTest(t) + ctx := t.Context() + if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1", + Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil { + t.Fatal(err) + } + if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok { + t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err) + } +} diff --git a/internal/inventory/walkphases_test.go b/internal/inventory/walkphases_test.go index 533e66a0..216d6d5a 100644 --- a/internal/inventory/walkphases_test.go +++ b/internal/inventory/walkphases_test.go @@ -230,76 +230,9 @@ func TestAGateKeepsItsReadings(t *testing.T) { // A walk's moments are kept and read back with it; a machine's first report after a send is read from the // controller's apply durations. -func TestAWalksMomentsAreKeptAndItsRestsReportRead(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - p := walkOf(t, recordedWalk) - p.Revision, p.Epoch = 0, 0 - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Class != ClassCore || back.Times.WindowClosed == nil || - !back.Times.WindowClosed.Equal(*p.Times.WindowClosed) { - t.Fatalf("the walk's moments were not kept: %v %+v", err, back.Times) - } - if back.Delivery.Merges[0].Merged.IsZero() { - t.Fatal("a merge's time was not kept") - } - sent := time.Now().UTC().Add(-time.Minute).Truncate(time.Millisecond) - if _, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("no report yet: %v %v", ok, err) - } - for _, d := range []Duration{ - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "old@1", Started: sent.Add(-time.Hour), Took: time.Second, Detail: OutcomeApplied}, - {Kind: DurationApply, Subject: "ace", Node: "ace", Ref: "d1@2", Started: sent.Add(-time.Second), Took: 12 * time.Second, Detail: OutcomeApplied}, - } { - if err := inv.RecordDuration(ctx, d); err != nil { - t.Fatal(err) - } - } - r, ok, err := inv.FirstAppliedAfter(ctx, "ace", SentDeclaration{}, sent) - if err != nil || !ok || r.Outcome != OutcomeApplied || !r.At.Equal(sent.Add(11*time.Second)) { - t.Fatalf("the report after the send: %+v %v %v", r, ok, err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || anyKept || total { - t.Fatalf("nothing kept yet: %v %v %v", anyKept, total, err) - } - if err := inv.RecordDuration(ctx, Duration{Kind: DurationWalkPhase, Subject: "core/total", Ref: p.ID + "/total", - Started: sent, Took: time.Minute}); err != nil { - t.Fatal(err) - } - if anyKept, total, err := inv.WalkPhasesKept(ctx, p.ID); err != nil || !anyKept || !total { - t.Fatalf("the total kept: %v %v %v", anyKept, total, err) - } -} // A machine that slept past the bound and reported later is listed silent: its late report never moves the // walk's end; and a report sent past the bound is not read as the walk's. -func TestALateReportIsSilentNotTheEnd(t *testing.T) { - sent := time.Date(2026, 10, 10, 18, 0, 0, 0, time.UTC) - p := walkOf(t, recordedWalk) - p.Modules["mesh-delivery"].SentAt = &sent - p.Modules["mesh-delivery"].Rest = map[string]SentDeclaration{"laptop": {Digest: "d"}, "server": {Digest: "e"}} - w := p.WalkPhases(sent.Add(3*time.Hour), func(node string, _ SentDeclaration, _ time.Time) (AppliedReport, bool) { - if node == "server" { - return AppliedReport{At: sent.Add(20 * time.Second), Outcome: OutcomeApplied}, true - } - return AppliedReport{At: sent.Add(2 * time.Hour), Outcome: OutcomeApplied}, true - }) - if len(w.Silent) != 1 || w.Silent[0] != "laptop" || w.End == nil || !w.End.Equal(sent.Add(20*time.Second)) { - t.Fatalf("a late report: silent %v, end %v", w.Silent, w.End) - } - inv := ForTest(t) - ctx := t.Context() - if err := inv.RecordDuration(ctx, Duration{Kind: DurationApply, Subject: "laptop", Node: "laptop", Ref: "late@1", - Started: sent.Add(time.Hour), Took: time.Second, Detail: OutcomeApplied}); err != nil { - t.Fatal(err) - } - if _, ok, err := inv.FirstAppliedAfter(ctx, "laptop", SentDeclaration{}, sent); err != nil || ok { - t.Fatalf("a send past the bound read as the walk's: %v %v", ok, err) - } -} // Moments out of order — #206's own walk on 2026-10-10 kept its first send 783 ms before its build — are said // unknown, never a negative phase: the phase before is unknown and its time joins the unknown time, the phase diff --git a/internal/inventory/walksaid_integration_test.go b/internal/inventory/walksaid_integration_test.go new file mode 100644 index 00000000..6bd21f1f --- /dev/null +++ b/internal/inventory/walksaid_integration_test.go @@ -0,0 +1,50 @@ +//go:build integration + +package inventory + +import ( + "testing" + "time" +) + +func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) { + inv := ForTest(t) + ctx := t.Context() + cut := time.Now().UTC().Add(-time.Minute) + p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding, + Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}}, + Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} + var said []string + was := PlanSaved + PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) } + t.Cleanup(func() { PlanSaved = was }) + if err := inv.SavePlan(ctx, &p); err != nil { + t.Fatal(err) + } + back, err := inv.PlanByID(ctx, p.ID) + if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf { + t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times) + } + // Read back by another controller and saved: not said again. + back.State = PlanRolling + if err := inv.SavePlan(ctx, &back); err != nil { + t.Fatal(err) + } + // A stale copy ending it is refused, and unmarked. + stale := p + stale.Revision = 1 + stale.State = PlanDone + if err := inv.SavePlan(ctx, &stale); err == nil { + t.Fatal("a stale save is refused") + } + if stale.WalkSaid != "" || stale.Times.Ended != nil { + t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times) + } + back.State = PlanDone + if err := inv.SavePlan(ctx, &back); err != nil { + t.Fatal(err) + } + if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded { + t.Fatalf("said: %q, want started, nothing, ended", said) + } +} diff --git a/internal/inventory/walksaid_test.go b/internal/inventory/walksaid_test.go index 69cf2e5e..6c0a5275 100644 --- a/internal/inventory/walksaid_test.go +++ b/internal/inventory/walksaid_test.go @@ -87,44 +87,3 @@ func TestAWalksOutcome(t *testing.T) { // The marks are kept with the walk, so a controller replaced mid-walk neither says its start again nor misses // its end; and a save refused leaves the walk unmarked, so the save that is made says it. -func TestAWalksStartIsKeptAndARefusedSaveLeavesItUnsaid(t *testing.T) { - inv := ForTest(t) - ctx := t.Context() - cut := time.Now().UTC().Add(-time.Minute) - p := Plan{ID: "plan-308", Repository: "novox/mesh-catalog", Commit: "abc", Created: cut, State: PlanBuilding, - Tiers: [][]string{{"messenger"}}, Modules: map[string]*PlanModule{"messenger": {}}, - Times: &PlanTimes{Cut: &cut, Class: ClassLeaf}} - var said []string - was := PlanSaved - PlanSaved = func(_ *Inventory, p Plan) { said = append(said, p.WalkSaid) } - t.Cleanup(func() { PlanSaved = was }) - if err := inv.SavePlan(ctx, &p); err != nil { - t.Fatal(err) - } - back, err := inv.PlanByID(ctx, p.ID) - if err != nil || back.Times == nil || back.Times.Started == nil || back.Times.Class != ClassLeaf { - t.Fatalf("the start is kept with the walk: %v %+v", err, back.Times) - } - // Read back by another controller and saved: not said again. - back.State = PlanRolling - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - // A stale copy ending it is refused, and unmarked. - stale := p - stale.Revision = 1 - stale.State = PlanDone - if err := inv.SavePlan(ctx, &stale); err == nil { - t.Fatal("a stale save is refused") - } - if stale.WalkSaid != "" || stale.Times.Ended != nil { - t.Fatalf("a refused save is unmarked: %q %+v", stale.WalkSaid, stale.Times) - } - back.State = PlanDone - if err := inv.SavePlan(ctx, &back); err != nil { - t.Fatal(err) - } - if len(said) != 3 || said[0] != WalkStarted || said[1] != "" || said[2] != WalkEnded { - t.Fatalf("said: %q, want started, nothing, ended", said) - } -} diff --git a/internal/lease/lease_test.go b/internal/lease/lease_test.go index c242b572..e2fcf4d9 100644 --- a/internal/lease/lease_test.go +++ b/internal/lease/lease_test.go @@ -1,3 +1,5 @@ +//go:build integration + package lease import ( diff --git a/internal/licences/licences_test.go b/internal/licences/licences_test.go index c941e92c..e322cc5d 100644 --- a/internal/licences/licences_test.go +++ b/internal/licences/licences_test.go @@ -1,3 +1,5 @@ +//go:build integration + package licences import ( diff --git a/internal/licences/refresh_test.go b/internal/licences/refresh_test.go index b7600bbf..9fa0e68f 100644 --- a/internal/licences/refresh_test.go +++ b/internal/licences/refresh_test.go @@ -1,3 +1,5 @@ +//go:build integration + package licences import ( diff --git a/internal/licences/submitrefresh_test.go b/internal/licences/submitrefresh_test.go index 0e325679..302db468 100644 --- a/internal/licences/submitrefresh_test.go +++ b/internal/licences/submitrefresh_test.go @@ -1,3 +1,5 @@ +//go:build integration + package licences import ( diff --git a/internal/link/advisories_integration_test.go b/internal/link/advisories_integration_test.go new file mode 100644 index 00000000..7ce43efe --- /dev/null +++ b/internal/link/advisories_integration_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package link + +import ( + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) { + url := os.Getenv("MESH_TEST_NATS_REFUSING") + if url == "" { + t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>") + } + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + started := time.Now() + _, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second) + if err == nil || !strings.Contains(err.Error(), "the bus refused") { + t.Fatalf("answered %v", err) + } + if took := time.Since(started); took > 3*time.Second { + t.Fatalf("a refusal took %s to be known", took) + } +} + +func TestNatsARefusalReachesItsWaiter(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor") + defer stop() + other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop") + defer stopOther() + conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q", + nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor")) + select { + case <-mine: + case <-time.After(time.Second): + t.Fatal("the waiter was not told") + } + select { + case <-other: + t.Fatal("another subject's waiter was told") + default: + } +} diff --git a/internal/link/advisories_test.go b/internal/link/advisories_test.go index 56d174f2..06a0a759 100644 --- a/internal/link/advisories_test.go +++ b/internal/link/advisories_test.go @@ -1,15 +1,7 @@ package link import ( - "fmt" - "os" - "strings" "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // **A reader's own consumer gone is not a consumer lost**: every bucket watch and stream read-back @@ -45,48 +37,5 @@ func TestOnlyTheMeshsOwnConsumersAreSaidLost(t *testing.T) { // **A question the bus refuses is answered as refused at once**, not after its timeout: against a // server whose only user may not publish where it asks. -func TestNatsARefusedQuestionIsSaidAtOnce(t *testing.T) { - url := os.Getenv("MESH_TEST_NATS_REFUSING") - if url == "" { - t.Skip("MESH_TEST_NATS_REFUSING unset: a server whose user may not publish mesh.seat.>") - } - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - started := time.Now() - _, err = AskSeatTool(t.Context(), conn, "node-intrusion-prevention", "banned", "anchor", map[string]any{}, 10*time.Second) - if err == nil || !strings.Contains(err.Error(), "the bus refused") { - t.Fatalf("answered %v", err) - } - if took := time.Since(started); took > 3*time.Second { - t.Fatalf("a refusal took %s to be known", took) - } -} // The refusal reaches whoever waits on that subject, and only them. -func TestNatsARefusalReachesItsWaiter(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - mine, stop := refusalsOf(conn, "mesh.seat.s.tool.v.anchor") - defer stop() - other, stopOther := refusalsOf(conn, "mesh.seat.s.tool.v.laptop") - defer stopOther() - conn.ErrorHandler()(conn, nil, fmt.Errorf("%w: Permissions Violation for Publish to %q", - nats.ErrPermissionViolation, "mesh.seat.s.tool.v.anchor")) - select { - case <-mine: - case <-time.After(time.Second): - t.Fatal("the waiter was not told") - } - select { - case <-other: - t.Fatal("another subject's waiter was told") - default: - } -} diff --git a/internal/link/build.go b/internal/link/build.go index e698f9da..d40ca40a 100644 --- a/internal/link/build.go +++ b/internal/link/build.go @@ -148,8 +148,12 @@ type CheckRequest struct { // 0406 §4): with the head, what the pull request's key says the current check is for, and what a waiting request // is dropped by and a running check stopped by when the key moves. Trunk string `json:"trunk,omitempty"` - // Ready says the pull request is marked ready (ADR 0406 §3): its check is queued before every other check. + // Ready says mesh-delivery wants this check queued before every other check (ADR 0406 §3); there is no ready mark + // any more (issue 1022). Ready bool `json:"ready,omitempty"` + // Composed says the pull request is a member of a delivery group whose gate is the composed check's (novox/hq ADR + // 0403 clause 1): this check runs the repository's own merge-check.sh alone, and its gate layer is not run. + Composed bool `json:"composed,omitempty"` // Paths are the files the pull request changes, for the width of its rebuild. Paths []string `json:"paths,omitempty"` // Beside are the repositories the check reads next to this one, each cloned at the ref given — the @@ -173,12 +177,6 @@ type CheckRequest struct { // the controller, judged by itself; JudgeValidator for a change to the node-engine, judged by the // running controller built with the change's validator in place of the one it vendors. Judge string `json:"judge,omitempty"` - // Release is the release branch the pull request merges into (novox/hq ADR 0358): its base, when that is not its - // repository's trunk and mesh-controller has a branch of that name. The gate is then judged by the controller - // built from mesh-controller's release branch — a change to the controller still by itself, a change to the - // node-engine with its own validator — and each repository beside it is read at its branch of that name where it - // has one (CheckedOut.Else otherwise). Empty for every other base, checked as before it. - Release string `json:"release,omitempty"` // Siblings are the other open pull requests of this one's delivery group the check reads beside it (novox/hq ADR // 0382 §1): each repository found under Dir is read at the sibling's head merged onto the sibling's base, in // place of the commit the mesh runs. SiblingsUnknown says mesh-delivery could not be asked, or answered without @@ -299,10 +297,6 @@ const ( type CheckedOut struct { Repository string `json:"repository"` Ref string `json:"ref,omitempty"` - // Else is the ref read when the repository has no branch Ref (novox/hq ADR 0358 §3): beside a check of a pull - // request into a release branch, Ref is that branch and Else the repository's trunk. Empty: Ref is read, or the - // check cannot run. - Else string `json:"else,omitempty"` } // CheckOutcome is a merge check's verdict. @@ -327,13 +321,7 @@ type CheckOutcome struct { Members []CheckedMember `json:"members,omitempty"` // Built are the builds it made for its merge (novox/hq ADR 0350 §1). Built []CheckBuilt `json:"built,omitempty"` - // Release is the release branch the check was judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the - // controller that judged its gate, as twelve characters, and BesideAt the branch each repository beside it was - // read at, by directory. Empty from a check of any other base, and from a build seat from before it. - Release string `json:"release,omitempty"` - JudgedBy string `json:"judged-by,omitempty"` - BesideAt map[string]string `json:"beside-at,omitempty"` - // Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2). + // Validator says which node-engine validator the group's judge was built with (ADR 0382 §3). Validator string `json:"validator,omitempty"` // Siblings are the pull requests of its delivery group it read beside it, each as its head merged onto its base // commit (ADR 0382 §4), SiblingsUnknown why it could not learn them, and JudgeFrom where the judge came from when diff --git a/internal/link/builds_nats_test.go b/internal/link/builds_nats_test.go index cc5931fb..e06038fa 100644 --- a/internal/link/builds_nats_test.go +++ b/internal/link/builds_nats_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/bus_integration_test.go b/internal/link/bus_integration_test.go new file mode 100644 index 00000000..38627827 --- /dev/null +++ b/internal/link/bus_integration_test.go @@ -0,0 +1,124 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) { + url := testbus.URL(t) + f := loadFixture(t, "events/module-event.json") + + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.AddStream(&nats.StreamConfig{ + Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}, + }); err != nil && err != nats.ErrStreamNameAlreadyInUse { + t.Fatal(err) + } + + bus := OverNATS{JS: js} + body, _ := json.Marshal(f.Given.Body) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil { + t.Fatal(err) + } + + // Read back from the stream, not from the thing that wrote it. + raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject) + if err != nil { + t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err) + } + for _, h := range f.Wire.RequiredHeaders { + if raw.Header.Get(h) == "" { + t.Errorf("%s is not set on the wire, and the fixture requires it", h) + } + } + if got := raw.Header.Get("x-source"); got != f.Given.Module { + t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module) + } + if string(raw.Data) != string(body) { + t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body) + } + // The envelope must not also be nested inside the payload. + var nested map[string]any + if json.Unmarshal(raw.Data, &nested) == nil { + if _, has := nested["key"]; has { + t.Error("the payload carries the envelope's own fields, which the fixture refuses") + } + } +} + +func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + bus := OverNATS{Conn: conn} + began := time.Now() + _, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second) + if err == nil { + t.Fatal("asking a tool nothing serves succeeded") + } + if took := time.Since(began); took > 2*time.Second { + t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took) + } + if !strings.Contains(err.Error(), "nothing serves") { + t.Errorf("the refusal does not say nobody is there: %v", err) + } + // And it is ErrNothingServes, through Ask too, so an asker can wait out a tool runner re-serving and + // never a refusal (novox/hq issue 486). + if !errors.Is(err, ErrNothingServes) { + t.Errorf("nothing serving the tool is not ErrNothingServes: %v", err) + } + if _, err := Ask(context.Background(), bus, "nobody", "home", nil, 30*time.Second); !errors.Is(err, ErrNothingServes) || + !strings.Contains(err.Error(), "nothing serves nobody.home") { + t.Errorf("Ask does not say ErrNothingServes in its words: %v", err) + } +} + +func TestAskingAServedToolAnswers(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + + sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) { + _ = m.Respond([]byte(`{"total":12}`)) + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + + got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price", + []byte(`{"qty":4}`), 5*time.Second) + if err != nil { + t.Fatal(err) + } + if string(got) != `{"total":12}` { + t.Fatalf("the answer came back as %s", got) + } +} diff --git a/internal/link/bus_test.go b/internal/link/bus_test.go index a8fa0745..24387a45 100644 --- a/internal/link/bus_test.go +++ b/internal/link/bus_test.go @@ -1,16 +1,7 @@ package link import ( - "context" - "encoding/json" - "errors" - "strings" "testing" - "time" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // **Both implementations, one fixture.** The point of the seam is not that the transport can be @@ -20,57 +11,6 @@ import ( // Against a real server, because what the fixture pins is what reaches the wire: // // go test ./internal/link/ -run TestTheNatsBus (each test on a bus of its own: internal/testbus) -func TestTheNatsBusEmitsTheEnvelopeTheFixturePins(t *testing.T) { - url := testbus.URL(t) - f := loadFixture(t, "events/module-event.json") - - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := js.AddStream(&nats.StreamConfig{ - Name: "EVENTS", Subjects: []string{"mesh.mod.*.event.>"}, - }); err != nil && err != nats.ErrStreamNameAlreadyInUse { - t.Fatal(err) - } - - bus := OverNATS{JS: js} - body, _ := json.Marshal(f.Given.Body) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := EmitEvent(ctx, bus, f.Given.Key, f.Given.Module, f.Given.Node, f.Given.Body); err != nil { - t.Fatal(err) - } - - // Read back from the stream, not from the thing that wrote it. - raw, err := js.GetLastMsg("EVENTS", f.Wire.Subject) - if err != nil { - t.Fatalf("nothing landed on %s, which the fixture names: %v", f.Wire.Subject, err) - } - for _, h := range f.Wire.RequiredHeaders { - if raw.Header.Get(h) == "" { - t.Errorf("%s is not set on the wire, and the fixture requires it", h) - } - } - if got := raw.Header.Get("x-source"); got != f.Given.Module { - t.Errorf("x-source is %q; the subject says %q", got, f.Given.Module) - } - if string(raw.Data) != string(body) { - t.Errorf("the payload is %s, expected the body alone: %s", raw.Data, body) - } - // The envelope must not also be nested inside the payload. - var nested map[string]any - if json.Unmarshal(raw.Data, &nested) == nil { - if _, has := nested["key"]; has { - t.Error("the payload carries the envelope's own fields, which the fixture refuses") - } - } -} // The subject a declaration lands on is one node's, and nothing else's — the state shape. func TestADeclarationIsAddressedToOneNode(t *testing.T) { @@ -96,60 +36,5 @@ func TestAnEventsSubjectIsDerivedFromItsSource(t *testing.T) { // A tool nobody serves says so at once. The difference between "that module is down" and "that // tool is slow" is the first thing a person asking wants, and waiting out the timeout to say it // is how a fast answer becomes a slow non-answer. -func TestAskingAToolNobodyServesFailsAtOnce(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - bus := OverNATS{Conn: conn} - began := time.Now() - _, err = bus.AskTool(context.Background(), "nobody", "home", nil, 30*time.Second) - if err == nil { - t.Fatal("asking a tool nothing serves succeeded") - } - if took := time.Since(began); took > 2*time.Second { - t.Errorf("took %v to say nothing serves it; the caller waited out the timeout", took) - } - if !strings.Contains(err.Error(), "nothing serves") { - t.Errorf("the refusal does not say nobody is there: %v", err) - } - // And it is ErrNothingServes, through Ask too, so an asker can wait out a tool runner re-serving and - // never a refusal (novox/hq issue 486). - if !errors.Is(err, ErrNothingServes) { - t.Errorf("nothing serving the tool is not ErrNothingServes: %v", err) - } - if _, err := Ask(context.Background(), bus, "nobody", "home", nil, 30*time.Second); !errors.Is(err, ErrNothingServes) || - !strings.Contains(err.Error(), "nothing serves nobody.home") { - t.Errorf("Ask does not say ErrNothingServes in its words: %v", err) - } -} // And a served tool answers, with no reply queue to declare and no correlation to check. -func TestAskingAServedToolAnswers(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - - sub, err := conn.Subscribe(ToolSubject("shop", "price"), func(m *nats.Msg) { - _ = m.Respond([]byte(`{"total":12}`)) - }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() - - got, err := OverNATS{Conn: conn}.AskTool(context.Background(), "shop", "price", - []byte(`{"qty":4}`), 5*time.Second) - if err != nil { - t.Fatal(err) - } - if string(got) != `{"total":12}` { - t.Fatalf("the answer came back as %s", got) - } -} diff --git a/internal/link/busfaults_integration_test.go b/internal/link/busfaults_integration_test.go new file mode 100644 index 00000000..bed450e7 --- /dev/null +++ b/internal/link/busfaults_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package link + +import ( + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBusWithEvents(t *testing.T) *Server { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + return ConnectNats(js, nil, nil) +} + +func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) { + s := aBusWithEvents(t) + now := time.Now().UTC() + slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", + Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000} + sayClosure(t, s, BusSlowConsumerEvent, slow) + sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again + again := slow + again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute) + sayClosure(t, s, BusSlowConsumerEvent, again) + sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7, + Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)}) + // Older than the hour: S9 no longer says it. + sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)", + Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)}) + + got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 { + t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got) + } + for _, a := range got { + switch a.User { + case "laptop.claude-code": + if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 || + !a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") { + t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a) + } + case "anchor.desk": + if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" { + t.Errorf("the closed connection read as %+v", a) + } + default: + t.Errorf("an advisory for %s", a.User) + } + } +} + +func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) { + s := aBusWithEvents(t) + started := time.Now() + got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour)) + if err != nil || len(got) != 0 { + t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err) + } + if took := time.Since(started); took > time.Second { + t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took) + } +} + +func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) { + s := aBusWithEvents(t) + was := MaxBusEventsRead + MaxBusEventsRead = 2 + t.Cleanup(func() { MaxBusEventsRead = was }) + now := time.Now().UTC() + for i := range 3 { + sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i), + Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)}) + } + got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") { + t.Fatalf("a read past its bound read as %+v", got) + } +} diff --git a/internal/link/busfaults_test.go b/internal/link/busfaults_test.go index d1eda4ef..ccc1f0bd 100644 --- a/internal/link/busfaults_test.go +++ b/internal/link/busfaults_test.go @@ -2,12 +2,8 @@ package link import ( "encoding/json" - "strings" "testing" "time" - - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // What the bus module says of its users' connections, read back as advisories (novox/hq ADR 0357, issue @@ -15,19 +11,6 @@ import ( // // go test ./internal/link/ -run TestNatsBusFaults (each test on a bus of its own: internal/testbus) -func aBusWithEvents(t *testing.T) *Server { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - return ConnectNats(js, nil, nil) -} - func sayClosure(t *testing.T, s *Server, event string, c BusClosure) { t.Helper() body, _ := json.Marshal(c) @@ -36,58 +19,6 @@ func sayClosure(t *testing.T, s *Server, event string, c BusClosure) { } } -func TestNatsBusFaultsAreOnePerUserAndKindEachClosureCountedOnce(t *testing.T) { - s := aBusWithEvents(t) - now := time.Now().UTC() - slow := BusClosure{User: "laptop.claude-code", Name: "laptop.claude-code", CID: 10, Reason: "Slow Consumer (Write Deadline)", - Opened: now.Add(-20 * time.Minute), Closed: now.Add(-2 * time.Minute), Pending: 334000} - sayClosure(t, s, BusSlowConsumerEvent, slow) - sayClosure(t, s, BusSlowConsumerEvent, slow) // said again: the module started again - again := slow - again.CID, again.Opened, again.Closed = 31, now.Add(-90*time.Second), now.Add(-time.Minute) - sayClosure(t, s, BusSlowConsumerEvent, again) - sayClosure(t, s, BusConnectionClosedEvent, BusClosure{User: "anchor.desk", CID: 7, - Reason: "Maximum Message Payload Exceeded", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-30 * time.Second)}) - // Older than the hour: S9 no longer says it. - sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "anchor.media", CID: 3, Reason: "Slow Consumer (Pending Bytes)", - Opened: now.Add(-3 * time.Hour), Closed: now.Add(-2 * time.Hour)}) - - got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) - if err != nil { - t.Fatal(err) - } - if len(got) != 2 { - t.Fatalf("read %d advisories, want one for each user within the hour: %+v", len(got), got) - } - for _, a := range got { - switch a.User { - case "laptop.claude-code": - if a.Kind != AdvisorySlowConsumer || a.ID != "laptop.claude-code" || a.Count != 2 || - !a.Last.Equal(again.Closed) || !a.First.Equal(slow.Closed) || !strings.Contains(a.Said, "connection 31") { - t.Errorf("the slow consumer read as %+v: two closures, the one said twice counted once", a) - } - case "anchor.desk": - if a.Kind != AdvisoryConnectionClosed || a.Count != 1 || a.Reason != "Maximum Message Payload Exceeded" { - t.Errorf("the closed connection read as %+v", a) - } - default: - t.Errorf("an advisory for %s", a.User) - } - } -} - -func TestNatsBusFaultsReadNothingWhenTheBusModuleSaidNothing(t *testing.T) { - s := aBusWithEvents(t) - started := time.Now() - got, err := s.BusFaults(t.Context(), time.Now().Add(-time.Hour)) - if err != nil || len(got) != 0 { - t.Fatalf("read %v, %v from a stream the bus module never wrote", got, err) - } - if took := time.Since(started); took > time.Second { - t.Fatalf("reading nothing took %s: a tick must not wait on an empty stream", took) - } -} - func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *testing.T) { at := time.Date(2026, 10, 14, 12, 0, 0, 0, time.UTC) own := []Advisory{{Kind: AdvisorySlowConsumer, ID: "controller", Said: "the controller fell behind", First: at, Last: at, Count: 1}} @@ -101,21 +32,3 @@ func TestTheControllersOwnSlowConsumerAndItsClosedConnectionAreOneAdvisory(t *te // **A busy hour is read up to a bound, and says it was cut** (review of #326): a storm of closures must not // spend a watchdog tick, and a count cut short is a floor, said as one. -func TestNatsBusFaultsReadAtMostTheBoundAndSaySo(t *testing.T) { - s := aBusWithEvents(t) - was := MaxBusEventsRead - MaxBusEventsRead = 2 - t.Cleanup(func() { MaxBusEventsRead = was }) - now := time.Now().UTC() - for i := range 3 { - sayClosure(t, s, BusSlowConsumerEvent, BusClosure{User: "laptop.claude-code", CID: uint64(10 + i), - Reason: "Slow Consumer (Pending Bytes)", Opened: now.Add(-time.Hour / 2), Closed: now.Add(-time.Duration(3-i) * time.Minute)}) - } - got, err := s.BusFaults(t.Context(), now.Add(-time.Hour)) - if err != nil { - t.Fatal(err) - } - if len(got) != 1 || got[0].Count != 2 || !strings.Contains(got[0].Said, "at least: the read stopped at 2") { - t.Fatalf("a read past its bound read as %+v", got) - } -} diff --git a/internal/link/calls_integration_test.go b/internal/link/calls_integration_test.go new file mode 100644 index 00000000..500d1c31 --- /dev/null +++ b/internal/link/calls_integration_test.go @@ -0,0 +1,38 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAHandlerThatPanicsAnswersAnError(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{ + "boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") }, + "fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil }, + }, nil) + if err != nil { + t.Fatal(err) + } + defer stop() + answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second) + if err != nil || !strings.Contains(answer.Error, "failed inside the controller") { + t.Fatalf("a panic answered %+v (%v)", answer, err) + } + answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second) + if err != nil || answer.Error != "" { + t.Fatalf("the call after a panic answered %+v (%v)", answer, err) + } +} diff --git a/internal/link/calls_kept_integration_test.go b/internal/link/calls_kept_integration_test.go new file mode 100644 index 00000000..db13270c --- /dev/null +++ b/internal/link/calls_kept_integration_test.go @@ -0,0 +1,219 @@ +//go:build integration + +package link + +import ( + "bytes" + "context" + "encoding/json" + "log" + "strings" + "testing" + + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" +) + +func callsBucket(t *testing.T) *BusCalls { + t.Helper() + js := aBus(t) + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + _ = api.DeleteKeyValue(t.Context(), broker.CallsBucket) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + keeper, err := CallsOnTheBus(t.Context(), js.Conn()) + if err != nil { + t.Fatal(err) + } + return keeper +} + +func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) { + keeper := callsBucket(t) + first := NewCallLog() + if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil { + t.Fatal(err) + } + a := newAnswers(t) + first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`), + "_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1", + func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil) + recent, _ := first.Recent() + finished := waitKept(t, keeper, recent[0].ID, CallAnswered) + // A second call, still running when its controller stops. + left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "") + waitKept(t, keeper, left.ID, CallRunning) + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + c, found, err := second.Get(finished.ID) + if err != nil || !found { + t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err) + } + if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" { + t.Fatalf("kept %+v", c) + } + if strings.Contains(string(c.Args), "secret") { + t.Fatalf("a setting was kept: %s", c.Args) + } + abandoned, _, _ := second.Get(left.ID) + if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) { + t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String()) + } + listed, err := second.Recent() + if err != nil || len(listed) != 2 { + t.Fatalf("the next controller lists %d calls: %v", len(listed), err) + } + // Its own running calls are not its predecessor's: a controller starting again under the same + // name leaves them alone. + mine := second.begin("mesh-controller", "status", nil, "") + waitKept(t, keeper, mine.ID, CallRunning) + if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil { + t.Fatal(err) + } + if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning { + t.Fatalf("a controller marked its own running call %q", c.State) + } +} + +func TestNatsAFinishRacingTheSweepsSnapshotReadsFinished(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-1", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + finished := running + finished.State = CallAnswered + finished.Answer = json.RawMessage(`"anchor told"`) + wrapped := racingRecent{CallKeeper: keeper, thenLand: func() { + if err := keeper.Keep(context.Background(), finished); err != nil { + t.Error(err) + } + waitKept(t, keeper, running.ID, CallAnswered) + }} + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), wrapped, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + got, found, err := second.Get(running.ID) + if err != nil || !found || got.State != CallAnswered || !strings.Contains(string(got.Answer), "anchor told") { + t.Fatalf("its finish did not stand: %+v %v %v", got, found, err) + } + if strings.Contains(said.String(), running.ID) { + t.Errorf("a finished call was said abandoned: %q", said.String()) + } +} + +func TestNatsARunningCallWithNoFinishIsStillAbandoned(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-2", Seat: "mesh-controller", Verb: "plans", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + got, found, err := second.Get(running.ID) + if err != nil || !found || got.State != CallAbandoned || !strings.Contains(said.String(), running.ID) { + t.Fatalf("got %+v %v %v; said %q", got, found, err, said.String()) + } +} + +func TestNatsAFinishRacingTheSweepsWriteWins(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-3", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + finished := running + finished.State = CallFinishedAfter + finished.Answer = json.RawMessage(`"anchor told"`) + landed := false + racey := &BusCalls{kv: delayedKV{KeyValue: keeper.kv, before: func() { + if landed { + return + } + landed = true + if err := keeper.Keep(context.Background(), finished); err != nil { + t.Error(err) + } + waitKept(t, keeper, running.ID, CallFinishedAfter) + }}} + + var said bytes.Buffer + second := NewCallLog() + if err := second.Durably(t.Context(), racey, "controller@two", log.New(&said, "", 0)); err != nil { + t.Fatal(err) + } + got, found, err := second.Get(running.ID) + if err != nil || !found || got.State != CallFinishedAfter || !strings.Contains(string(got.Answer), "anchor told") { + t.Fatalf("the finish did not win the race: %+v %v %v", got, found, err) + } + if strings.Contains(said.String(), running.ID) { + t.Errorf("a finished call was said abandoned: %q", said.String()) + } +} + +func TestNatsAMarkAbandonedGivesUpAfterTenTries(t *testing.T) { + keeper := callsBucket(t) + running := Call{ID: "call-711-4", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} + if err := keeper.Keep(t.Context(), running); err != nil { + t.Fatal(err) + } + waitKept(t, keeper, running.ID, CallRunning) + + tries := 0 + stuck := &BusCalls{kv: alwaysMismatchKV{KeyValue: keeper.kv, tries: &tries}} + _, err := stuck.MarkAbandoned(t.Context(), running.ID) + if err == nil || !strings.Contains(err.Error(), running.ID) { + t.Fatalf("expected an error naming %s, got %v", running.ID, err) + } + if tries != markAbandonedTries { + t.Fatalf("tried %d times, want %d", tries, markAbandonedTries) + } +} + +func TestNatsTheCallsBucketIsBounded(t *testing.T) { + callsBucket(t) + js := aBus(t) + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket) + if err != nil { + t.Fatal(err) + } + if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor { + t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge) + } +} + +func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) { + keeper := callsBucket(t) + big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)}) + c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big} + if err := keeper.Keep(t.Context(), c); err != nil { + t.Fatal(err) + } + got, found, err := keeper.Kept(t.Context(), "call-1-1") + if err != nil || !found || !strings.Contains(string(got.Answer), "the first") { + t.Fatalf("%v %v %.200s", found, err, got.Answer) + } +} diff --git a/internal/link/calls_kept_test.go b/internal/link/calls_kept_test.go index 175d14a2..1b285137 100644 --- a/internal/link/calls_kept_test.go +++ b/internal/link/calls_kept_test.go @@ -1,40 +1,16 @@ package link import ( - "bytes" "context" - "encoding/json" - "log" - "strings" "testing" "time" "github.com/nats-io/nats.go/jetstream" - - "github.com/novox/mesh-controller/internal/broker" ) // The calls bucket against a real server (novox/hq to-be 45 §6): whether a call's outcome is // readable by id from a controller that did not serve it is a claim about what the bus keeps. -func callsBucket(t *testing.T) *BusCalls { - t.Helper() - js := aBus(t) - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - _ = api.DeleteKeyValue(t.Context(), broker.CallsBucket) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - keeper, err := CallsOnTheBus(t.Context(), js.Conn()) - if err != nil { - t.Fatal(err) - } - return keeper -} - // waitKept waits until the keeper holds a call in a state, the writes being queued. func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call { t.Helper() @@ -54,56 +30,6 @@ func waitKept(t *testing.T, keeper CallKeeper, id, state string) Call { // **A controller restart keeps every call's outcome** (to-be 45 Phase 0): a call one controller // answered is read whole by id from the next, and a call it left running is said abandoned rather // than running for ever. -func TestNatsACallsOutcomeOutlivesItsController(t *testing.T) { - keeper := callsBucket(t) - first := NewCallLog() - if err := first.Durably(t.Context(), keeper, "controller@one", nil); err != nil { - t.Fatal(err) - } - a := newAnswers(t) - first.serveCall("mesh-controller", "push", json.RawMessage(`{"node":"anchor","values":"secret"}`), - "_INBOX.node-tools.g14.Pe3sGzAtv8jUBYQ6sKSvKz.1", - func(context.Context, json.RawMessage) (any, error) { return "anchor told", nil }, a.respond, nil) - recent, _ := first.Recent() - finished := waitKept(t, keeper, recent[0].ID, CallAnswered) - // A second call, still running when its controller stops. - left := first.begin("mesh-controller", "plans", json.RawMessage(`{}`), "") - waitKept(t, keeper, left.ID, CallRunning) - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - c, found, err := second.Get(finished.ID) - if err != nil || !found { - t.Fatalf("the next controller cannot read %s: %v %v", finished.ID, found, err) - } - if !strings.Contains(string(c.Answer), "anchor told") || c.Caller != "node-tools.g14" || c.Holder != "controller@one" { - t.Fatalf("kept %+v", c) - } - if strings.Contains(string(c.Args), "secret") { - t.Fatalf("a setting was kept: %s", c.Args) - } - abandoned, _, _ := second.Get(left.ID) - if abandoned.State != CallAbandoned || !strings.Contains(said.String(), left.ID) { - t.Fatalf("a call left running reads %q, and was said: %q", abandoned.State, said.String()) - } - listed, err := second.Recent() - if err != nil || len(listed) != 2 { - t.Fatalf("the next controller lists %d calls: %v", len(listed), err) - } - // Its own running calls are not its predecessor's: a controller starting again under the same - // name leaves them alone. - mine := second.begin("mesh-controller", "status", nil, "") - waitKept(t, keeper, mine.ID, CallRunning) - if err := second.Durably(t.Context(), keeper, "controller@two", nil); err != nil { - t.Fatal(err) - } - if c, _, _ := keeper.Kept(t.Context(), mine.ID); c.State != CallRunning { - t.Fatalf("a controller marked its own running call %q", c.State) - } -} // racingRecent wraps a CallKeeper so a finish lands right after its Recent() snapshot is taken, as // if that finish raced the moment the snapshot was read (novox/hq issue 711): the snapshot still @@ -125,58 +51,9 @@ func (r racingRecent) Recent(ctx context.Context) ([]Call, error) { // **A finish recorded before the sweep reaches a call stands, with its kept answer** (novox/hq issue // 711): a stale "running" snapshot is no excuse to overwrite a record that has since said how the // call ended. A caller reading `abandoned` for a call that succeeded would redo work already done. -func TestNatsAFinishRacingTheSweepsSnapshotReadsFinished(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-1", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - finished := running - finished.State = CallAnswered - finished.Answer = json.RawMessage(`"anchor told"`) - wrapped := racingRecent{CallKeeper: keeper, thenLand: func() { - if err := keeper.Keep(context.Background(), finished); err != nil { - t.Error(err) - } - waitKept(t, keeper, running.ID, CallAnswered) - }} - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), wrapped, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - got, found, err := second.Get(running.ID) - if err != nil || !found || got.State != CallAnswered || !strings.Contains(string(got.Answer), "anchor told") { - t.Fatalf("its finish did not stand: %+v %v %v", got, found, err) - } - if strings.Contains(said.String(), running.ID) { - t.Errorf("a finished call was said abandoned: %q", said.String()) - } -} // A call left running with no finish recorded is still marked abandoned — unchanged (novox/hq issue // 711): only a finish actually recorded for it may stand instead. -func TestNatsARunningCallWithNoFinishIsStillAbandoned(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-2", Seat: "mesh-controller", Verb: "plans", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), keeper, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - got, found, err := second.Get(running.ID) - if err != nil || !found || got.State != CallAbandoned || !strings.Contains(said.String(), running.ID) { - t.Fatalf("got %+v %v %v; said %q", got, found, err, said.String()) - } -} // delayedKV wraps a jetstream.KeyValue so a hook runs just before Update, to land a write between a // reader's Get and its own compare-and-set write. @@ -193,42 +70,6 @@ func (d delayedKV) Update(ctx context.Context, key string, value []byte, revisio // **The race where a finish lands after the sweep's own read but before its write: the finish // wins** (novox/hq issue 711). The sweep's write is a compare-and-set on the revision it read, so a // finish landing in between invalidates it; the sweep rereads, finds the finish, and leaves it alone. -func TestNatsAFinishRacingTheSweepsWriteWins(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-3", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - finished := running - finished.State = CallFinishedAfter - finished.Answer = json.RawMessage(`"anchor told"`) - landed := false - racey := &BusCalls{kv: delayedKV{KeyValue: keeper.kv, before: func() { - if landed { - return - } - landed = true - if err := keeper.Keep(context.Background(), finished); err != nil { - t.Error(err) - } - waitKept(t, keeper, running.ID, CallFinishedAfter) - }}} - - var said bytes.Buffer - second := NewCallLog() - if err := second.Durably(t.Context(), racey, "controller@two", log.New(&said, "", 0)); err != nil { - t.Fatal(err) - } - got, found, err := second.Get(running.ID) - if err != nil || !found || got.State != CallFinishedAfter || !strings.Contains(string(got.Answer), "anchor told") { - t.Fatalf("the finish did not win the race: %+v %v %v", got, found, err) - } - if strings.Contains(said.String(), running.ID) { - t.Errorf("a finished call was said abandoned: %q", said.String()) - } -} // alwaysMismatchKV wraps a jetstream.KeyValue whose Update always fails with a revision mismatch, as // if some other writer kept winning the race forever (novox/hq issue 711 review round 1). @@ -245,55 +86,11 @@ func (a alwaysMismatchKV) Update(ctx context.Context, key string, value []byte, // **MarkAbandoned gives up after markAbandonedTries rather than loop forever** (novox/hq issue 711 // review round 1): a record whose compare-and-set keeps losing is left as it stands, said in an // error naming the call, not retried without end. -func TestNatsAMarkAbandonedGivesUpAfterTenTries(t *testing.T) { - keeper := callsBucket(t) - running := Call{ID: "call-711-4", Seat: "mesh-controller", Verb: "push", State: CallRunning, Holder: "controller@one"} - if err := keeper.Keep(t.Context(), running); err != nil { - t.Fatal(err) - } - waitKept(t, keeper, running.ID, CallRunning) - - tries := 0 - stuck := &BusCalls{kv: alwaysMismatchKV{KeyValue: keeper.kv, tries: &tries}} - _, err := stuck.MarkAbandoned(t.Context(), running.ID) - if err == nil || !strings.Contains(err.Error(), running.ID) { - t.Fatalf("expected an error naming %s, got %v", running.ID, err) - } - if tries != markAbandonedTries { - t.Fatalf("tried %d times, want %d", tries, markAbandonedTries) - } -} // The bucket holds the last thousand calls or fourteen days: a call is two keys, so the stream under // it holds twice as many messages, and asserting it again keeps the count. -func TestNatsTheCallsBucketIsBounded(t *testing.T) { - callsBucket(t) - js := aBus(t) - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - info, err := js.Context().StreamInfo("KV_" + broker.CallsBucket) - if err != nil { - t.Fatal(err) - } - if info.Config.MaxMsgs != 2*broker.KeptCallsDurably || info.Config.MaxAge != broker.CallsKeptFor { - t.Fatalf("kept %d messages for %s", info.Config.MaxMsgs, info.Config.MaxAge) - } -} // An answer larger than the bound is cut and says so, and the record is still written. -func TestNatsAnAnswerLargerThanTheBoundIsCut(t *testing.T) { - keeper := callsBucket(t) - big, _ := json.Marshal(map[string]string{"result": strings.Repeat("x", broker.CallAnswerBytes+10)}) - c := Call{ID: "call-1-1", Seat: "mesh-controller", Verb: "plan", State: CallAnswered, Answer: big} - if err := keeper.Keep(t.Context(), c); err != nil { - t.Fatal(err) - } - got, found, err := keeper.Kept(t.Context(), "call-1-1") - if err != nil || !found || !strings.Contains(string(got.Answer), "the first") { - t.Fatalf("%v %v %.200s", found, err, got.Answer) - } -} // Who asked is read from the inbox the answer goes to. func TestTheCallerIsTheInboxsPrincipal(t *testing.T) { diff --git a/internal/link/calls_test.go b/internal/link/calls_test.go index 419d6800..577f7c22 100644 --- a/internal/link/calls_test.go +++ b/internal/link/calls_test.go @@ -12,10 +12,6 @@ import ( "testing" "time" "unicode" - - "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // answers collects what a call answered, and fails a second answer: the bus permits one. @@ -280,26 +276,3 @@ func TestAJoinTokenIsShownToItsCallerAndNotKept(t *testing.T) { // A verb whose handler panics answers an error, and the controller serves the next call (review of // novox/hq issue 327: read verbs are answered in the serving process now). -func TestAHandlerThatPanicsAnswersAnError(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - stop, err := OverNATS{Conn: conn}.ServeSeatTools("panicky", map[string]ToolHandler{ - "boom": func(context.Context, json.RawMessage) (any, error) { panic("nil map") }, - "fine": func(context.Context, json.RawMessage) (any, error) { return "ok", nil }, - }, nil) - if err != nil { - t.Fatal(err) - } - defer stop() - answer, err := AskMeshSeatTool(context.Background(), conn, "panicky", "boom", map[string]any{}, 5*time.Second) - if err != nil || !strings.Contains(answer.Error, "failed inside the controller") { - t.Fatalf("a panic answered %+v (%v)", answer, err) - } - answer, err = AskMeshSeatTool(context.Background(), conn, "panicky", "fine", map[string]any{}, 5*time.Second) - if err != nil || answer.Error != "" { - t.Fatalf("the call after a panic answered %+v (%v)", answer, err) - } -} diff --git a/internal/link/check_builds_integration_test.go b/internal/link/check_builds_integration_test.go new file mode 100644 index 00000000..47e22acf --- /dev/null +++ b/internal/link/check_builds_integration_test.go @@ -0,0 +1,67 @@ +//go:build integration + +package link + +import ( + "encoding/json" + "testing" + "time" +) + +func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) { + js := aBus(t) + since := time.Now().Add(-time.Minute) + say := func(r BuildResult) { + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { + t.Fatal(err) + } + } + mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} + built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} + say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built, + {Module: "web", NotBuilt: "it has no reuse key"}}}}) + say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8}, + Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}}) + say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check + newer := built + newer.Key = "r1:2" + say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}}) + + got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" || + got[1].Pull.Number != 7 { + t.Fatalf("read %+v", got) + } +} + +func TestNatsAKeptBuildIsTheBuildersCheckAndEveryBuildSaysItsHead(t *testing.T) { + js := aBus(t) + since := time.Now().Add(-time.Minute) + say := func(r BuildResult) { + body, _ := json.Marshal(r) + if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { + t.Fatal(err) + } + } + mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} + app := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} + say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", + Built: []CheckBuilt{app}}}) + kept := app + kept.Kept = "build-1" + say(BuildResult{ID: "build-2", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", + Built: []CheckBuilt{kept}}}) + + got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Check != "build-1" || got[0].Built.Kept != "build-1" || got[0].Head != "aaaaaaaaaaaa" || + got[1].Check != "build-1" || got[1].Head != "aaaaaaaaaaaa" { + t.Fatalf("read %+v", got) + } +} diff --git a/internal/link/check_builds_test.go b/internal/link/check_builds_test.go index ca938fe2..ed03bb5c 100644 --- a/internal/link/check_builds_test.go +++ b/internal/link/check_builds_test.go @@ -1,72 +1,14 @@ package link import ( - "encoding/json" "testing" - "time" ) // The builds a walk may take are read from the build seat's own outcomes of its checks (novox/hq ADR 0350 §2): only // checks of the walk's pull requests, only builds made, newest first. -func TestNatsTheBuildsAWalkMayTakeAreTheChecksOutcomesOfItsPullRequests(t *testing.T) { - js := aBus(t) - since := time.Now().Add(-time.Minute) - say := func(r BuildResult) { - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { - t.Fatal(err) - } - } - mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} - built := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} - say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built, - {Module: "web", NotBuilt: "it has no reuse key"}}}}) - say(BuildResult{ID: "build-2", Checked: &CheckRequest{Owner: "novox", Repo: "shop", Number: 8}, - Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{built}}}) - say(BuildResult{ID: "build-3", Repository: "novox/shop", Commit: "c0ffee"}) // a build, not a check - newer := built - newer.Key = "r1:2" - say(BuildResult{ID: "build-4", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Built: []CheckBuilt{newer}}}) - - got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) - if err != nil { - t.Fatal(err) - } - if len(got) != 2 || got[0].Check != "build-4" || got[0].Built.Key != "r1:2" || got[1].Check != "build-1" || - got[1].Pull.Number != 7 { - t.Fatalf("read %+v", got) - } -} // A build is read with the head its check judged, and a kept build is named as the check's that built it (novox/hq // issue 703). -func TestNatsAKeptBuildIsTheBuildersCheckAndEveryBuildSaysItsHead(t *testing.T) { - js := aBus(t) - since := time.Now().Add(-time.Minute) - say := func(r BuildResult) { - body, _ := json.Marshal(r) - if _, err := js.Context().Publish(BuildOutcomeOf(TheBuildMachine), body); err != nil { - t.Fatal(err) - } - } - mine := &CheckRequest{Owner: "novox", Repo: "shop", Number: 7} - app := CheckBuilt{Module: "app", Key: "r1:1", Made: []MadeArtifact{{Name: "server", Reference: "s/app/server@sha256:a"}}} - say(BuildResult{ID: "build-1", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", - Built: []CheckBuilt{app}}}) - kept := app - kept.Kept = "build-1" - say(BuildResult{ID: "build-2", Checked: mine, Check: &CheckOutcome{Verdict: "pass", Head: "aaaaaaaaaaaa", - Built: []CheckBuilt{kept}}}) - - got, err := ReadCheckBuilds(t.Context(), js, TheBuildMachine, []PullRef{{Owner: "novox", Repo: "shop", Number: 7}}, since) - if err != nil { - t.Fatal(err) - } - if len(got) != 2 || got[0].Check != "build-1" || got[0].Built.Kept != "build-1" || got[0].Head != "aaaaaaaaaaaa" || - got[1].Check != "build-1" || got[1].Head != "aaaaaaaaaaaa" { - t.Fatalf("read %+v", got) - } -} // Two spellings of a head are the same head only when both say at least twelve characters of it. func TestSameHeadNeedsTwelveCharactersOfBoth(t *testing.T) { diff --git a/internal/link/checks_nats_integration_test.go b/internal/link/checks_nats_integration_test.go new file mode 100644 index 00000000..71959b63 --- /dev/null +++ b/internal/link/checks_nats_integration_test.go @@ -0,0 +1,512 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" +) + +func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx := t.Context() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false), + aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false), + aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk + aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false), + aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + var got []string + for _, a := range queueOf(t, js).Asks { + if a.State != AskWaiting { + t.Errorf("%s is %s", a.ID, a.State) + } + got = append(got, a.ID) + } + if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) { + t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want) + } +} + +func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("other-1", 1, "h1", "t", false), + aPullCheck("other-2", 2, "h2", "t", false), + aPullCheck("ready-3", 3, "h3", "t", true), + {ID: "delivery-4", Repository: "/d"}, + aPullCheck("other-5", 5, "h5", "t", false), + aPullCheck("ready-6", 6, "h6", "t", true), + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + took := make(chan string, 8) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + var order []string + for len(order) < 6 { + select { + case id := <-took: + order = append(order, id) + case <-time.After(15 * time.Second): + t.Fatalf("taken so far: %v", order) + } + } + if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) { + t.Fatalf("taken as %v; want %v", order, want) + } +} + +func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) + putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged + putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false), + aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false), + aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false), + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + took := make(chan string, 8) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + var ran []string + for len(ran) < 2 { + select { + case id := <-took: + ran = append(ran, id) + case <-time.After(15 * time.Second): + t.Fatalf("run so far: %v", ran) + } + } + if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) { + t.Fatalf("run %v; want %v, the judged one dropped", ran, want) + } + time.Sleep(300 * time.Millisecond) + if n := len(queueOf(t, js).Asks); n != 0 { + t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n) + } +} + +func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + moved := make(chan PullKey, 4) + req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false) + end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end() + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"}) + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) + select { + case k := <-moved: + t.Fatalf("stopped by a key that moved nothing: %+v", k) + case <-time.After(500 * time.Millisecond): + } + putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"}) + select { + case k := <-moved: + if k.Head != "head-7-111111111" { + t.Errorf("stopped by %+v", k) + } + case <-time.After(5 * time.Second): + t.Fatal("a key whose head moved did not stop the check") + } + + // And the trunk commit. + req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false) + end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end2() + putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"}) + select { + case <-moved: + case <-time.After(5 * time.Second): + t.Fatal("a key whose trunk commit moved did not stop the check") + } + + // A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does + // have is the pull request's own. + group := BuildRequest{ID: "group-1", Ref: "head-7-000000000", + Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}} + end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k }) + if err != nil { + t.Fatal(err) + } + defer end3() + putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"}) + select { + case k := <-moved: + t.Fatalf("a group's check was stopped by %+v", k) + case <-time.After(500 * time.Millisecond): + } +} + +func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) { + js := aBusWithTheBuildRole(t) + ctx := t.Context() + asks := BuildsOn(js, TheBuildMachine) + for _, r := range []BuildRequest{ + aPullCheck("other-1", 1, "h1", "t", false), + aPullCheck("ready-2", 2, "h2", "t", true), + {ID: "delivery-3", Repository: "/d"}, + } { + if err := asks.Ask(ctx, r); err != nil { + t.Fatal(err) + } + } + q := queueOf(t, js) + var ids, streams []string + for _, a := range q.Asks { + ids, streams = append(ids, a.ID), append(streams, a.Stream) + } + if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) { + t.Fatalf("read %v; want the order they are handed out, %v", ids, want) + } + if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) { + t.Fatalf("streams %v; want %v", streams, want) + } + ready, _ := q.Find("ready-2") + if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker { + t.Errorf("the ready check is handed out by %s", w.Name) + } + if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq { + t.Fatalf("deleted %d, %v", removed, err) + } + if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 { + t.Fatalf("deleted again: %d, %v", removed, err) + } + if n := len(queueOf(t, js).Asks); n != 2 { + t.Errorf("%d asks left; want 2", n) + } +} + +func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) { + js := aBusWithTheBuildRole(t) + aKeyBucket(t, js) + aRunningBucket(t, js, 2*time.Minute) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + running := make(chan string, 4) + release := make(chan struct{}) + a := MachineOverNATS(js, "builder-a") + defer a.Close() + go func() { + _ = a.Take(ctx, func(ctx context.Context, work Build) { + running <- work.Request().ID + <-release + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case <-running: + case <-time.After(10 * time.Second): + t.Fatal("builder a took nothing") + } + // The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it. + if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + b := MachineOverNATS(js, "builder-b") + defer b.Close() + took := make(chan string, 4) + go func() { + _ = b.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-took: + t.Fatalf("builder b took %s, which builder a is running a check of", id) + case <-time.After(3 * time.Second): + } + if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 { + t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n) + } + // Another trunk commit is another check. + if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + if id != "moved" { + t.Fatalf("builder b took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("a check against another trunk commit was dropped too") + } + close(release) +} + +func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) { + js := aBusWithTheBuildRole(t) + aKeyBucket(t, js) + kv := aRunningBucket(t, js, 2*time.Second) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"}) + if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil { + t.Fatal(err) + } + asks := BuildsOn(js, TheBuildMachine) + took := make(chan string, 4) + b := MachineOverNATS(js, "builder-b") + defer b.Close() + go func() { + _ = b.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("%s was taken while the key was fresh", id) + case <-time.After(1200 * time.Millisecond): + } + time.Sleep(2500 * time.Millisecond) // the key was not renewed + if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + if id != "after-it-aged" { + t.Fatalf("took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("the request was not taken after the key aged out") + } +} + +func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aRunningBucket(t, js, 2*time.Second) + was := heartbeatWhileWorking + heartbeatWhileWorking = 400 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = was }) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + running := make(chan struct{}, 1) + release := make(chan struct{}) + m := MachineOverNATS(js, "builder-a") + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + running <- struct{}{} + <-release + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) + _ = work.Done() + }) + }() + if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + <-running + key := PullKeyOf("novox", "a", 7) + e, err := kv.Get(ctx, key) + if err != nil { + t.Fatalf("no key while the check runs: %v", err) + } + var got CheckRunning + _ = json.Unmarshal(e.Value(), &got) + if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" { + t.Errorf("the key says %+v", got) + } + time.Sleep(3 * time.Second) // longer than the bucket's age + if _, err := kv.Get(ctx, key); err != nil { + t.Fatalf("the key aged out under a check still running: %v", err) + } + close(release) + for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) { + if _, err := kv.Get(ctx, key); err != nil { + break + } + if time.Now().After(end) { + t.Fatal("the key stays after the check ended") + } + } +} + +func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) { + js := aBusWithTheBuildRole(t) + kv := aKeyBucket(t, js) + aRunningBucket(t, js, 2*time.Minute) + ctx, stop := context.WithCancel(t.Context()) + defer stop() + asks := BuildsOn(js, TheBuildMachine) + outcomes := make(chan BuildResult, 8) + watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) { + var r BuildResult + if json.Unmarshal(m.Data, &r) == nil { + outcomes <- r + } + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = watching.Unsubscribe() }() + + ran := make(chan string, 4) + m := MachineOverNATS(js, "builder-a") + defer m.Close() + go func() { + _ = m.Take(ctx, func(ctx context.Context, work Build) { + building, cancel := context.WithCancel(ctx) + defer cancel() + ran <- work.Request().ID + stopped := make(chan struct{}, 1) + end, err := work.(interface { + WatchPull(context.Context, func(PullKey)) (func(), error) + }).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() }) + if err != nil { + t.Error(err) + return + } + defer end() + if work.Request().ID == "older" { + select { + case <-stopped: + _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey}) + case <-time.After(15 * time.Second): + _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"}) + } + } else { + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}}) + } + _ = work.Done() + }) + }() + if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { + t.Fatal(err) + } + if id := <-ran; id != "older" { + t.Fatalf("took %s", id) + } + time.Sleep(300 * time.Millisecond) // the watch is made + // The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked. + putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"}) + if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { + t.Fatal(err) + } + select { + case id := <-ran: + if id != "newer" { + t.Fatalf("took %s", id) + } + case <-time.After(15 * time.Second): + t.Fatal("the newer request, for the same head on the new trunk commit, was not taken") + } + got := map[string]string{} + for len(got) < 2 { + select { + case r := <-outcomes: + got[r.ID] = r.Failed + case <-time.After(10 * time.Second): + t.Fatalf("outcomes so far: %v", got) + } + } + if got["older"] != SupersededByKey || got["newer"] != "" { + t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got) + } +} + +func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) { + for _, ready := range []bool{false, true} { + js := aBusWithTheBuildRole(t) + name, worker := "other", broker.ChecksOtherWorker + if ready { + name, worker = "ready", broker.ChecksReadyWorker + } + // A worker whose wait is a second, as the mesh's is a minute. + for _, w := range broker.CheckWorkers() { + if w.Name == worker { + w.AckWaitSeconds = 1 + if err := js.EnsureConsumer(w); err != nil { + t.Fatal(err) + } + } + } + ctx, stop := context.WithCancel(t.Context()) + if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil { + t.Fatal(err) + } + // The first builder takes it and dies: its pull is fetched, never acknowledged. + if _, err := fetchOne(js, worker); err != nil { + t.Fatal(err) + } + took := make(chan BuildRequest, 2) + second := MachineOverNATS(js, "builder-b") + go func() { + _ = second.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request() + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case r := <-took: + if r.ID != "dies-"+name { + t.Errorf("%s: took %s", name, r.ID) + } + case <-time.After(15 * time.Second): + t.Errorf("%s: the unacknowledged check did not go to a second builder", name) + } + second.Close() + stop() + } +} diff --git a/internal/link/checks_nats_test.go b/internal/link/checks_nats_test.go index b5a2f51c..e4a7bc7a 100644 --- a/internal/link/checks_nats_test.go +++ b/internal/link/checks_nats_test.go @@ -3,7 +3,6 @@ package link import ( "context" "encoding/json" - "reflect" "testing" "time" @@ -56,75 +55,9 @@ func queueOf(t *testing.T, js *broker.JetStream) Queue { // A newer request for a pull request replaces the one still waiting, by the stream's rule, and each open pull request is // left with one queued check (acceptance 5). -func TestANewerCheckRequestReplacesOneStillWaiting(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("check-7-a", 7, "head-a-0000000", "trunk-0-00000", false), - aPullCheck("check-8-a", 8, "head-b-0000000", "trunk-0-00000", false), - aPullCheck("check-7-b", 7, "head-a-0000000", "trunk-1-00000", false), // a merge moved the trunk - aPullCheck("check-8-b", 8, "head-b-0000000", "trunk-1-00000", false), - aPullCheck("check-7-c", 7, "head-a-0000000", "trunk-2-00000", false), // and a second merge within the minute - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - var got []string - for _, a := range queueOf(t, js).Asks { - if a.State != AskWaiting { - t.Errorf("%s is %s", a.ID, a.State) - } - got = append(got, a.ID) - } - if want := []string{"check-8-b", "check-7-c"}; !reflect.DeepEqual(got, want) { - t.Fatalf("queued %v; want one check a pull request, the newest: %v", got, want) - } -} // A delivery build is taken before a check of a ready pull request, that before any other check, and the others in the // order they were asked. -func TestADeliveryBuildIsTakenBeforeAReadyCheckAndThatBeforeAnyOther(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("other-1", 1, "h1", "t", false), - aPullCheck("other-2", 2, "h2", "t", false), - aPullCheck("ready-3", 3, "h3", "t", true), - {ID: "delivery-4", Repository: "/d"}, - aPullCheck("other-5", 5, "h5", "t", false), - aPullCheck("ready-6", 6, "h6", "t", true), - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - took := make(chan string, 8) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - var order []string - for len(order) < 6 { - select { - case id := <-took: - order = append(order, id) - case <-time.After(15 * time.Second): - t.Fatalf("taken so far: %v", order) - } - } - if want := []string{"delivery-4", "ready-3", "ready-6", "other-1", "other-2", "other-5"}; !reflect.DeepEqual(order, want) { - t.Fatalf("taken as %v; want %v", order, want) - } -} // aKeyBucket makes mesh-delivery's bucket of pull requests' keys, which the bus here has not been given by that module. func aKeyBucket(t *testing.T, js *broker.JetStream) jetstream.KeyValue { @@ -150,113 +83,9 @@ func putKey(t *testing.T, kv jetstream.KeyValue, number int, k PullKey) { // A request left on the third queue for a head and a trunk commit its key says were judged is acknowledged and dropped when a // builder takes it; one for a head the key does not say judged is run. -func TestARequestLeftBehindForAJudgedHeadIsDroppedWhenTaken(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aKeyBucket(t, js) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) - putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-1-000000"}) // asked, not judged - putKey(t, kv, 9, PullKey{Head: "head-9-000000000", Trunk: "trunk-2-000000", Verdict: "pass"}) // judged against another trunk - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("left-7", 7, "head-7-000000000", "trunk-1-000000", false), - aPullCheck("fresh-8", 8, "head-8-000000000", "trunk-1-000000", false), - aPullCheck("moved-9", 9, "head-9-000000000", "trunk-1-000000", false), - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - took := make(chan string, 8) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - var ran []string - for len(ran) < 2 { - select { - case id := <-took: - ran = append(ran, id) - case <-time.After(15 * time.Second): - t.Fatalf("run so far: %v", ran) - } - } - if want := []string{"fresh-8", "moved-9"}; !reflect.DeepEqual(ran, want) { - t.Fatalf("run %v; want %v, the judged one dropped", ran, want) - } - time.Sleep(300 * time.Millisecond) - if n := len(queueOf(t, js).Asks); n != 0 { - t.Errorf("%d ask(s) are left in the queues; the dropped one was to be acknowledged", n) - } -} // A running check stops when its key's head moves or its trunk commit does, not when the key only gains its verdict, and a // group's check, which a member's merge in the group's order writes nothing to the key of, is not stopped at all. -func TestARunningCheckStopsWhenItsKeysHeadMovesAndAGroupsDoesNot(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aKeyBucket(t, js) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - moved := make(chan PullKey, 4) - req := aPullCheck("check-7", 7, "head-7-000000000", "trunk-1-000000", false) - end, err := WatchPull(ctx, js.Conn(), req, func(k PullKey) { moved <- k }) - if err != nil { - t.Fatal(err) - } - defer end() - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000"}) - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-1-000000", Verdict: "pass"}) - select { - case k := <-moved: - t.Fatalf("stopped by a key that moved nothing: %+v", k) - case <-time.After(500 * time.Millisecond): - } - putKey(t, kv, 7, PullKey{Head: "head-7-111111111", Trunk: "trunk-1-000000"}) - select { - case k := <-moved: - if k.Head != "head-7-111111111" { - t.Errorf("stopped by %+v", k) - } - case <-time.After(5 * time.Second): - t.Fatal("a key whose head moved did not stop the check") - } - - // And the trunk commit. - req2 := aPullCheck("check-8", 8, "head-8-000000000", "trunk-1-000000", false) - end2, err := WatchPull(ctx, js.Conn(), req2, func(k PullKey) { moved <- k }) - if err != nil { - t.Fatal(err) - } - defer end2() - putKey(t, kv, 8, PullKey{Head: "head-8-000000000", Trunk: "trunk-2-000000"}) - select { - case <-moved: - case <-time.After(5 * time.Second): - t.Fatal("a key whose trunk commit moved did not stop the check") - } - - // A group's check watches nothing: a sibling's merge in the group's order writes nothing to a key, and a key it does - // have is the pull request's own. - group := BuildRequest{ID: "group-1", Ref: "head-7-000000000", - Check: &CheckRequest{Owner: "novox", Repo: "a", Number: 7, Group: "g1"}} - end3, err := WatchPull(ctx, js.Conn(), group, func(k PullKey) { moved <- k }) - if err != nil { - t.Fatal(err) - } - defer end3() - putKey(t, kv, 7, PullKey{Head: "head-7-222222222", Trunk: "trunk-9-000000"}) - select { - case k := <-moved: - t.Fatalf("a group's check was stopped by %+v", k) - case <-time.After(500 * time.Millisecond): - } -} func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) { k := PullKey{Head: "aaaaaaaaaaaa1111", Trunk: "tttttttttttt1111", Verdict: "pass"} @@ -270,44 +99,6 @@ func TestAPullKeyIsMovedByHeadOrTrunkAndJudgedByVerdict(t *testing.T) { // The queue is read across the three streams in the order they are handed out, each ask says the stream it waits in, and an // ask is removed from that stream. -func TestTheQueueIsReadAcrossTheThreeStreamsAndAnAskIsDeletedFromItsOwn(t *testing.T) { - js := aBusWithTheBuildRole(t) - ctx := t.Context() - asks := BuildsOn(js, TheBuildMachine) - for _, r := range []BuildRequest{ - aPullCheck("other-1", 1, "h1", "t", false), - aPullCheck("ready-2", 2, "h2", "t", true), - {ID: "delivery-3", Repository: "/d"}, - } { - if err := asks.Ask(ctx, r); err != nil { - t.Fatal(err) - } - } - q := queueOf(t, js) - var ids, streams []string - for _, a := range q.Asks { - ids, streams = append(ids, a.ID), append(streams, a.Stream) - } - if want := []string{"delivery-3", "ready-2", "other-1"}; !reflect.DeepEqual(ids, want) { - t.Fatalf("read %v; want the order they are handed out, %v", ids, want) - } - if want := []string{"SEAT_NODE_BUILD_AGENT", broker.ChecksReadyStream, broker.ChecksOtherStream}; !reflect.DeepEqual(streams, want) { - t.Fatalf("streams %v; want %v", streams, want) - } - ready, _ := q.Find("ready-2") - if w := WorkerOf(TheBuildMachine, ready); w.Name != broker.ChecksReadyWorker { - t.Errorf("the ready check is handed out by %s", w.Name) - } - if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != ready.Seq { - t.Fatalf("deleted %d, %v", removed, err) - } - if removed, err := DeleteAsk(ctx, js, TheBuildMachine, ready); err != nil || removed != 0 { - t.Fatalf("deleted again: %d, %v", removed, err) - } - if n := len(queueOf(t, js).Asks); n != 2 { - t.Errorf("%d asks left; want 2", n) - } -} // aRunningBucket makes the bucket the builders say the checks they run in, with an age of its own short enough for a test // (the mesh's is two minutes): a key not renewed within it is gone. @@ -326,165 +117,12 @@ func aRunningBucket(t *testing.T, js *broker.JetStream, age time.Duration) jetst // A builder taking a request for a pull request's head and trunk commit that another builder is running a check of drops it: // the bucket of running checks names the head and the trunk, the builder and the build (issue 974). -func TestARunningCheckOnOneBuilderMakesAnotherDropTheSameRequestWaiting(t *testing.T) { - js := aBusWithTheBuildRole(t) - aKeyBucket(t, js) - aRunningBucket(t, js, 2*time.Minute) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - running := make(chan string, 4) - release := make(chan struct{}) - a := MachineOverNATS(js, "builder-a") - defer a.Close() - go func() { - _ = a.Take(ctx, func(ctx context.Context, work Build) { - running <- work.Request().ID - <-release - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) - _ = work.Done() - }) - }() - if err := asks.Ask(ctx, aPullCheck("first", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - select { - case <-running: - case <-time.After(10 * time.Second): - t.Fatal("builder a took nothing") - } - // The same head and trunk commit asked again while it runs, and a pull request's other head asked beside it. - if err := asks.Ask(ctx, aPullCheck("again", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - b := MachineOverNATS(js, "builder-b") - defer b.Close() - took := make(chan string, 4) - go func() { - _ = b.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) - _ = work.Done() - }) - }() - select { - case id := <-took: - t.Fatalf("builder b took %s, which builder a is running a check of", id) - case <-time.After(3 * time.Second): - } - if n := len(queueOf(t, js).Of(AskWaiting)); n != 0 { - t.Errorf("%d request(s) still wait; builder b was to drop the one for a head and trunk commit being checked", n) - } - // Another trunk commit is another check. - if err := asks.Ask(ctx, aPullCheck("moved", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - if id != "moved" { - t.Fatalf("builder b took %s", id) - } - case <-time.After(10 * time.Second): - t.Fatal("a check against another trunk commit was dropped too") - } - close(release) -} // A key that is not renewed ages out, and the request for its head and trunk commit is then taken: a builder that died // running a check leaves nothing that holds the next one back for longer than the bucket's age. -func TestAKeyNotRenewedAgesOutAndTheRequestIsThenTaken(t *testing.T) { - js := aBusWithTheBuildRole(t) - aKeyBucket(t, js) - kv := aRunningBucket(t, js, 2*time.Second) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - body, _ := json.Marshal(CheckRunning{Head: "head-7-000000000", Trunk: "trunk-1-000000", Builder: "dead", Build: "gone"}) - if _, err := kv.Put(ctx, PullKeyOf("novox", "a", 7), body); err != nil { - t.Fatal(err) - } - asks := BuildsOn(js, TheBuildMachine) - took := make(chan string, 4) - b := MachineOverNATS(js, "builder-b") - defer b.Close() - go func() { - _ = b.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) - _ = work.Done() - }) - }() - if err := asks.Ask(ctx, aPullCheck("while-it-lives", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - t.Fatalf("%s was taken while the key was fresh", id) - case <-time.After(1200 * time.Millisecond): - } - time.Sleep(2500 * time.Millisecond) // the key was not renewed - if err := asks.Ask(ctx, aPullCheck("after-it-aged", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - if id != "after-it-aged" { - t.Fatalf("took %s", id) - } - case <-time.After(10 * time.Second): - t.Fatal("the request was not taken after the key aged out") - } -} // A builder running a check renews its key on each heartbeat, so it outlives the bucket's age, and deletes it when the check // ends, whatever the outcome. -func TestARunningCheckRenewsItsKeyAndDeletesItWhenItEnds(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aRunningBucket(t, js, 2*time.Second) - was := heartbeatWhileWorking - heartbeatWhileWorking = 400 * time.Millisecond - t.Cleanup(func() { heartbeatWhileWorking = was }) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - running := make(chan struct{}, 1) - release := make(chan struct{}) - m := MachineOverNATS(js, "builder-a") - defer m.Close() - go func() { - _ = m.Take(ctx, func(ctx context.Context, work Build) { - running <- struct{}{} - <-release - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Failed: "no"}) - _ = work.Done() - }) - }() - if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("long", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - <-running - key := PullKeyOf("novox", "a", 7) - e, err := kv.Get(ctx, key) - if err != nil { - t.Fatalf("no key while the check runs: %v", err) - } - var got CheckRunning - _ = json.Unmarshal(e.Value(), &got) - if got.Head != "head-7-000000000" || got.Trunk != "trunk-1-000000" || got.Builder != "builder-a" || got.Build != "long" { - t.Errorf("the key says %+v", got) - } - time.Sleep(3 * time.Second) // longer than the bucket's age - if _, err := kv.Get(ctx, key); err != nil { - t.Fatalf("the key aged out under a check still running: %v", err) - } - close(release) - for end := time.Now().Add(5 * time.Second); ; time.Sleep(50 * time.Millisecond) { - if _, err := kv.Get(ctx, key); err != nil { - break - } - if time.Now().After(end) { - t.Fatal("the key stays after the check ended") - } - } -} // A key whose verdict is "error" — a check that could not run, or was stopped — judged nothing: a request for that head and // trunk commit must run (issue 974, review 2). @@ -499,136 +137,9 @@ func TestAnErrorVerdictJudgesNothing(t *testing.T) { // A check stopped because the trunk moved leaves the newer request, for the head against the new trunk commit, taken and // run: the stop is the stopped one's failure, not the key's verdict, and the newer request is not dropped by it. -func TestACheckStoppedByATrunkMoveLeavesTheNewerRequestTakenAndRun(t *testing.T) { - js := aBusWithTheBuildRole(t) - kv := aKeyBucket(t, js) - aRunningBucket(t, js, 2*time.Minute) - ctx, stop := context.WithCancel(t.Context()) - defer stop() - asks := BuildsOn(js, TheBuildMachine) - outcomes := make(chan BuildResult, 8) - watching, err := js.Conn().Subscribe(BuildOutcome(), func(m *nats.Msg) { - var r BuildResult - if json.Unmarshal(m.Data, &r) == nil { - outcomes <- r - } - }) - if err != nil { - t.Fatal(err) - } - defer func() { _ = watching.Unsubscribe() }() - - ran := make(chan string, 4) - m := MachineOverNATS(js, "builder-a") - defer m.Close() - go func() { - _ = m.Take(ctx, func(ctx context.Context, work Build) { - building, cancel := context.WithCancel(ctx) - defer cancel() - ran <- work.Request().ID - stopped := make(chan struct{}, 1) - end, err := work.(interface { - WatchPull(context.Context, func(PullKey)) (func(), error) - }).WatchPull(building, func(PullKey) { stopped <- struct{}{}; cancel() }) - if err != nil { - t.Error(err) - return - } - defer end() - if work.Request().ID == "older" { - select { - case <-stopped: - _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: SupersededByKey}) - case <-time.After(15 * time.Second): - _ = work.Announce(ctx, BuildResult{ID: "older", On: "builder-a", Failed: "never stopped"}) - } - } else { - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-a", Check: &CheckOutcome{Verdict: "pass"}}) - } - _ = work.Done() - }) - }() - if err := asks.Ask(ctx, aPullCheck("older", 7, "head-7-000000000", "trunk-1-000000", false)); err != nil { - t.Fatal(err) - } - if id := <-ran; id != "older" { - t.Fatalf("took %s", id) - } - time.Sleep(300 * time.Millisecond) // the watch is made - // The trunk moved; the head did not. mesh-delivery writes the key for the new request, and the request is asked. - putKey(t, kv, 7, PullKey{Head: "head-7-000000000", Trunk: "trunk-2-000000"}) - if err := asks.Ask(ctx, aPullCheck("newer", 7, "head-7-000000000", "trunk-2-000000", false)); err != nil { - t.Fatal(err) - } - select { - case id := <-ran: - if id != "newer" { - t.Fatalf("took %s", id) - } - case <-time.After(15 * time.Second): - t.Fatal("the newer request, for the same head on the new trunk commit, was not taken") - } - got := map[string]string{} - for len(got) < 2 { - select { - case r := <-outcomes: - got[r.ID] = r.Failed - case <-time.After(10 * time.Second): - t.Fatalf("outcomes so far: %v", got) - } - } - if got["older"] != SupersededByKey || got["newer"] != "" { - t.Errorf("outcomes %v; want the older stopped and the newer run to a verdict", got) - } -} // An unacknowledged check goes to a second builder: taken from a check worker and not acknowledged, it is handed out again // when the worker's wait is up. -func TestAnUnacknowledgedCheckGoesToASecondBuilder(t *testing.T) { - for _, ready := range []bool{false, true} { - js := aBusWithTheBuildRole(t) - name, worker := "other", broker.ChecksOtherWorker - if ready { - name, worker = "ready", broker.ChecksReadyWorker - } - // A worker whose wait is a second, as the mesh's is a minute. - for _, w := range broker.CheckWorkers() { - if w.Name == worker { - w.AckWaitSeconds = 1 - if err := js.EnsureConsumer(w); err != nil { - t.Fatal(err) - } - } - } - ctx, stop := context.WithCancel(t.Context()) - if err := BuildsOn(js, TheBuildMachine).Ask(ctx, aPullCheck("dies-"+name, 7, "head-7-000000000", "trunk-1-000000", ready)); err != nil { - t.Fatal(err) - } - // The first builder takes it and dies: its pull is fetched, never acknowledged. - if _, err := fetchOne(js, worker); err != nil { - t.Fatal(err) - } - took := make(chan BuildRequest, 2) - second := MachineOverNATS(js, "builder-b") - go func() { - _ = second.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request() - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "builder-b", Failed: "no"}) - _ = work.Done() - }) - }() - select { - case r := <-took: - if r.ID != "dies-"+name { - t.Errorf("%s: took %s", name, r.ID) - } - case <-time.After(15 * time.Second): - t.Errorf("%s: the unacknowledged check did not go to a second builder", name) - } - second.Close() - stop() - } -} // fetchOne takes the next message of a worker and leaves it unacknowledged, as a builder that died would. func fetchOne(js *broker.JetStream, worker string) (*nats.Msg, error) { diff --git a/internal/link/conformance_integration_test.go b/internal/link/conformance_integration_test.go new file mode 100644 index 00000000..f7897c67 --- /dev/null +++ b/internal/link/conformance_integration_test.go @@ -0,0 +1,148 @@ +//go:build integration + +package link + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/beside" +) + +func loadFixture(t *testing.T, name string) fixture { + t.Helper() + raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name) + if err != nil { + // Failed, never skipped: a skip here passed the suite with nothing judged. + t.Fatal(err) + } + t.Logf("%s: %s", name, from) + if strings.HasPrefix(from, notJudged) { + // Said on the check's own output too, where `go test` without -v prints no log of a passing test. + saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) }) + } + var f fixture + if err := json.Unmarshal(raw, &f); err != nil { + t.Fatalf("%s: %v", name, err) + } + return f +} + +func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { + file := path.Join("conformance", name) + readCaptured := func() ([]byte, error) { + raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, fmt.Errorf("the captured SDK's %s: %w", file, err) + } + return raw, nil + } + if root == "" { + raw, err := readCaptured() + return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + + "at this tree's pin", err + } + clone := filepath.Join(root, "mesh-sdk") + if _, err := os.Stat(clone); err != nil { + // A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the + // check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the + // captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this + // a failure again. + at, cerr := capturedCommit(capturedLog) + if cerr != nil { + return nil, "", cerr + } + raw, rerr := readCaptured() + return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+ + "%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env, + root, at), rerr + } + ref, err := sdkRef(goMod) + if err != nil { + return nil, "", err + } + raw, err := gitShow(clone, ref, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+ + "go.mod pins: %w", file, ref, err) + } + // The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so. + at, err := capturedCommit(capturedLog) + if err != nil { + return nil, "", err + } + theirs, err := gitShow(clone, at, file) + if err != nil { + return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err) + } + ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) + if err != nil { + return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) + } + if !bytes.Equal(ours, theirs) { + return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+ + "edited or captured wrong; capture it again as CAPTURED says", file, at) + } + return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil +} + +func gitShow(repository, ref, file string) ([]byte, error) { + cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file) + var stderr bytes.Buffer + cmd.Stderr = &stderr + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + for _, want := range f.Wire.RequiredHeaders { + if _, ok := sent[want]; !ok { + t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+ + "emits is one a conforming consumer refuses", want) + } + } +} + +func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) + + if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] { + t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"]) + } + if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil { + t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err) + } + if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" { + if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) { + t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern) + } + } + // The origin the envelope claims is the one the bus enforces by namespace. A disagreement + // here means the envelope is lying about where it came from. + if sent["x-source"] != f.Given.Module { + t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module) + } +} + +func TestTheGoSubjectMatchesTheFixture(t *testing.T) { + f := loadFixture(t, "events/module-event.json") + got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key + if got != f.Wire.Subject { + t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject) + } +} diff --git a/internal/link/conformance_sources_test.go b/internal/link/conformance_sources_test.go index 06a0fcd7..1b58ba9c 100644 --- a/internal/link/conformance_sources_test.go +++ b/internal/link/conformance_sources_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/conformance_test.go b/internal/link/conformance_test.go index 048ba4f3..ed9cd038 100644 --- a/internal/link/conformance_test.go +++ b/internal/link/conformance_test.go @@ -1,12 +1,8 @@ package link import ( - "bytes" - "encoding/json" "fmt" "os" - "os/exec" - "path" "path/filepath" "regexp" "strings" @@ -61,25 +57,6 @@ var ( capturedLog = filepath.Join(beside.Captured(), "CAPTURED") ) -func loadFixture(t *testing.T, name string) fixture { - t.Helper() - raw, from, err := sdkFixture(os.Getenv(beside.Env), goModFile, capturedSDK, capturedLog, name) - if err != nil { - // Failed, never skipped: a skip here passed the suite with nothing judged. - t.Fatal(err) - } - t.Logf("%s: %s", name, from) - if strings.HasPrefix(from, notJudged) { - // Said on the check's own output too, where `go test` without -v prints no log of a passing test. - saidNotJudged.Do(func() { fmt.Fprintln(os.Stderr, "internal/link conformance: "+from) }) - } - var f fixture - if err := json.Unmarshal(raw, &f); err != nil { - t.Fatalf("%s: %v", name, err) - } - return f -} - // notJudged opens what a merge check without the SDK's clone says; saidNotJudged says it once on stderr. const notJudged = "NOT JUDGED AGAINST THE CLONE" @@ -87,76 +64,9 @@ var saidNotJudged sync.Once // sdkFixture is one conformance fixture and where it was read: from the mesh-sdk clone in root (a merge // check's MESH_CHECK_BESIDE) at the pin of goMod, or, with no root, from the captured copy. -func sdkFixture(root, goMod, captured, capturedLog, name string) ([]byte, string, error) { - file := path.Join("conformance", name) - readCaptured := func() ([]byte, error) { - raw, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) - if err != nil { - return nil, fmt.Errorf("the captured SDK's %s: %w", file, err) - } - return raw, nil - } - if root == "" { - raw, err := readCaptured() - return raw, "the copy captured in testdata/beside (see CAPTURED); a merge check reads the SDK's clone " + - "at this tree's pin", err - } - clone := filepath.Join(root, "mesh-sdk") - if _, err := os.Stat(clone); err != nil { - // A build seat asked by a controller from before mesh-controller #220 places no mesh-sdk beside the - // check, so #220 could never pass its own check if this failed. Until #220 has rolled out, the - // captured copy is read and the log says so loudly; the follow-up of novox/hq issue 449 makes this - // a failure again. - at, cerr := capturedCommit(capturedLog) - if cerr != nil { - return nil, "", cerr - } - raw, rerr := readCaptured() - return raw, fmt.Sprintf(notJudged+": the seat placed no mesh-sdk beside this check in "+ - "%s=%s (it does once mesh-controller #220 has rolled out); read the captured copy at %s", beside.Env, - root, at), rerr - } - ref, err := sdkRef(goMod) - if err != nil { - return nil, "", err - } - raw, err := gitShow(clone, ref, file) - if err != nil { - return nil, "", fmt.Errorf("the mesh-sdk clone beside this check has no %s at %s, the SDK this tree's "+ - "go.mod pins: %w", file, ref, err) - } - // The captured copy is the SDK's own, never edited by hand: the clone at the captured commit says so. - at, err := capturedCommit(capturedLog) - if err != nil { - return nil, "", err - } - theirs, err := gitShow(clone, at, file) - if err != nil { - return nil, "", fmt.Errorf("the mesh-sdk clone has no %s at %s, the commit CAPTURED names: %w", file, at, err) - } - ours, err := os.ReadFile(filepath.Join(captured, filepath.FromSlash(file))) - if err != nil { - return nil, "", fmt.Errorf("the captured SDK's %s: %w", file, err) - } - if !bytes.Equal(ours, theirs) { - return nil, "", fmt.Errorf("the captured %s is not mesh-sdk's at %s, the commit CAPTURED names: it was "+ - "edited or captured wrong; capture it again as CAPTURED says", file, at) - } - return raw, "mesh-sdk cloned beside this check, at " + ref + " (this tree's go.mod)", nil -} // gitShow is one file of a repository at a ref. safe.directory, because the clone is the build seat's and // the test may run as another user. -func gitShow(repository, ref, file string) ([]byte, error) { - cmd := exec.Command("git", "-c", "safe.directory=*", "-C", repository, "show", ref+":"+file) - var stderr bytes.Buffer - cmd.Stderr = &stderr - out, err := cmd.Output() - if err != nil { - return nil, fmt.Errorf("git show %s:%s: %v: %s", ref, file, err, strings.TrimSpace(stderr.String())) - } - return out, nil -} // pseudoCommit is the commit a Go pseudo-version names: v0.1.11-0.20261009143344-f047d0a4a970 → f047d0a4a970. var pseudoCommit = regexp.MustCompile(`-([0-9a-f]{12})$`) @@ -216,50 +126,12 @@ func TestTheCapturedSDKIsTheOneGoModPins(t *testing.T) { } // Every header the fixture requires is one this implementation actually sets. -func TestTheGoEmitterSetsEveryRequiredHeader(t *testing.T) { - f := loadFixture(t, "events/module-event.json") - sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) - for _, want := range f.Wire.RequiredHeaders { - if _, ok := sent[want]; !ok { - t.Errorf("the Go emitter does not set %q, which the fixture requires — an event it "+ - "emits is one a conforming consumer refuses", want) - } - } -} // And each value is in the shape the fixture pins, because a header present but differently // formatted is the disagreement that does not announce itself. -func TestTheGoEmittersHeaderFormatsMatch(t *testing.T) { - f := loadFixture(t, "events/module-event.json") - sent := goEventHeaders(f.Given.Key, f.Given.Module, f.Given.Node) - - if got := sent["content-type"]; got != f.Wire.HeaderFormats["content-type"] { - t.Errorf("content-type is %q, the fixture says %q", got, f.Wire.HeaderFormats["content-type"]) - } - if _, err := time.Parse(time.RFC3339, sent["x-time"]); err != nil { - t.Errorf("x-time %q is not RFC3339, which the fixture requires: %v", sent["x-time"], err) - } - if pattern := f.Wire.HeaderFormats["x-event-id"]; pattern != "" { - if !regexp.MustCompile(pattern).MatchString(sent["x-event-id"]) { - t.Errorf("x-event-id %q does not match %q", sent["x-event-id"], pattern) - } - } - // The origin the envelope claims is the one the bus enforces by namespace. A disagreement - // here means the envelope is lying about where it came from. - if sent["x-source"] != f.Given.Module { - t.Errorf("x-source is %q for module %q", sent["x-source"], f.Given.Module) - } -} // The subject a module's event lands on is derived, not carried — so this implementation must // derive the same one the fixture names. -func TestTheGoSubjectMatchesTheFixture(t *testing.T) { - f := loadFixture(t, "events/module-event.json") - got := "mesh.mod." + f.Given.Module + ".event." + f.Given.Key - if got != f.Wire.Subject { - t.Errorf("this implementation would publish on %q; the fixture says %q", got, f.Wire.Subject) - } -} // goEventHeaders is the header set EmitEvent produces, factored so conformance can see it // without a broker. Kept beside the emitter so the two cannot drift apart silently. diff --git a/internal/link/deadletters_ask_test.go b/internal/link/deadletters_ask_test.go index 2def6f3e..f3533527 100644 --- a/internal/link/deadletters_ask_test.go +++ b/internal/link/deadletters_ask_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/deadletters_integration_test.go b/internal/link/deadletters_integration_test.go new file mode 100644 index 00000000..23d98fe2 --- /dev/null +++ b/internal/link/deadletters_integration_test.go @@ -0,0 +1,279 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" +) + +func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) { + js := aBus(t) + keeping(t, js) + failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") + handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged") + + const subject = "mesh.mod.gitea.event.pull.merged" + msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}} + msg.Header.Set("x-node", "forge") + msg.Header.Set(nats.MsgIdHdr, "event-1") + if _, err := js.Context().PublishMsg(msg); err != nil { + t.Fatal(err) + } + if m := next(t, handling); m == nil { + t.Fatal("the other module was not handed the event") + } else { + _ = m.Ack() + } + d := givenUp(t, js, failing) + if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 || + d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" { + t.Fatalf("kept as %+v", d) + } + held, err := HeldDeadLetters(js.Context()) + if err != nil || held["EVENTS.media_sonarr"] != 1 { + t.Fatalf("held %v (%v)", held, err) + } + + whole, err := DeadLetterNamed(js.Context(), d.ID) + if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" { + t.Fatalf("one asked whole is %+v (%v)", whole, err) + } + + _, to, err := DeliverAgain(js.Context(), d.ID) + if err != nil { + t.Fatal(err) + } + if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" { + t.Fatalf("delivered again on %s", to) + } + again := next(t, failing) + if again == nil { + t.Fatal("the consumer that gave it up was not handed it again") + } + if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" || + again.Headers().Get(AgainHeader) == "" { + t.Fatalf("handed again as %s %v", again.Data(), again.Headers()) + } + if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject { + t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject()) + } + _ = again.Ack() + if m := next(t, handling); m != nil { + t.Fatalf("the module that handled it was handed it twice: %s", m.Subject()) + } + if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { + t.Fatalf("still kept after it was delivered again: %v", err) + } + held, _ = HeldDeadLetters(js.Context()) + if held["EVENTS.media_sonarr"] != 0 { + t.Fatalf("still held: %v", held) + } + // Asked twice, delivered once: the second finds nothing kept. + if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { + t.Fatalf("a second delivery answered %v", err) + } +} + +func TestADeadLetterDroppedIsGone(t *testing.T) { + js := aBus(t) + keeping(t, js) + failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") + if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { + t.Fatal(err) + } + d := givenUp(t, js, failing) + if _, err := DropDeadLetter(js.Context(), d.ID); err != nil { + t.Fatal(err) + } + if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 { + t.Fatalf("after the drop: %v %d %v", left, total, err) + } + if m := next(t, failing); m != nil { + t.Fatalf("a dropped event was handed over: %s", m.Subject()) + } +} + +func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) { + js := aBus(t) + d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ + `"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" { + t.Fatalf("kept as %+v", d) + } + if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") { + t.Fatalf("a record without its message was delivered again: %v", err) + } + // The same notice taken twice is kept once. + if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ + `"deliveries":5}`)); err != nil { + t.Fatal(err) + } + if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 { + t.Fatalf("one notice taken twice is kept %d times", total) + } +} + +func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) { + js := aBus(t) + keeping(t, js) + failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") + if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { + t.Fatal(err) + } + d := givenUp(t, js, failing) + + // A consumer as it was before this fix: its filters do not take what is delivered again. + c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr", + Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"}) + c.Filters = c.Filters[:len(c.Filters)-1] + if err := js.EnsureConsumer(c); err != nil { + t.Fatal(err) + } + if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") { + t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err) + } + if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { + t.Fatalf("a refused delivery let the dead letter go: %v", err) + } + + // The consumer gone: nothing would receive it. + if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil { + t.Fatal(err) + } + if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") { + t.Fatalf("delivered to a consumer that is gone: %v", err) + } + if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { + t.Fatalf("a refused delivery let the dead letter go: %v", err) + } + + // Not an event: refused, whatever stream it is from. + for _, other := range []DeadLetter{ + {ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"}, + {ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"}, + } { + if _, err := AgainTo(js.Context(), other); err == nil { + t.Errorf("%s was given a subject to be delivered again on", other.Stream) + } + } +} + +func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) { + js := aBus(t) + for seq := 1; seq <= 5; seq++ { + if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+ + `"stream_seq":%d,"deliveries":5}`, seq))); err != nil { + t.Fatal(err) + } + } + list, total, err := DeadLetters(js.Context(), "media_sonarr", 2) + if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 { + t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list) + } + if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 { + t.Fatalf("another consumer's total is %d", total) + } +} + +func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) { + js := aBus(t) + told := &toldAbout{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(told); err != nil { + t.Fatal(err) + } + if err := s.Answers(replaysWith{}); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + eventually(t, "the controller's event consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil + }) + moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil { + t.Fatal(err) + } + eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 }) +} + +func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) { + js := aBus(t) + if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil { + t.Fatal(err) + } + stop := keepingGivenUp(context.Background(), js, quiet()) + defer stop() + eventually(t, "the failure said", func() bool { + for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) { + if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream { + return true + } + } + return false + }) + held := &counted{} + _, stopServing := servingOn(t, js, held) + defer stopServing() + if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil { + t.Fatal(err) + } + eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 }) +} + +func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) { + js := aBus(t) + gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) + keep := func(consumer string, seq int, at time.Time) { + t.Helper() + if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+ + `"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil { + t.Fatal(err) + } + } + look := func() map[string]time.Time { + t.Helper() + held, err := HeldDeadLetters(js.Context()) + if err != nil { + t.Fatal(err) + } + newest, err := NewestDeadLetters(js.Context(), held) + if err != nil { + t.Fatal(err) + } + return newest + } + + // Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it. + keep("media_sonarr", 2, gaveUp.Add(time.Minute)) + keep("media_radarr", 3, gaveUp) + before := look() + if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() || + !before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) { + t.Fatalf("%v", before) + } + // Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen. + keep("media_sonarr", 1, gaveUp) + after := look() + if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) || + !after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) { + t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after) + } + if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) { + t.Fatalf("radarr's newest moved: before %v, after %v", before, after) + } +} diff --git a/internal/link/deadletters_module_stream_test.go b/internal/link/deadletters_module_stream_test.go index ce08a68c..ec075178 100644 --- a/internal/link/deadletters_module_stream_test.go +++ b/internal/link/deadletters_module_stream_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/deadletters_restricted_test.go b/internal/link/deadletters_restricted_test.go index c621bbc2..aefad688 100644 --- a/internal/link/deadletters_restricted_test.go +++ b/internal/link/deadletters_restricted_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/deadletters_test.go b/internal/link/deadletters_test.go index f575c4ce..3db61457 100644 --- a/internal/link/deadletters_test.go +++ b/internal/link/deadletters_test.go @@ -1,17 +1,12 @@ package link import ( - "context" - "encoding/json" - "errors" - "fmt" "io" "log" "strings" "testing" "time" - "github.com/nats-io/nats.go" "github.com/nats-io/nats.go/jetstream" "github.com/novox/mesh-controller/internal/broker" @@ -85,117 +80,9 @@ func givenUp(t *testing.T, js *broker.JetStream, c jetstream.Consumer) DeadLette // **Delivered again to the consumer that gave it up, and nobody else**: another module consuming the // same event handled it the first time and must not handle it twice. -func TestADeadLetterIsDeliveredAgainToItsConsumerAlone(t *testing.T) { - js := aBus(t) - keeping(t, js) - failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") - handling := aModuleConsumer(t, js, "media", "radarr", "gitea.pull.merged") - - const subject = "mesh.mod.gitea.event.pull.merged" - msg := &nats.Msg{Subject: subject, Data: []byte(`{"n":1}`), Header: nats.Header{}} - msg.Header.Set("x-node", "forge") - msg.Header.Set(nats.MsgIdHdr, "event-1") - if _, err := js.Context().PublishMsg(msg); err != nil { - t.Fatal(err) - } - if m := next(t, handling); m == nil { - t.Fatal("the other module was not handed the event") - } else { - _ = m.Ack() - } - d := givenUp(t, js, failing) - if d.Consumer != "media_sonarr" || d.Stream != "EVENTS" || d.Subject != subject || d.Deliveries != 5 || - d.GaveUp.IsZero() || d.Published.IsZero() || d.Lost != "" { - t.Fatalf("kept as %+v", d) - } - held, err := HeldDeadLetters(js.Context()) - if err != nil || held["EVENTS.media_sonarr"] != 1 { - t.Fatalf("held %v (%v)", held, err) - } - - whole, err := DeadLetterNamed(js.Context(), d.ID) - if err != nil || whole.Body != `{"n":1}` || len(whole.Headers["x-node"]) != 1 || whole.Headers["x-node"][0] != "forge" { - t.Fatalf("one asked whole is %+v (%v)", whole, err) - } - - _, to, err := DeliverAgain(js.Context(), d.ID) - if err != nil { - t.Fatal(err) - } - if to != "mesh.again.media_sonarr.mod.gitea.event.pull.merged" { - t.Fatalf("delivered again on %s", to) - } - again := next(t, failing) - if again == nil { - t.Fatal("the consumer that gave it up was not handed it again") - } - if string(again.Data()) != `{"n":1}` || again.Headers().Get("x-node") != "forge" || - again.Headers().Get(AgainHeader) == "" { - t.Fatalf("handed again as %s %v", again.Data(), again.Headers()) - } - if original, ok := broker.OriginalOfAgain(again.Subject()); !ok || original != subject { - t.Fatalf("delivered again on %s, which does not say the event's own subject", again.Subject()) - } - _ = again.Ack() - if m := next(t, handling); m != nil { - t.Fatalf("the module that handled it was handed it twice: %s", m.Subject()) - } - if _, err := DeadLetterNamed(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { - t.Fatalf("still kept after it was delivered again: %v", err) - } - held, _ = HeldDeadLetters(js.Context()) - if held["EVENTS.media_sonarr"] != 0 { - t.Fatalf("still held: %v", held) - } - // Asked twice, delivered once: the second finds nothing kept. - if _, _, err := DeliverAgain(js.Context(), d.ID); !errors.Is(err, ErrNoDeadLetter) { - t.Fatalf("a second delivery answered %v", err) - } -} - -func TestADeadLetterDroppedIsGone(t *testing.T) { - js := aBus(t) - keeping(t, js) - failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") - if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { - t.Fatal(err) - } - d := givenUp(t, js, failing) - if _, err := DropDeadLetter(js.Context(), d.ID); err != nil { - t.Fatal(err) - } - if left, total, err := DeadLetters(js.Context(), "", 0); err != nil || total != 0 || len(left) != 0 { - t.Fatalf("after the drop: %v %d %v", left, total, err) - } - if m := next(t, failing); m != nil { - t.Fatalf("a dropped event was handed over: %s", m.Subject()) - } -} // A notice whose message its stream no longer holds is kept as a record that says so — said, and // dropped by a person, never silently missing — and cannot be delivered again. -func TestANoticeWhoseMessageIsGoneIsKeptAndSaysSo(t *testing.T) { - js := aBus(t) - d, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ - `"deliveries":5,"timestamp":"2026-10-06T15:51:00Z"}`)) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(d.Lost, "no longer held message 42") || d.Consumer != "media_sonarr" { - t.Fatalf("kept as %+v", d) - } - if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "Drop it") { - t.Fatalf("a record without its message was delivered again: %v", err) - } - // The same notice taken twice is kept once. - if _, err := KeepDeadLetter(js.Context(), []byte(`{"stream":"EVENTS","consumer":"media_sonarr","stream_seq":42,`+ - `"deliveries":5}`)); err != nil { - t.Fatal(err) - } - if _, total, _ := DeadLetters(js.Context(), "media_sonarr", 0); total != 1 { - t.Fatalf("one notice taken twice is kept %d times", total) - } -} // What only the giving-up consumer filters is its own: a subject delivered again reads back as the // event's, and a module's runtime reads the same key from it (the tokens around `.event.`). @@ -221,163 +108,15 @@ func TestASubjectDeliveredAgainSaysTheEventsOwn(t *testing.T) { // Only an event is delivered again, and only to a consumer that is on the bus and filters its again // subject: a dead letter is never let go as delivered while nobody receives it. -func TestADeadLetterIsDeliveredAgainOnlyWhereItIsReceived(t *testing.T) { - js := aBus(t) - keeping(t, js) - failing := aModuleConsumer(t, js, "media", "sonarr", "gitea.pull.merged") - if _, err := js.Context().Publish("mesh.mod.gitea.event.pull.merged", []byte(`{}`)); err != nil { - t.Fatal(err) - } - d := givenUp(t, js, failing) - - // A consumer as it was before this fix: its filters do not take what is delivered again. - c, _ := broker.ConsumerFor(broker.Principal{Kind: broker.KindModule, Node: "media", Module: "sonarr", - Consumes: []string{"gitea.pull.merged"}, PasswordHash: "x"}) - c.Filters = c.Filters[:len(c.Filters)-1] - if err := js.EnsureConsumer(c); err != nil { - t.Fatal(err) - } - if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "does not yet take") { - t.Fatalf("delivered to a consumer that does not filter its again subject: %v", err) - } - if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { - t.Fatalf("a refused delivery let the dead letter go: %v", err) - } - - // The consumer gone: nothing would receive it. - if err := js.Context().DeleteConsumer(broker.EventsStream, c.Name); err != nil { - t.Fatal(err) - } - if _, _, err := DeliverAgain(js.Context(), d.ID); err == nil || !strings.Contains(err.Error(), "no longer on the bus") { - t.Fatalf("delivered to a consumer that is gone: %v", err) - } - if _, err := DeadLetterNamed(js.Context(), d.ID); err != nil { - t.Fatalf("a refused delivery let the dead letter go: %v", err) - } - - // Not an event: refused, whatever stream it is from. - for _, other := range []DeadLetter{ - {ID: 2, Stream: "SEAT_TELEGRAM_SENDER", Consumer: "SEAT_TELEGRAM_SENDER_worker", Subject: "mesh.seat.telegram-sender.accept.send"}, - {ID: 3, Stream: "KV_x", Consumer: "y", Subject: "$KV.x.k"}, - } { - if _, err := AgainTo(js.Context(), other); err == nil { - t.Errorf("%s was given a subject to be delivered again on", other.Stream) - } - } -} // A total says how many are held, however few the list carries. -func TestTheListSaysTheTotalAndStopsAtItsLimit(t *testing.T) { - js := aBus(t) - for seq := 1; seq <= 5; seq++ { - if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":"media_sonarr",`+ - `"stream_seq":%d,"deliveries":5}`, seq))); err != nil { - t.Fatal(err) - } - } - list, total, err := DeadLetters(js.Context(), "media_sonarr", 2) - if err != nil || total != 5 || len(list) != 2 || list[0].Sequence != 5 { - t.Fatalf("%d of %d (%v): %+v", len(list), total, err, list) - } - if _, total, _ := DeadLetters(js.Context(), "another_one", 2); total != 0 { - t.Fatalf("another consumer's total is %d", total) - } -} // An event the controller itself gave up on, delivered again by a person, is acted on as the event it // was: it arrives under the controller's again subject, which its consumer filters. -func TestTheControllerActsOnAnEventDeliveredAgain(t *testing.T) { - js := aBus(t) - told := &toldAbout{} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(told); err != nil { - t.Fatal(err) - } - if err := s.Answers(replaysWith{}); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - eventually(t, "the controller's event consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil - }) - moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) - if _, err := js.Context().Publish(broker.AgainSubject(broker.ControllerName, broker.ControllerFollows[0]), moved); err != nil { - t.Fatal(err) - } - eventually(t, "the upgrade delivered again reaching the controller", func() bool { return told.count() == 1 }) -} // The notices cannot be taken (their stream is gone): said as a condition, and the controller serves on. -func TestNoticesThatCannotBeTakenAreSaidAndServingGoesOn(t *testing.T) { - js := aBus(t) - if err := js.Context().DeleteStream(broker.DeadLetterNoticesStream); err != nil { - t.Fatal(err) - } - stop := keepingGivenUp(context.Background(), js, quiet()) - defer stop() - eventually(t, "the failure said", func() bool { - for _, a := range Advisories.Since(time.Now().Add(-time.Minute)) { - if a.Token == AdvisoryNotKept && a.Stream == broker.DeadLetterNoticesStream { - return true - } - } - return false - }) - held := &counted{} - _, stopServing := servingOn(t, js, held) - defer stopServing() - if _, err := js.Context().Publish("mesh.control.anchor.report", []byte(`{"node":"anchor"}`)); err != nil { - t.Fatal(err) - } - eventually(t, "a report heard while the notices cannot be taken", func() bool { return held.count() == 1 }) -} // When DEAD_LETTERS kept each consumer's newest message, for the condition that counts what was given up // on rather than how often it was looked at (novox/hq issue 440). It rises with every message kept, even // one the consumer gave up on before the last: a notice that could not be kept is offered again a minute // later, so a later give-up can be kept first, and the one kept after it must still count. -func TestTheNewestHeldDeadLetterIsWhenItWasKept(t *testing.T) { - js := aBus(t) - gaveUp := time.Date(2026, 10, 10, 21, 4, 0, 0, time.UTC) - keep := func(consumer string, seq int, at time.Time) { - t.Helper() - if _, err := KeepDeadLetter(js.Context(), []byte(fmt.Sprintf(`{"stream":"EVENTS","consumer":%q,`+ - `"stream_seq":%d,"deliveries":5,"timestamp":%q}`, consumer, seq, at.Format(time.RFC3339Nano)))); err != nil { - t.Fatal(err) - } - } - look := func() map[string]time.Time { - t.Helper() - held, err := HeldDeadLetters(js.Context()) - if err != nil { - t.Fatal(err) - } - newest, err := NewestDeadLetters(js.Context(), held) - if err != nil { - t.Fatal(err) - } - return newest - } - - // Sonarr gave up at 21:05 and that is kept first; radarr's letter is kept after it. - keep("media_sonarr", 2, gaveUp.Add(time.Minute)) - keep("media_radarr", 3, gaveUp) - before := look() - if len(before) != 2 || before["EVENTS.media_sonarr"].IsZero() || - !before["EVENTS.media_radarr"].After(before["EVENTS.media_sonarr"]) { - t.Fatalf("%v", before) - } - // Sonarr's give-up of 21:04, kept late, is newer than anything the condition has seen. - keep("media_sonarr", 1, gaveUp) - after := look() - if !after["EVENTS.media_sonarr"].After(before["EVENTS.media_sonarr"]) || - !after["EVENTS.media_sonarr"].After(before["EVENTS.media_radarr"]) { - t.Fatalf("a letter kept late does not read as newer: before %v, after %v", before, after) - } - if !after["EVENTS.media_radarr"].Equal(before["EVENTS.media_radarr"]) { - t.Fatalf("radarr's newest moved: before %v, after %v", before, after) - } -} diff --git a/internal/link/enrol_bus_credential_test.go b/internal/link/enrol_bus_credential_test.go index 9a35404e..6282d603 100644 --- a/internal/link/enrol_bus_credential_test.go +++ b/internal/link/enrol_bus_credential_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/enrol_claim_test.go b/internal/link/enrol_claim_test.go index 0d4eafd7..a48aadb8 100644 --- a/internal/link/enrol_claim_test.go +++ b/internal/link/enrol_claim_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/enrol_reply_probe_test.go b/internal/link/enrol_reply_probe_test.go index d9acfb4b..30597a73 100644 --- a/internal/link/enrol_reply_probe_test.go +++ b/internal/link/enrol_reply_probe_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/enrol_shape_test.go b/internal/link/enrol_shape_test.go index 54a1b54c..68cd60c4 100644 --- a/internal/link/enrol_shape_test.go +++ b/internal/link/enrol_shape_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/enrol_tunnel_key_test.go b/internal/link/enrol_tunnel_key_test.go index f5604f42..105fc356 100644 --- a/internal/link/enrol_tunnel_key_test.go +++ b/internal/link/enrol_tunnel_key_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/events.go b/internal/link/events.go index e545802d..84b41651 100644 --- a/internal/link/events.go +++ b/internal/link/events.go @@ -224,9 +224,8 @@ type SourceMoved struct { // stands: a directory is a module only when the merge changed its manifest. ModuleDirsSaid bool `json:"module_dirs_said,omitempty"` - // Number is the pull request's, Title its title and Body its description (novox/hq ADR 0276): a delivery - // group's `after:` lines, read when its members are merged into one batch and their order becomes the - // walk's, and the words `plans` names a walk by. + // Number is the pull request's, Title its title and Body its description (novox/hq ADR 0276): the + // pull request's number and words, which `plans` names a walk by (no `after:` line is read, ADR 0419). Number int `json:"number,omitempty"` Title string `json:"title,omitempty"` Body string `json:"body,omitempty"` @@ -295,8 +294,9 @@ type Checked struct { // and send, posted with the verdict. Plan *ChangePlan `json:"plan,omitempty"` // Group is set on a delivery group's composed check (novox/hq ADR 0239): every member's head composed - // together as one future state. Members are the heads judged. A forge's holder sets no merge-gate - // status from it — the group's verdict is its owner's, mesh-delivery, to say on each member's head. + // together as one future state. Members are the heads judged, each with the trunk commit it was merged into. **The + // forge's holder sets every member's mesh/merge-gate from it** (novox/hq ADR 0403 clauses 1 and 5): the group's + // verdict is each member's gate, and a member's own verdict, whose gate layer is "composed", sets none. Group string `json:"group,omitempty"` Members []CheckedMember `json:"members,omitempty"` // Trunk is the trunk commit the head was merged into and judged as, twelve characters (novox/hq ADR 0323 @@ -307,13 +307,7 @@ type Checked struct { Trunk string `json:"trunk,omitempty"` Beside map[string]string `json:"beside,omitempty"` Took string `json:"took,omitempty"` - // Release is the release branch the check judged for (novox/hq ADR 0358 §4), JudgedBy the commit of the - // controller that judged its gate (twelve characters), and BesideAt the branch each repository beside it was - // read at, by directory: so a reader sees a release-branch judgement apart from a trunk one. Empty otherwise. - Release string `json:"release,omitempty"` - JudgedBy string `json:"judged-by,omitempty"` - BesideAt map[string]string `json:"beside-at,omitempty"` - // Validator says which node-engine validator the judge of a release-branch check was built with (ADR 0358 §2). + // Validator says which node-engine validator the group's judge was built with (ADR 0382 §3). Validator string `json:"validator,omitempty"` // Siblings are the other pull requests of the delivery group the check read beside it (novox/hq ADR 0382 §4), each // with its head and the base commit it was merged onto; SiblingsUnknown is why it could not learn them, set only diff --git a/internal/link/handacts_test.go b/internal/link/handacts_test.go index e670fe7c..c1f864c7 100644 --- a/internal/link/handacts_test.go +++ b/internal/link/handacts_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/handover_integration_test.go b/internal/link/handover_integration_test.go new file mode 100644 index 00000000..ec53ac50 --- /dev/null +++ b/internal/link/handover_integration_test.go @@ -0,0 +1,75 @@ +//go:build integration + +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) { + url := testbus.URL(t) + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + + signer, public := testSigner(t) + var heard HandOverAsk + engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) { + var signed SignedHandOver + _ = json.Unmarshal(msg.Data, &signed) + if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) { + _ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`)) + return + } + _ = json.Unmarshal(signed.Ask, &heard) + switch heard.Path { + case "/srv/notes": + body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By}) + _ = msg.Respond(body) + case "/srv/empty": + _ = msg.Respond([]byte(`{}`)) + default: + body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"}) + _ = msg.Respond(body) + } + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Unsubscribe() }) + + ctx := context.Background() + a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second) + if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") { + t.Fatalf("answered %+v, %v", a, err) + } + if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" { + t.Fatalf("the engine heard %+v", heard) + } + a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second) + if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") { + t.Fatalf("a refusal came back as %+v, %v", a, err) + } + if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil || + !strings.Contains(err.Error(), "neither") { + t.Fatalf("an empty answer was taken: %v", err) + } + if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil || + !strings.Contains(err.Error(), "node-engine") { + t.Fatalf("a machine with no engine listening: %v", err) + } + if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil { + t.Fatal("asked with no bus") + } +} diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index bf97a68f..188af5e9 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -4,14 +4,10 @@ import ( "context" "crypto/ed25519" "encoding/json" - "strings" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // The hand-over's ask and answer hold these field names; the engine's side holds the same list (mesh-host @@ -99,64 +95,6 @@ func TestAHandOverIsSignedWithAContextANonceAndAnExpiry(t *testing.T) { // The ask reaches the machine's engine on its own subject and its answer comes back whole: what it recorded, or // its refusal as the engine worded it. A machine with no engine listening is said as not answering, and an // answer that is neither is refused rather than read as a record. -func TestAHandOverIsAskedOfTheMachineAndItsAnswerComesBack(t *testing.T) { - url := testbus.URL(t) - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - - signer, public := testSigner(t) - var heard HandOverAsk - engine, err := conn.Subscribe(broker.AskHandOverSubject("laptop"), func(msg *nats.Msg) { - var signed SignedHandOver - _ = json.Unmarshal(msg.Data, &signed) - if !ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) { - _ = msg.Respond([]byte(`{"refused":"not the mesh's signature"}`)) - return - } - _ = json.Unmarshal(signed.Ask, &heard) - switch heard.Path { - case "/srv/notes": - body, _ := json.Marshal(HandOverAnswer{Said: "/srv/notes (notes.data) is handed to the mesh by " + heard.By}) - _ = msg.Respond(body) - case "/srv/empty": - _ = msg.Respond([]byte(`{}`)) - default: - body, _ := json.Marshal(HandOverAnswer{Refused: heard.Path + " is not a directory this machine uses as found; nothing was handed over"}) - _ = msg.Respond(body) - } - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = engine.Unsubscribe() }) - - ctx := context.Background() - a, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/notes", "jo through mesh-cli on anchor", 5*time.Second) - if err != nil || a.Refused != "" || !strings.Contains(a.Said, "handed to the mesh by jo through mesh-cli on anchor") { - t.Fatalf("answered %+v, %v", a, err) - } - if heard.Node != "laptop" || heard.Path != "/srv/notes" || heard.By != "jo through mesh-cli on anchor" { - t.Fatalf("the engine heard %+v", heard) - } - a, err = AskHandOver(ctx, conn, signer, "laptop", "/srv/other", "jo", 5*time.Second) - if err != nil || a.Said != "" || !strings.Contains(a.Refused, "nothing was handed over") { - t.Fatalf("a refusal came back as %+v, %v", a, err) - } - if _, err := AskHandOver(ctx, conn, signer, "laptop", "/srv/empty", "jo", 5*time.Second); err == nil || - !strings.Contains(err.Error(), "neither") { - t.Fatalf("an empty answer was taken: %v", err) - } - if _, err := AskHandOver(ctx, conn, signer, "anchor", "/srv/notes", "jo", 5*time.Second); err == nil || - !strings.Contains(err.Error(), "node-engine") { - t.Fatalf("a machine with no engine listening: %v", err) - } - if _, err := AskHandOver(ctx, nil, signer, "anchor", "/srv/notes", "jo", time.Second); err == nil { - t.Fatal("asked with no bus") - } -} // A machine's grant hears its own hand-over ask and nobody else's, and may answer it: the one request a node is // asked (novox/hq issue 356). diff --git a/internal/link/heard_order_integration_test.go b/internal/link/heard_order_integration_test.go new file mode 100644 index 00000000..5941b196 --- /dev/null +++ b/internal/link/heard_order_integration_test.go @@ -0,0 +1,118 @@ +//go:build integration + +package link_test + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/inventory" + "github.com/novox/mesh-controller/internal/link" +) + +func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) { + t.Helper() + inv := inventory.ForTest(t) + node, err := inv.AddNode(context.Background(), "home-server") + if err != nil { + t.Fatal(err) + } + return inv, link.Enrolment{Inventory: inv}, node.ID +} + +func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) { + inv, heard, id := anOrderedMachine(t) + ctx := context.Background() + before := countFor(link.WriterNodeEngine("home-server")) + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { + t.Fatal(err) + } + if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil { + t.Fatal(err) + } + doing, _, err := inv.DoingOf(ctx, "home-server") + if err != nil || doing.Declared != "d12" || doing.Applied != 2 { + t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err) + } + if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 { + t.Fatalf("the refusal was counted %d times, want once", got) + } + kept, err := inv.KeptOrder(ctx, id) + if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) { + t.Fatalf("the order kept is %+v (%v)", kept, err) + } +} + +func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) { + inv, heard, _ := anOrderedMachine(t) + ctx := context.Background() + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { + t.Fatal(err) + } + failed := ordered("d12", 57, 12, 40) + failed.Failed = map[string]string{"b": "it failed a moment before it applied"} + if _, err := heard.Heard(ctx, failed); err != nil { + t.Fatal(err) + } + doing, _, _ := inv.DoingOf(ctx, "home-server") + if doing.Outcome != inventory.OutcomeApplied { + t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing) + } + // A newer report of it — the next reconcile — is kept. + again := ordered("d12", 57, 12, 42) + again.Failed = map[string]string{"b": "it failed since"} + if _, err := heard.Heard(ctx, again); err != nil { + t.Fatal(err) + } + if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed { + t.Fatalf("a newer report of the same declaration was not kept: %+v", doing) + } +} + +func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) { + inv, heard, id := anOrderedMachine(t) + ctx := context.Background() + if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { + t.Fatal(err) + } + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { + t.Fatal(err) + } + if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" { + t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing) + } + // And the machine reads an epoch: it orders its reports. + if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads { + t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err) + } +} + +func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { + inv, heard, id := anOrderedMachine(t) + ctx := context.Background() + if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { + t.Fatal(err) + } + if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { + t.Fatal(err) + } + if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil { + t.Fatal(err) + } + if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" { + t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing) + } + if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) { + t.Fatalf("an unordered account left the order kept: %+v", kept) + } + if reads, _ := inv.ReadsEpoch(ctx, id); reads { + t.Fatal("a node-engine that orders nothing is still sent an epoch") + } + // The next ordered account, whatever its numbers, has nothing to be older than. + if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil { + t.Fatal(err) + } + if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" { + t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing) + } +} diff --git a/internal/link/heard_order_test.go b/internal/link/heard_order_test.go index b00ab0c8..de59da59 100644 --- a/internal/link/heard_order_test.go +++ b/internal/link/heard_order_test.go @@ -1,11 +1,9 @@ package link_test import ( - "context" "testing" "time" - "github.com/novox/mesh-controller/internal/inventory" "github.com/novox/mesh-controller/internal/link" ) @@ -14,122 +12,18 @@ import ( // Of the same declaration, report r then r−1: refused. The digest the mesh last sent decides nothing // for an ordered report; an unordered one, from an older node-engine, is judged by it as before. -func anOrderedMachine(t *testing.T) (*inventory.Inventory, link.Enrolment, string) { - t.Helper() - inv := inventory.ForTest(t) - node, err := inv.AddNode(context.Background(), "home-server") - if err != nil { - t.Fatal(err) - } - return inv, link.Enrolment{Inventory: inv}, node.ID -} - func ordered(declared string, epoch, sequence, report int64, applied ...string) link.Report { return link.Report{Node: "home-server", Declared: declared, Applied: applied, Order: link.Order{Epoch: epoch, Sequence: sequence}, ReportSequence: report} } // Issue 267, ordered: a reconcile's account of declaration 11 reaches the mesh after the apply's of 12. -func TestAnAccountOfAnOlderDeclarationIsRefusedByItsSequence(t *testing.T) { - inv, heard, id := anOrderedMachine(t) - ctx := context.Background() - before := countFor(link.WriterNodeEngine("home-server")) - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, ordered("d11", 57, 11, 40, "a")); err != nil { - t.Fatal(err) - } - doing, _, err := inv.DoingOf(ctx, "home-server") - if err != nil || doing.Declared != "d12" || doing.Applied != 2 { - t.Fatalf("the older account replaced the newer: %+v (%v)", doing, err) - } - if got := countFor(link.WriterNodeEngine("home-server")) - before; got != 1 { - t.Fatalf("the refusal was counted %d times, want once", got) - } - kept, err := inv.KeptOrder(ctx, id) - if err != nil || kept != (inventory.ReportOrder{Epoch: 57, Sequence: 12, ReportSequence: 41}) { - t.Fatalf("the order kept is %+v (%v)", kept, err) - } -} - -func TestAnOlderReportOfTheSameDeclarationIsRefused(t *testing.T) { - inv, heard, _ := anOrderedMachine(t) - ctx := context.Background() - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a", "b")); err != nil { - t.Fatal(err) - } - failed := ordered("d12", 57, 12, 40) - failed.Failed = map[string]string{"b": "it failed a moment before it applied"} - if _, err := heard.Heard(ctx, failed); err != nil { - t.Fatal(err) - } - doing, _, _ := inv.DoingOf(ctx, "home-server") - if doing.Outcome != inventory.OutcomeApplied { - t.Fatalf("an older report of the same declaration replaced the newer: %+v", doing) - } - // A newer report of it — the next reconcile — is kept. - again := ordered("d12", 57, 12, 42) - again.Failed = map[string]string{"b": "it failed since"} - if _, err := heard.Heard(ctx, again); err != nil { - t.Fatal(err) - } - if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Outcome != inventory.OutcomeFailed { - t.Fatalf("a newer report of the same declaration was not kept: %+v", doing) - } -} // An ordered account is judged by its order, not by the digest the mesh last sent: the account of 12 // is kept although the mesh has sent 13 since — it is still the newest account of the machine. -func TestAnOrderedAccountIsNotJudgedByTheDigestSent(t *testing.T) { - inv, heard, id := anOrderedMachine(t) - ctx := context.Background() - if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { - t.Fatal(err) - } - if doing, said, _ := inv.DoingOf(ctx, "home-server"); !said || doing.Declared != "d12" { - t.Fatalf("the newest account the machine gave was set aside by the digest: %+v", doing) - } - // And the machine reads an epoch: it orders its reports. - if reads, err := inv.ReadsEpoch(ctx, id); err != nil || !reads { - t.Fatalf("a machine whose reports carry a report sequence is not recorded as reading an epoch: %v", err) - } -} // A node-engine rolled back to one that orders nothing: its account is taken by the digest rule, the // order kept is cleared, and it is no longer sent an epoch. -func TestAnUnorderedAccountClearsTheOrderAndTheEpoch(t *testing.T) { - inv, heard, id := anOrderedMachine(t) - ctx := context.Background() - if _, err := heard.Heard(ctx, ordered("d12", 57, 12, 41, "a")); err != nil { - t.Fatal(err) - } - if err := inv.RecordSent(ctx, id, "d13", nil); err != nil { - t.Fatal(err) - } - if _, err := heard.Heard(ctx, link.Report{Node: "home-server", Declared: "d13", Applied: []string{"a", "b"}}); err != nil { - t.Fatal(err) - } - if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d13" { - t.Fatalf("an older node-engine's account of what was sent was not kept: %+v", doing) - } - if kept, _ := inv.KeptOrder(ctx, id); kept != (inventory.ReportOrder{}) { - t.Fatalf("an unordered account left the order kept: %+v", kept) - } - if reads, _ := inv.ReadsEpoch(ctx, id); reads { - t.Fatal("a node-engine that orders nothing is still sent an epoch") - } - // The next ordered account, whatever its numbers, has nothing to be older than. - if _, err := heard.Heard(ctx, ordered("d14", 58, 14, 1, "a")); err != nil { - t.Fatal(err) - } - if doing, _, _ := inv.DoingOf(ctx, "home-server"); doing.Declared != "d14" { - t.Fatalf("the first ordered account after an unordered one was refused: %+v", doing) - } -} // A stale refusal names its writer by the epoch the refused declaration claimed, and a refusal whose // own report was lost is still counted from the machine's own tally. diff --git a/internal/link/heard_test.go b/internal/link/heard_test.go index 22d97779..93a2dd0d 100644 --- a/internal/link/heard_test.go +++ b/internal/link/heard_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/judge_directories_test.go b/internal/link/judge_directories_test.go index 10cfdde1..315869b3 100644 --- a/internal/link/judge_directories_test.go +++ b/internal/link/judge_directories_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/meshcli_integration_test.go b/internal/link/meshcli_integration_test.go new file mode 100644 index 00000000..3da2b222 --- /dev/null +++ b/internal/link/meshcli_integration_test.go @@ -0,0 +1,96 @@ +//go:build integration + +package link + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn { + t.Helper() + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(stop) + return conn +} + +func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) { + was := AnswerWithin + AnswerWithin = 50 * time.Millisecond + t.Cleanup(func() { AnswerWithin = was }) + release := make(chan struct{}) + conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer { + <-release + return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true} + }) + _, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000}) + call, _ := first["call"].(string) + if failed != "" || first["running"] != true || call == "" { + t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed) + } + _, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) + if still["running"] != true { + t.Fatalf("following a running line answered %v", still) + } + close(release) + deadline := time.Now().Add(5 * time.Second) + for { + a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) + if failed != "" { + t.Fatalf("following answered %q", failed) + } + if raw["running"] != true { + if string(a.Stdout) != "done on laptop" || !a.Terminal { + t.Fatalf("followed to %+v", a) + } + break + } + if time.Now().After(deadline) { + t.Fatal("the line never finished for its follower") + } + time.Sleep(20 * time.Millisecond) + } + if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" { + t.Fatal("another account followed the operator's line") + } + if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" { + t.Fatal("another node followed the line") + } +} + +func TestMeshCLILinesAtOnceAreBounded(t *testing.T) { + was := AnswerWithin + AnswerWithin = 50 * time.Millisecond + t.Cleanup(func() { AnswerWithin = was }) + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + ran := make(chan struct{}, 4) + conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer { + ran <- struct{}{} + <-release + return CLIAnswer{} + }) + if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true { + t.Fatalf("the first line answered %v", first) + } + a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}) + if !strings.Contains(a.Refused, "busy") || a.Exit == 0 { + t.Fatalf("a line over the bound answered %+v", a) + } + if len(ran) != 1 { + t.Fatalf("%d lines ran, one was bound", len(ran)) + } +} diff --git a/internal/link/meshcli_test.go b/internal/link/meshcli_test.go index 1fc50085..7e5db365 100644 --- a/internal/link/meshcli_test.go +++ b/internal/link/meshcli_test.go @@ -10,8 +10,6 @@ import ( "time" "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // The node-engine's request and the answer it hands mesh-cli hold these field names; the engine's side holds the @@ -94,20 +92,6 @@ func TestALargeMeshCLIAnswerIsCutAndSaysSo(t *testing.T) { } // cliBus serves mesh-cli on a bus of the test's own with a call log of its own, and returns a connection to ask on. -func cliBus(t *testing.T, l *CallLog, atOnce int, handle CLIHandler) *nats.Conn { - t.Helper() - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - stop, err := OverNATS{Conn: conn}.serveCLI(l, atOnce, handle, nil) - if err != nil { - t.Fatal(err) - } - t.Cleanup(stop) - return conn -} // askCLI asks one request on a node's subject and reads the envelope. func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnswer, map[string]any, string) { @@ -134,74 +118,8 @@ func askCLI(t *testing.T, conn *nats.Conn, node string, asked CLIAsked) (CLIAnsw // **A line that outlasts the bus's window for an answer is followed to its answer, never lost** (review of ADR // 0272): the first answer says it is running and names its call, and following the call by the same account on // the same node gives what the line printed once it ends. Another account, or another node, is told no such call. -func TestALongMeshCLILineIsFollowedToItsAnswer(t *testing.T) { - was := AnswerWithin - AnswerWithin = 50 * time.Millisecond - t.Cleanup(func() { AnswerWithin = was }) - release := make(chan struct{}) - conn := cliBus(t, NewCallLog(), 4, func(ctx context.Context, node string, asked CLIAsked) CLIAnswer { - <-release - return CLIAnswer{Stdout: []byte("done on " + node), Terminal: true} - }) - _, first, failed := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000}) - call, _ := first["call"].(string) - if failed != "" || first["running"] != true || call == "" { - t.Fatalf("a long line was not answered as running with its call: %v %q", first, failed) - } - _, still, _ := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) - if still["running"] != true { - t.Fatalf("following a running line answered %v", still) - } - close(release) - deadline := time.Now().Add(5 * time.Second) - for { - a, raw, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "op", UID: 1000}) - if failed != "" { - t.Fatalf("following answered %q", failed) - } - if raw["running"] != true { - if string(a.Stdout) != "done on laptop" || !a.Terminal { - t.Fatalf("followed to %+v", a) - } - break - } - if time.Now().After(deadline) { - t.Fatal("the line never finished for its follower") - } - time.Sleep(20 * time.Millisecond) - } - if _, _, failed := askCLI(t, conn, "laptop", CLIAsked{Follow: call, Account: "agent", UID: 1001}); failed == "" { - t.Fatal("another account followed the operator's line") - } - if _, _, failed := askCLI(t, conn, "desktop", CLIAsked{Follow: call, Account: "op", UID: 1000}); failed == "" { - t.Fatal("another node followed the line") - } -} // Lines running at once are bounded: one over the bound is answered busy at once, and nothing ran. -func TestMeshCLILinesAtOnceAreBounded(t *testing.T) { - was := AnswerWithin - AnswerWithin = 50 * time.Millisecond - t.Cleanup(func() { AnswerWithin = was }) - release := make(chan struct{}) - t.Cleanup(func() { close(release) }) - ran := make(chan struct{}, 4) - conn := cliBus(t, NewCallLog(), 1, func(context.Context, string, CLIAsked) CLIAnswer { - ran <- struct{}{} - <-release - return CLIAnswer{} - }) - if _, first, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}); first["running"] != true { - t.Fatalf("the first line answered %v", first) - } - a, _, _ := askCLI(t, conn, "laptop", CLIAsked{Line: []string{"status"}, Account: "op"}) - if !strings.Contains(a.Refused, "busy") || a.Exit == 0 { - t.Fatalf("a line over the bound answered %+v", a) - } - if len(ran) != 1 { - t.Fatalf("%d lines ran, one was bound", len(ran)) - } -} // A mesh-cli line's record keeps its first word, never the rest of its line, and its answer is never kept on the // bus, where `calls` answers anyone who may call the seat. diff --git a/internal/link/movesecret_integration_test.go b/internal/link/movesecret_integration_test.go new file mode 100644 index 00000000..848b5b10 --- /dev/null +++ b/internal/link/movesecret_integration_test.go @@ -0,0 +1,58 @@ +//go:build integration + +package link + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAMoveIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(conn.Close) + signer, public := testSigner(t) + var heard HandOverAsk + var row MoveSecretAsk + engine, err := conn.Subscribe(broker.AskMoveSecretSubject("laptop"), func(msg *nats.Msg) { + var signed SignedHandOver + _ = json.Unmarshal(msg.Data, &signed) + if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) || + !ed25519.Verify(public, append([]byte(MoveSecretContext), signed.Ask...), signed.Signature) { + _ = msg.Respond([]byte(`{"refused":"not signed as a move"}`)) + return + } + _ = json.Unmarshal(signed.Ask, &heard) + _ = json.Unmarshal(signed.Ask, &row) + body, _ := json.Marshal(MoveSecretAnswer{Said: "moved", Secret: "own/laptop/" + row.Module + "/" + row.Name, + Generation: 1}) + _ = msg.Respond(body) + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Unsubscribe() }) + a, err := AskMoveSecret(context.Background(), conn, signer, "laptop", "the move", + MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}, 5*time.Second) + if err != nil || a.Secret != "own/laptop/postgres/superuser-password" || a.Generation != 1 { + t.Fatalf("answered %+v, %v", a, err) + } + if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() || heard.By != "the move" || + row.Sealed != "c2Vh" || row.Origin != "made" { + t.Fatalf("the engine heard %+v %+v", heard, row) + } + if _, err := AskMoveSecret(context.Background(), conn, signer, "anchor", "the move", MoveSecretAsk{}, 2*time.Second); err == nil || + !strings.Contains(err.Error(), "nothing was moved") { + t.Fatalf("a machine with no engine listening: %v", err) + } +} diff --git a/internal/link/movesecret_test.go b/internal/link/movesecret_test.go index 4dc05905..cfad665f 100644 --- a/internal/link/movesecret_test.go +++ b/internal/link/movesecret_test.go @@ -1,18 +1,13 @@ package link import ( - "context" - "crypto/ed25519" "encoding/json" "sort" "strings" "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - "github.com/novox/mesh-controller/internal/testbus" ) // The move's ask and answer keep the engine's field names (mesh-host internal/link, @@ -47,44 +42,3 @@ func TestTheMovesAskAndAnswerKeepTheirFieldNames(t *testing.T) { // A move is asked of the machine on its own subject, signed under its own context — never a hand-over's — naming // no path and carrying the row beside the hand-over's fields; the engine's answer comes back whole. -func TestAMoveIsAskedOfTheMachineSignedUnderItsOwnContext(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(conn.Close) - signer, public := testSigner(t) - var heard HandOverAsk - var row MoveSecretAsk - engine, err := conn.Subscribe(broker.AskMoveSecretSubject("laptop"), func(msg *nats.Msg) { - var signed SignedHandOver - _ = json.Unmarshal(msg.Data, &signed) - if ed25519.Verify(public, append([]byte(HandOverContext), signed.Ask...), signed.Signature) || - !ed25519.Verify(public, append([]byte(MoveSecretContext), signed.Ask...), signed.Signature) { - _ = msg.Respond([]byte(`{"refused":"not signed as a move"}`)) - return - } - _ = json.Unmarshal(signed.Ask, &heard) - _ = json.Unmarshal(signed.Ask, &row) - body, _ := json.Marshal(MoveSecretAnswer{Said: "moved", Secret: "own/laptop/" + row.Module + "/" + row.Name, - Generation: 1}) - _ = msg.Respond(body) - }) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = engine.Unsubscribe() }) - a, err := AskMoveSecret(context.Background(), conn, signer, "laptop", "the move", - MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}, 5*time.Second) - if err != nil || a.Secret != "own/laptop/postgres/superuser-password" || a.Generation != 1 { - t.Fatalf("answered %+v, %v", a, err) - } - if heard.Node != "laptop" || heard.Path != "" || heard.Nonce == "" || heard.Expires.IsZero() || heard.By != "the move" || - row.Sealed != "c2Vh" || row.Origin != "made" { - t.Fatalf("the engine heard %+v %+v", heard, row) - } - if _, err := AskMoveSecret(context.Background(), conn, signer, "anchor", "the move", MoveSecretAsk{}, 2*time.Second); err == nil || - !strings.Contains(err.Error(), "nothing was moved") { - t.Fatalf("a machine with no engine listening: %v", err) - } -} diff --git a/internal/link/pages_integration_test.go b/internal/link/pages_integration_test.go new file mode 100644 index 00000000..5467a161 --- /dev/null +++ b/internal/link/pages_integration_test.go @@ -0,0 +1,59 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/testbus" +) + +func TestAPartGoesOnlyToItsCaller(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + l := NewCallLog() + c := l.begin("mesh-controller", "status", nil, "_INBOX.operator.ABCDEFGHIJKLMNOPQRSTUV") + l.page(c, []byte(strings.Repeat("z", 5000)), 2048) + sub, err := conn.Subscribe("pages.test", func(m *nats.Msg) { l.servePage("mesh-controller", "status", m, nil) }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: c.ID, Parts: 3}, 0) + if err == nil || !strings.Contains(err.Error(), "another caller") { + t.Fatalf("a part was handed to a caller it was not paged to: %v", err) + } + _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: "call-0-0", Parts: 3}, 0) + if err == nil || !strings.Contains(err.Error(), "not held here") { + t.Fatalf("a part of nothing held: %v", err) + } +} + +func TestAMissingPartFailsTheWhole(t *testing.T) { + conn, err := nats.Connect(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + sub, err := conn.Subscribe("pages.none", func(m *nats.Msg) { _ = m.Respond([]byte(`{"error":"gone"}`)) }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() + first, _ := json.Marshal(map[string]any{"error": "too large", "paged": Paged{Call: "call-1-1", Bytes: 10, Parts: 2}}) + _, err = Whole(context.Background(), conn, "pages.none", first) + if err == nil || !strings.Contains(err.Error(), "part 0") || !strings.Contains(err.Error(), "gone") { + t.Fatalf("a missing part read as %v", err) + } + if plain, err := Whole(context.Background(), conn, "pages.none", []byte(`{"result":1}`)); err != nil || + string(plain) != `{"result":1}` { + t.Fatalf("an answer that fits was changed: %s %v", plain, err) + } +} diff --git a/internal/link/pages_test.go b/internal/link/pages_test.go index 844fb099..08f28e03 100644 --- a/internal/link/pages_test.go +++ b/internal/link/pages_test.go @@ -7,8 +7,6 @@ import ( "testing" "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/testbus" ) // An answer too large for one message is recorded as paged, and a call's record says so: `calls` no @@ -53,49 +51,5 @@ func TestAnAnswerTheBusRefusesIsSaid(t *testing.T) { } // A part goes only to the caller its answer was paged to, on the verb that answered it. -func TestAPartGoesOnlyToItsCaller(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - l := NewCallLog() - c := l.begin("mesh-controller", "status", nil, "_INBOX.operator.ABCDEFGHIJKLMNOPQRSTUV") - l.page(c, []byte(strings.Repeat("z", 5000)), 2048) - sub, err := conn.Subscribe("pages.test", func(m *nats.Msg) { l.servePage("mesh-controller", "status", m, nil) }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() - _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: c.ID, Parts: 3}, 0) - if err == nil || !strings.Contains(err.Error(), "another caller") { - t.Fatalf("a part was handed to a caller it was not paged to: %v", err) - } - _, err = askPart(context.Background(), conn, "pages.test", Paged{Call: "call-0-0", Parts: 3}, 0) - if err == nil || !strings.Contains(err.Error(), "not held here") { - t.Fatalf("a part of nothing held: %v", err) - } -} // A part that cannot be had fails the whole answer, naming it — never a shorter answer. -func TestAMissingPartFailsTheWhole(t *testing.T) { - conn, err := nats.Connect(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - sub, err := conn.Subscribe("pages.none", func(m *nats.Msg) { _ = m.Respond([]byte(`{"error":"gone"}`)) }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() - first, _ := json.Marshal(map[string]any{"error": "too large", "paged": Paged{Call: "call-1-1", Bytes: 10, Parts: 2}}) - _, err = Whole(context.Background(), conn, "pages.none", first) - if err == nil || !strings.Contains(err.Error(), "part 0") || !strings.Contains(err.Error(), "gone") { - t.Fatalf("a missing part read as %v", err) - } - if plain, err := Whole(context.Background(), conn, "pages.none", []byte(`{"result":1}`)); err != nil || - string(plain) != `{"result":1}` { - t.Fatalf("an answer that fits was changed: %s %v", plain, err) - } -} diff --git a/internal/link/profile_report_test.go b/internal/link/profile_report_test.go index 5477f1f2..889fae48 100644 --- a/internal/link/profile_report_test.go +++ b/internal/link/profile_report_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/queue_integration_test.go b/internal/link/queue_integration_test.go new file mode 100644 index 00000000..d4684406 --- /dev/null +++ b/internal/link/queue_integration_test.go @@ -0,0 +1,220 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "sync" + "testing" + "time" + + "github.com/novox/mesh-controller/internal/broker" +) + +func aBusWithACancelledSet(t *testing.T) *broker.JetStream { + t.Helper() + js := aBusWithTheBuildRole(t) + seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}} + if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) }) + return js +} + +func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + + if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil { + t.Fatal(err) + } + outcomes, err := js.Conn().SubscribeSync(BuildOutcome()) + if err != nil { + t.Fatal(err) + } + defer func() { _ = outcomes.Unsubscribe() }() + _ = js.Conn().Flush() + for _, id := range []string{"build-cancelled", "build-kept"} { + body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + } + + built := make(chan string, 2) + machine := MachineOverNATS(js, "anchor") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + built <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + select { + case id := <-built: + if id != "build-kept" { + t.Fatalf("%s was built", id) + } + case <-time.After(10 * time.Second): + t.Fatal("the ask that was not cancelled was never built") + } + msg, err := outcomes.NextMsg(5 * time.Second) + if err != nil { + t.Fatal(err) + } + var said BuildResult + if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" || + said.Failed != CancelledByHand || said.On != "anchor" { + t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err) + } + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatal("the cancelled ask is still queued: terminated, it would not be") +} + +func TestNatsAPausedHolderTakesNothingNew(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + paused := make(chan bool, 1) + paused <- true + isPaused := func() bool { + p := <-paused + paused <- p + return p + } + took := make(chan string, 1) + machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused}) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + took <- work.Request().ID + _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) + _ = work.Done() + }) + }() + body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("a paused holder took %s", id) + case <-time.After(3 * time.Second): + } + q, err := ReadQueue(ctx, js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting { + t.Fatalf("while paused the queue reads %+v", q.Asks) + } + <-paused + paused <- false + select { + case id := <-took: + if id != "build-waits" { + t.Fatalf("took %s", id) + } + case <-time.After(10 * time.Second): + t.Fatal("resumed, the holder never took what waited") + } +} + +func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) { + js := aBusWithTheBuildRole(t) + for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} { + body, _ := json.Marshal(s) + if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil { + t.Fatal(err) + } + } + said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"}) + if err != nil { + t.Fatal(err) + } + if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 { + t.Fatalf("read back %+v", said) + } +} + +func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) { + js := aBusWithACancelledSet(t) + was := cancelLook + cancelLook = 200 * time.Millisecond + t.Cleanup(func() { cancelLook = was }) + ask := &natsBuilds{js: js, seat: TheBuildMachine} + + go func() { + time.Sleep(500 * time.Millisecond) + _ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled") + }() + result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second) + if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" { + t.Fatalf("the waiter heard %+v (%v)", result, err) + } + + // Killed: the holder announces the failure as any outcome, and the waiter has it. + ctx, stop := context.WithCancel(context.Background()) + defer stop() + machine := MachineOverNATS(js, "ace") + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { + r := work.Request() + _ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand}) + _ = work.Done() + }) + }() + result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second) + if err != nil || result.Failed != KilledByHand || result.On != "ace" { + t.Fatalf("the waiter heard %+v (%v)", result, err) + } +} + +func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) { + js := aBusWithACancelledSet(t) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + // Not paused when it asks; paused by the time the answer is read. + var looks int + var mu sync.Mutex + paused := func() bool { + mu.Lock() + defer mu.Unlock() + looks++ + return looks > 1 + } + took := make(chan string, 1) + machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused}) + defer machine.Close() + go func() { + _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID }) + }() + time.Sleep(200 * time.Millisecond) + body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"}) + if _, err := js.Context().Publish(BuildWork(), body); err != nil { + t.Fatal(err) + } + select { + case id := <-took: + t.Fatalf("a holder paused as its pull was answered built %s", id) + case <-time.After(2 * time.Second): + } + q, err := ReadQueue(ctx, js, TheBuildMachine) + if err != nil { + t.Fatal(err) + } + if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" { + t.Fatalf("the ask is not back in the queue: %+v", q.Asks) + } +} diff --git a/internal/link/queue_test.go b/internal/link/queue_test.go index 4ff3a74f..a3d7716f 100644 --- a/internal/link/queue_test.go +++ b/internal/link/queue_test.go @@ -1,15 +1,10 @@ package link import ( - "context" - "encoding/json" - "sync" "testing" "time" "github.com/nats-io/nats.go" - - "github.com/novox/mesh-controller/internal/broker" ) // The build queue read and changed by hand (novox/hq ADR 0219). @@ -121,216 +116,14 @@ func TestACancelledSetReadSaysWhatWasCancelled(t *testing.T) { // --- against a real server ---------------------------------------------------------------------- -func aBusWithACancelledSet(t *testing.T) *broker.JetStream { - t.Helper() - js := aBusWithTheBuildRole(t) - seat := broker.DeclaredSeat{Name: TheBuildMachine, Accepts: []string{"build"}} - if err := broker.RaiseCancelledSets(js, []broker.DeclaredSeat{seat}); err != nil { - t.Fatal(err) - } - t.Cleanup(func() { _ = js.Context().DeleteKeyValue(broker.CancelledSetName(TheBuildMachine)) }) - return js -} - // **The race a delete alone leaves open**: an ask fetched in the moment it was cancelled is ended by // the holder that took it — terminated, never built, its outcome said as failed. -func TestNatsAnAskCancelledAsItWasTakenIsNotBuilt(t *testing.T) { - js := aBusWithACancelledSet(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - - if err := MarkCancelled(ctx, js, TheBuildMachine, "build-cancelled"); err != nil { - t.Fatal(err) - } - outcomes, err := js.Conn().SubscribeSync(BuildOutcome()) - if err != nil { - t.Fatal(err) - } - defer func() { _ = outcomes.Unsubscribe() }() - _ = js.Conn().Flush() - for _, id := range []string{"build-cancelled", "build-kept"} { - body, _ := json.Marshal(BuildRequest{ID: id, Repository: "/r"}) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } - } - - built := make(chan string, 2) - machine := MachineOverNATS(js, "anchor") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - built <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - select { - case id := <-built: - if id != "build-kept" { - t.Fatalf("%s was built", id) - } - case <-time.After(10 * time.Second): - t.Fatal("the ask that was not cancelled was never built") - } - msg, err := outcomes.NextMsg(5 * time.Second) - if err != nil { - t.Fatal(err) - } - var said BuildResult - if err := json.Unmarshal(msg.Data, &said); err != nil || said.ID != "build-cancelled" || - said.Failed != CancelledByHand || said.On != "anchor" { - t.Fatalf("the cancelled ask's outcome is %+v (%v)", said, err) - } - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if info, err := js.Context().StreamInfo("SEAT_NODE_BUILD_AGENT"); err == nil && info.State.Msgs == 0 { - return - } - time.Sleep(20 * time.Millisecond) - } - t.Fatal("the cancelled ask is still queued: terminated, it would not be") -} // A paused holder takes nothing; resumed, it takes what waited. -func TestNatsAPausedHolderTakesNothingNew(t *testing.T) { - js := aBusWithACancelledSet(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - paused := make(chan bool, 1) - paused <- true - isPaused := func() bool { - p := <-paused - paused <- p - return p - } - took := make(chan string, 1) - machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: isPaused}) - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - took <- work.Request().ID - _ = work.Announce(ctx, BuildResult{ID: work.Request().ID, On: "anchor", Failed: "no"}) - _ = work.Done() - }) - }() - body, _ := json.Marshal(BuildRequest{ID: "build-waits", Repository: "/r"}) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - t.Fatalf("a paused holder took %s", id) - case <-time.After(3 * time.Second): - } - q, err := ReadQueue(ctx, js, TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 1 || q.Asks[0].State != AskWaiting { - t.Fatalf("while paused the queue reads %+v", q.Asks) - } - <-paused - paused <- false - select { - case id := <-took: - if id != "build-waits" { - t.Fatalf("took %s", id) - } - case <-time.After(10 * time.Second): - t.Fatal("resumed, the holder never took what waited") - } -} // What a holder last said about taking work is read back per machine. -func TestNatsAHoldersPausedStateIsReadBack(t *testing.T) { - js := aBusWithTheBuildRole(t) - for _, s := range []HolderState{{On: "ace", Paused: true}, {On: "g14", Paused: true}, {On: "g14", Paused: false}} { - body, _ := json.Marshal(s) - if _, err := js.Context().Publish(BuildPausedOf(TheBuildMachine, s.On), body); err != nil { - t.Fatal(err) - } - } - said, err := PausedSaid(js, TheBuildMachine, []string{"ace", "g14", "novox"}) - if err != nil { - t.Fatal(err) - } - if !said["ace"].Paused || said["g14"].Paused || len(said) != 2 { - t.Fatalf("read back %+v", said) - } -} // A `build` waiting on its outcome hears a cancel — which publishes no outcome — from the cancelled // set, and a kill from the outcome the holder announces (novox/hq ADR 0219). -func TestNatsAWaitingAskerHearsItsAskCancelledOrKilled(t *testing.T) { - js := aBusWithACancelledSet(t) - was := cancelLook - cancelLook = 200 * time.Millisecond - t.Cleanup(func() { cancelLook = was }) - ask := &natsBuilds{js: js, seat: TheBuildMachine} - - go func() { - time.Sleep(500 * time.Millisecond) - _ = MarkCancelled(context.Background(), js, TheBuildMachine, "build-waited-cancelled") - }() - result, err := ask.Submit(context.Background(), BuildRequest{ID: "build-waited-cancelled", Repository: "/r"}, 10*time.Second) - if err != nil || result.Failed != CancelledByHand || result.ID != "build-waited-cancelled" { - t.Fatalf("the waiter heard %+v (%v)", result, err) - } - - // Killed: the holder announces the failure as any outcome, and the waiter has it. - ctx, stop := context.WithCancel(context.Background()) - defer stop() - machine := MachineOverNATS(js, "ace") - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { - r := work.Request() - _ = work.Announce(ctx, BuildResult{ID: r.ID, Repository: r.Repository, On: "ace", Failed: KilledByHand}) - _ = work.Done() - }) - }() - result, err = ask.Submit(context.Background(), BuildRequest{ID: "build-waited-killed", Repository: "/r"}, 10*time.Second) - if err != nil || result.Failed != KilledByHand || result.On != "ace" { - t.Fatalf("the waiter heard %+v (%v)", result, err) - } -} // Paused in the moment a pull was answered: the ask is handed back, not built (ADR 0219). -func TestNatsAHolderPausedAsItsPullWasAnsweredHandsTheAskBack(t *testing.T) { - js := aBusWithACancelledSet(t) - ctx, stop := context.WithCancel(context.Background()) - defer stop() - // Not paused when it asks; paused by the time the answer is read. - var looks int - var mu sync.Mutex - paused := func() bool { - mu.Lock() - defer mu.Unlock() - looks++ - return looks > 1 - } - took := make(chan string, 1) - machine := MachineOverNATSWith(js, "anchor", TheBuildMachine, MachineOptions{Paused: paused}) - defer machine.Close() - go func() { - _ = machine.Take(ctx, func(ctx context.Context, work Build) { took <- work.Request().ID }) - }() - time.Sleep(200 * time.Millisecond) - body, _ := json.Marshal(BuildRequest{ID: "build-handed-back", Repository: "/r"}) - if _, err := js.Context().Publish(BuildWork(), body); err != nil { - t.Fatal(err) - } - select { - case id := <-took: - t.Fatalf("a holder paused as its pull was answered built %s", id) - case <-time.After(2 * time.Second): - } - q, err := ReadQueue(ctx, js, TheBuildMachine) - if err != nil { - t.Fatal(err) - } - if len(q.Asks) != 1 || q.Asks[0].ID != "build-handed-back" { - t.Fatalf("the ask is not back in the queue: %+v", q.Asks) - } -} diff --git a/internal/link/receive_nats_integration_test.go b/internal/link/receive_nats_integration_test.go new file mode 100644 index 00000000..7806ef70 --- /dev/null +++ b/internal/link/receive_nats_integration_test.go @@ -0,0 +1,440 @@ +//go:build integration + +package link + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/nats-io/nats.go" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aBus(t *testing.T) *broker.JetStream { + t.Helper() + url := testbus.URL(t) + js, err := broker.Dial(url) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + + // **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong. + // + // The streams are emptied rather than deleted and recreated, because delete-then-add is not a + // reset: the server's teardown races the creation, and a test then inherits the previous one's + // messages — which reads as a redelivery bug in the code under test. + // + // The consumers are removed, because deleting a stream used to take them with it and purging + // does not. A durable *push* consumer that survives between tests keeps pushing to a delivery + // subject the previous test's subscription has gone from: the messages count as delivered, go + // nowhere, and the next test waits out its timeout for an announcement the server believes it + // already sent. The controller recreates what it needs on start, so leaving none is correct. + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + clean := func() { + for _, c := range broker.MeshConsumers() { + _ = js.Context().DeleteConsumer(c.Stream, c.Name) + } + for _, s := range broker.MeshStreams() { + _ = js.Context().PurgeStream(s.Name) + } + } + clean() + t.Cleanup(clean) + return js +} + +func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "a report being recorded", func() bool { return store.count() == 1 }) + eventually(t, "the report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up"))) + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + + // Held: the message is the server's, unacknowledged, and still in the stream. + eventually(t, "the report being redelivered at least once", func() bool { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + return err == nil && info.NumRedelivered >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 1 { + t.Fatalf("a held report did not stay on the server: %+v, %v", info, err) + } + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that was refusing it") + } + + store.refusing(nil) + eventually(t, "the report being recorded once the store was back", + func() bool { return store.count() == 1 }) + eventually(t, "the recorded report leaving the work queue", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) +} + +func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { + js := aBus(t) + store := &sentAndHeardSafely{sent: "d2"} + _, stop := servingOn(t, js, store) + defer stop() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the superseded report leaving the stream", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report about a superseded declaration was acted on") + } +} + +func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { + js := aBus(t) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + inbox := nats.NewInbox() + answers, err := js.Conn().SubscribeSync(inbox) + if err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox}) + if _, err := js.Context().Publish(EnrolSubject, body); err != nil { + t.Fatal(err) + } + + msg, err := answers.NextMsg(8 * time.Second) + if err != nil { + t.Fatalf("no answer reached the address the request named: %v", err) + } + var reply EnrolReply + if err := json.Unmarshal(msg.Data, &reply); err != nil { + t.Fatal(err) + } + if !reply.Accepted || reply.Node != "anchor" { + t.Fatalf("the answer was not the mesh's: %+v", reply) + } + // And the address really is not the one the transport carried: what the consumer saw was its + // own ack subject, which is why this had to travel in the payload. + if msg.Subject != inbox { + t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject) + } +} + +func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { + js := aBus(t) + store := &counted{} + _, stop := servingOn(t, js, store) + defer stop() + + // Given time to subscribe: a core subscription that is not yet up misses what is published, + // which is the guarantee a heartbeat has and not a fault. + eventually(t, "the heartbeat subscription coming up", func() bool { + body, _ := json.Marshal(Alive{Node: "anchor"}) + _ = js.Conn().Publish(AliveSubject("anchor"), body) + _ = js.Conn().Flush() + return store.count() >= 1 + }) + info, err := js.Context().StreamInfo("CONTROL") + if err != nil || info.State.Msgs != 0 { + t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+ + "for retention with its most valuable: %+v, %v", info, err) + } +} + +func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) { + js := aBus(t) + told := &toldAbout{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(told); err != nil { + t.Fatal(err) + } + if err := s.Answers(replaysWith{}); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + // The controller's event consumer is made from now (novox/hq issue 248): what was published before + // it existed is history it never replays. So the announcements are made once it is there. + eventually(t, "the controller's event consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil + }) + + moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil { + t.Fatal(err) + } + + eventually(t, "the catalogue's upgrade reaching the controller", + func() bool { return told.count() == 1 }) + eventually(t, "both announcements being acknowledged", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 + }) +} + +func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { + js := aBus(t) + store := &counted{} + store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused"))) + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, + listener: store, log: quiet(), giveUp: 1500 * time.Millisecond} + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + + body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + eventually(t, "the report being let go once the bound passed", func() bool { + info, err := js.Context().StreamInfo("CONTROL") + return err == nil && info.State.Msgs == 0 + }) + if store.count() != 0 { + t.Fatalf("a report was recorded by a store that never came back") + } +} + +func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { + // Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq + // issue 717: the test once set its window before the serving loop asserted the controller's own of thirty + // seconds, so it waited nine seconds and passed with the heartbeat removed). + before := heartbeatWhileWorking + heartbeatWhileWorking = 250 * time.Millisecond + t.Cleanup(func() { heartbeatWhileWorking = before }) + js := aBus(t) + var mu sync.Mutex + handled := 0 + slow := make(chan struct{}) + s, stop := servingOn(t, js, nil) + defer stop() + s.listener = slowly{func() { + mu.Lock() + handled++ + first := handled == 1 + mu.Unlock() + if first { + time.Sleep(3 * time.Second) + close(slow) + } + }} + // The controller's own consumer, given a window short enough to outlive in a test once the serving loop has + // asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds, + // which is how this test once waited out a window it never had. + for deadline := time.Now().Add(10 * time.Second); ; { + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err == nil && info.PushBound { + break + } + if time.Now().After(deadline) { + t.Fatalf("the serving loop never bound the controller's consumer: %v", err) + } + time.Sleep(10 * time.Millisecond) + } + if err := js.EnsureConsumer(broker.Consumer{ + Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1, + Why: "a window short enough to outlive in a test", + }); err != nil { + t.Fatal(err) + } + if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second { + t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err) + } + + body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + select { + case <-slow: + case <-time.After(30 * time.Second): + t.Fatal("the slow work never finished") + } + // A moment for a redelivery to arrive, if the bus were going to send one. + time.Sleep(1500 * time.Millisecond) + mu.Lock() + defer mu.Unlock() + if handled != 1 { + t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled) + } + // Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone + // would pass on a bus that redelivered the work every window; the bus's own count says it did not. + info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) + if err != nil { + t.Fatal(err) + } + if info.NumRedelivered != 0 { + t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered) + } +} + +func TestNatsTheMeshSaysWhatAMachineApplied(t *testing.T) { + js := aBus(t) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { + heard <- m + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection + + _, stop := servingOn(t, js, &counted{}) + defer stop() + + // A report that changed something: the store says it was news. + body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store", "broker"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + if m.Subject != SeatEventSubject(MeshControllerSeat, KeyApplied) { + t.Fatalf("the mesh stated %q", m.Subject) + } + var said Applied + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + if said.Node != "anchor" || said.Declared != "d1" || said.Resources != 2 { + t.Fatalf("it said %+v", said) + } + case <-time.After(10 * time.Second): + t.Fatal("the mesh said nothing about a machine that now runs something else") + } + + // A refusal is its own fact, with the reason in it rather than only in a log. + refusal, err := json.Marshal(Report{Node: "anchor", Declared: "d2", + Failed: map[string]string{"gitea.server": "no such image"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), refusal); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + if m.Subject != SeatEventSubject(MeshControllerSeat, KeyRefused) { + t.Fatalf("a refusal was stated as %q", m.Subject) + } + var said Refused + if err := json.Unmarshal(m.Data, &said); err != nil { + t.Fatal(err) + } + if said.Failed["gitea.server"] == "" { + t.Fatalf("the refusal does not say which resource or why: %+v", said) + } + case <-time.After(10 * time.Second): + t.Fatal("the mesh said nothing about a machine that refused what it was sent") + } +} + +func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) { + js := aBus(t) + heard := make(chan *nats.Msg, 4) + sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { + heard <- m + }) + if err != nil { + t.Fatal(err) + } + defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection + + // A store that records the report and says it was nothing new — which is what the mesh's own + // store says about a machine repeating itself. + _, stop := servingOn(t, js, sameAgain{}) + defer stop() + + body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) + if err != nil { + t.Fatal(err) + } + if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { + t.Fatal(err) + } + select { + case m := <-heard: + t.Fatalf("the mesh stated %q about a machine that changed nothing", m.Subject) + case <-time.After(3 * time.Second): + } +} + +func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) { + first, most := MergeHeldFirst, MergeHeldMost + MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond + t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most }) + js := aBus(t) + up := &manyTimesBusy{busy: 8} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(up); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + eventually(t, "the merges consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) + return err == nil + }) + + body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}) + if _, err := js.Context().Publish(MergedSubject, body); err != nil { + t.Fatal(err) + } + eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 }) + eventually(t, "the merge being acknowledged", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) + return err == nil && info.NumAckPending == 0 && info.NumPending == 0 + }) + if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 { + t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs) + } +} diff --git a/internal/link/receive_nats_test.go b/internal/link/receive_nats_test.go index b5940ca5..382bceac 100644 --- a/internal/link/receive_nats_test.go +++ b/internal/link/receive_nats_test.go @@ -2,7 +2,6 @@ package link import ( "context" - "encoding/json" "errors" "io" "log" @@ -10,11 +9,7 @@ import ( "testing" "time" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/broker" - - "github.com/novox/mesh-controller/internal/testbus" ) // The consume side against a real server, because what is being checked is what the server does. @@ -26,42 +21,6 @@ import ( // // go test ./internal/link/ -run TestNats (each test on a bus of its own: internal/testbus) -func aBus(t *testing.T) *broker.JetStream { - t.Helper() - url := testbus.URL(t) - js, err := broker.Dial(url) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - - // **Streams purged, consumers removed.** Both halves, and each was learned by getting it wrong. - // - // The streams are emptied rather than deleted and recreated, because delete-then-add is not a - // reset: the server's teardown races the creation, and a test then inherits the previous one's - // messages — which reads as a redelivery bug in the code under test. - // - // The consumers are removed, because deleting a stream used to take them with it and purging - // does not. A durable *push* consumer that survives between tests keeps pushing to a delivery - // subject the previous test's subscription has gone from: the messages count as delivered, go - // nowhere, and the next test waits out its timeout for an announcement the server believes it - // already sent. The controller recreates what it needs on start, so leaving none is correct. - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - clean := func() { - for _, c := range broker.MeshConsumers() { - _ = js.Context().DeleteConsumer(c.Stream, c.Name) - } - for _, s := range broker.MeshStreams() { - _ = js.Context().PurgeStream(s.Name) - } - } - clean() - t.Cleanup(clean) - return js -} - // serving1 is a controller reading from a real bus, and a way to stop it. func servingOn(t *testing.T, js *broker.JetStream, l Listener) (*Server, func()) { t.Helper() @@ -122,81 +81,13 @@ func eventually(t *testing.T, what string, is func() bool) { // stream does not hold it. A work queue is the check: what is acknowledged leaves it. func quiet() *log.Logger { return log.New(io.Discard, "", 0) } -func TestNatsAReportIsHeardAndLeavesTheStream(t *testing.T) { - js := aBus(t) - store := &counted{} - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "a report being recorded", func() bool { return store.count() == 1 }) - eventually(t, "the report leaving the work queue", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) -} - // **The store window, in the server.** A report the store cannot take is naked with a delay and // comes back; once the store is there it is recorded and leaves the stream. The controller holds // nothing in the meantime — which is what the sequence check below is for: the message is still on // the server while it waits. -func TestNatsAReportTheStoreCannotTakeIsHeldByTheServerAndComesBack(t *testing.T) { - js := aBus(t) - store := &counted{} - store.refusing(errors.Join(ErrTryAgain, errors.New("the database system is starting up"))) - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - - // Held: the message is the server's, unacknowledged, and still in the stream. - eventually(t, "the report being redelivered at least once", func() bool { - info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) - return err == nil && info.NumRedelivered >= 1 - }) - info, err := js.Context().StreamInfo("CONTROL") - if err != nil || info.State.Msgs != 1 { - t.Fatalf("a held report did not stay on the server: %+v, %v", info, err) - } - if store.count() != 0 { - t.Fatalf("a report was recorded by a store that was refusing it") - } - - store.refusing(nil) - eventually(t, "the report being recorded once the store was back", - func() bool { return store.count() == 1 }) - eventually(t, "the recorded report leaving the work queue", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) -} // A report about a declaration the mesh has moved past is settled without being acted on, and // leaves the stream rather than coming back for ever. -func TestNatsASupersededReportIsSettledAndNotActedOn(t *testing.T) { - js := aBus(t) - store := &sentAndHeardSafely{sent: "d2"} - _, stop := servingOn(t, js, store) - defer stop() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "the superseded report leaving the stream", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) - if store.count() != 0 { - t.Fatalf("a report about a superseded declaration was acted on") - } -} // sentAndHeardSafely is sentAndHeard, read from two goroutines. type sentAndHeardSafely struct { @@ -228,41 +119,6 @@ func (s *sentAndHeardSafely) count() int { // that the answer arrives there — which is the whole reason the address is a field rather than the // transport's reply, and this is the test design 25 §2 asks for so the reason cannot quietly become // folklore. -func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { - js := aBus(t) - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, - enroller: enrolsAs{reply: EnrolReply{Accepted: true, Node: "anchor"}}, log: quiet()} - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - - inbox := nats.NewInbox() - answers, err := js.Conn().SubscribeSync(inbox) - if err != nil { - t.Fatal(err) - } - body, _ := json.Marshal(EnrolRequest{Node: "anchor", Secret: "t", ReplyTo: inbox}) - if _, err := js.Context().Publish(EnrolSubject, body); err != nil { - t.Fatal(err) - } - - msg, err := answers.NextMsg(8 * time.Second) - if err != nil { - t.Fatalf("no answer reached the address the request named: %v", err) - } - var reply EnrolReply - if err := json.Unmarshal(msg.Data, &reply); err != nil { - t.Fatal(err) - } - if !reply.Accepted || reply.Node != "anchor" { - t.Fatalf("the answer was not the mesh's: %+v", reply) - } - // And the address really is not the one the transport carried: what the consumer saw was its - // own ack subject, which is why this had to travel in the payload. - if msg.Subject != inbox { - t.Fatalf("the answer arrived on %s, not the address the request named", msg.Subject) - } -} // enrolsAs answers every request the same way. type enrolsAs struct{ reply EnrolReply } @@ -271,26 +127,6 @@ func (e enrolsAs) Enrol(context.Context, EnrolRequest) (EnrolReply, error) { ret // A heartbeat is core NATS: it reaches the controller and nothing is persisted, so the stream the // reports live in stays empty. -func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { - js := aBus(t) - store := &counted{} - _, stop := servingOn(t, js, store) - defer stop() - - // Given time to subscribe: a core subscription that is not yet up misses what is published, - // which is the guarantee a heartbeat has and not a fault. - eventually(t, "the heartbeat subscription coming up", func() bool { - body, _ := json.Marshal(Alive{Node: "anchor"}) - _ = js.Conn().Publish(AliveSubject("anchor"), body) - _ = js.Conn().Flush() - return store.count() >= 1 - }) - info, err := js.Context().StreamInfo("CONTROL") - if err != nil || info.State.Msgs != 0 { - t.Fatalf("a heartbeat was persisted, and the mesh's least valuable message now competes "+ - "for retention with its most valuable: %+v, %v", info, err) - } -} // The two events the controller follows arrive over one durable consumer with two filters, and it // can acknowledge them. @@ -299,41 +135,6 @@ func TestNatsAHeartbeatIsHeardAndNothingIsKept(t *testing.T) { // is the first thing in the mesh to use one; and a delivery from the events stream is acknowledged // on a different ack subject from a delivery from the control stream, which the controller's own // permission list has to cover or every announcement is redelivered for ever. -func TestNatsTheEventsTheControllerFollowsArriveAndAreAcknowledged(t *testing.T) { - js := aBus(t) - told := &toldAbout{} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(told); err != nil { - t.Fatal(err) - } - if err := s.Answers(replaysWith{}); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - // The controller's event consumer is made from now (novox/hq issue 248): what was published before - // it existed is history it never replays. So the announcements are made once it is there. - eventually(t, "the controller's event consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil - }) - - moved, _ := json.Marshal(Upgraded{Module: "gitea", Commit: "abcdef0123"}) - if _, err := js.Context().Publish(broker.ControllerFollows[0], moved); err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(broker.ControllerFollows[1], []byte(`{}`)); err != nil { - t.Fatal(err) - } - - eventually(t, "the catalogue's upgrade reaching the controller", - func() bool { return told.count() == 1 }) - eventually(t, "both announcements being acknowledged", func() bool { - info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 - }) -} type toldAbout struct { mu sync.Mutex @@ -360,28 +161,6 @@ func (u *toldAbout) count() int { // A store that never comes back: the report is let go once the bound passes, and it leaves the // stream rather than being held for ever. The bound is the controller's, not the server's — nothing // here sets max-deliver, and that is deliberate (streams.go). -func TestNatsAReportIsLetGoOnceTheStoreHasBeenGoneTooLong(t *testing.T) { - js := aBus(t) - store := &counted{} - store.refusing(errors.Join(ErrTryAgain, errors.New("connection refused"))) - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, - listener: store, log: quiet(), giveUp: 1500 * time.Millisecond} - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - - body, _ := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - eventually(t, "the report being let go once the bound passed", func() bool { - info, err := js.Context().StreamInfo("CONTROL") - return err == nil && info.State.Msgs == 0 - }) - if store.count() != 0 { - t.Fatalf("a report was recorded by a store that never came back") - } -} // A merge announcement is not what these tests are about; taken and forgotten. func (t *toldAbout) SourceMoved(context.Context, SourceMoved) error { return nil } @@ -406,81 +185,6 @@ func TestEverySubjectTheControllerFollowsDecodesToAKind(t *testing.T) { // a consumer that died and wrong for one that is busy. Acting on a merge builds modules — minutes // against a thirty-second window — and the same merge was handed over five times while the first // build was still running (2026-09-28). Here the window is a second and the work takes three. -func TestNatsWorkSlowerThanTheWindowIsNotHandedOverAgain(t *testing.T) { - // Work that outlives a window of a second three times over, said while it runs four times a window (novox/hq - // issue 717: the test once set its window before the serving loop asserted the controller's own of thirty - // seconds, so it waited nine seconds and passed with the heartbeat removed). - before := heartbeatWhileWorking - heartbeatWhileWorking = 250 * time.Millisecond - t.Cleanup(func() { heartbeatWhileWorking = before }) - js := aBus(t) - var mu sync.Mutex - handled := 0 - slow := make(chan struct{}) - s, stop := servingOn(t, js, nil) - defer stop() - s.listener = slowly{func() { - mu.Lock() - handled++ - first := handled == 1 - mu.Unlock() - if first { - time.Sleep(3 * time.Second) - close(slow) - } - }} - // The controller's own consumer, given a window short enough to outlive in a test once the serving loop has - // asserted it and bound to it: asserted after this, it would be the controller's window of thirty seconds, - // which is how this test once waited out a window it never had. - for deadline := time.Now().Add(10 * time.Second); ; { - info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) - if err == nil && info.PushBound { - break - } - if time.Now().After(deadline) { - t.Fatalf("the serving loop never bound the controller's consumer: %v", err) - } - time.Sleep(10 * time.Millisecond) - } - if err := js.EnsureConsumer(broker.Consumer{ - Name: broker.ControllerName, Stream: "CONTROL", Push: true, AckWaitSeconds: 1, - Why: "a window short enough to outlive in a test", - }); err != nil { - t.Fatal(err) - } - if info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName); err != nil || info.Config.AckWait != time.Second { - t.Fatalf("the controller's consumer does not have the test's window of a second: %v", err) - } - - body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - select { - case <-slow: - case <-time.After(30 * time.Second): - t.Fatal("the slow work never finished") - } - // A moment for a redelivery to arrive, if the bus were going to send one. - time.Sleep(1500 * time.Millisecond) - mu.Lock() - defer mu.Unlock() - if handled != 1 { - t.Fatalf("one report was handled %d times, so slow work is run again while it is running", handled) - } - // Not handed over again at all: the loop drops a copy of what it is already doing, so the count above alone - // would pass on a bus that redelivered the work every window; the bus's own count says it did not. - info, err := js.Context().ConsumerInfo("CONTROL", broker.ControllerName) - if err != nil { - t.Fatal(err) - } - if info.NumRedelivered != 0 { - t.Fatalf("the bus handed the report over again %d time(s) while it was being worked on", info.NumRedelivered) - } -} // slowly is a listener that runs whatever it was given. type slowly struct{ work func() } @@ -490,101 +194,9 @@ func (s slowly) Heard(context.Context, Report) (bool, error) { s.work(); return // **The mesh says what it applied** (novox/hq ADR 0134), under the seat the control plane holds — and // says nothing when a report is the same state said again, which is what a machine reconciling every // minute sends. -func TestNatsTheMeshSaysWhatAMachineApplied(t *testing.T) { - js := aBus(t) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { - heard <- m - }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection - - _, stop := servingOn(t, js, &counted{}) - defer stop() - - // A report that changed something: the store says it was news. - body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store", "broker"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - if m.Subject != SeatEventSubject(MeshControllerSeat, KeyApplied) { - t.Fatalf("the mesh stated %q", m.Subject) - } - var said Applied - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said.Node != "anchor" || said.Declared != "d1" || said.Resources != 2 { - t.Fatalf("it said %+v", said) - } - case <-time.After(10 * time.Second): - t.Fatal("the mesh said nothing about a machine that now runs something else") - } - - // A refusal is its own fact, with the reason in it rather than only in a log. - refusal, err := json.Marshal(Report{Node: "anchor", Declared: "d2", - Failed: map[string]string{"gitea.server": "no such image"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), refusal); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - if m.Subject != SeatEventSubject(MeshControllerSeat, KeyRefused) { - t.Fatalf("a refusal was stated as %q", m.Subject) - } - var said Refused - if err := json.Unmarshal(m.Data, &said); err != nil { - t.Fatal(err) - } - if said.Failed["gitea.server"] == "" { - t.Fatalf("the refusal does not say which resource or why: %+v", said) - } - case <-time.After(10 * time.Second): - t.Fatal("the mesh said nothing about a machine that refused what it was sent") - } -} // And a report that is not news is not a fact. A machine reconciles every minute; a fact per report // would be a fact per minute per machine, which is a stream nobody reads. -func TestNatsAReportThatIsNotNewsIsNotStated(t *testing.T) { - js := aBus(t) - heard := make(chan *nats.Msg, 4) - sub, err := js.Conn().Subscribe(SeatEventSubject(MeshControllerSeat, ">"), func(m *nats.Msg) { - heard <- m - }) - if err != nil { - t.Fatal(err) - } - defer sub.Unsubscribe() //nolint:errcheck // the subscription dies with the connection - - // A store that records the report and says it was nothing new — which is what the mesh's own - // store says about a machine repeating itself. - _, stop := servingOn(t, js, sameAgain{}) - defer stop() - - body, err := json.Marshal(Report{Node: "anchor", Declared: "d1", Applied: []string{"store"}}) - if err != nil { - t.Fatal(err) - } - if _, err := js.Context().Publish(ReportSubject("anchor"), body); err != nil { - t.Fatal(err) - } - select { - case m := <-heard: - t.Fatalf("the mesh stated %q about a machine that changed nothing", m.Subject) - case <-time.After(3 * time.Second): - } -} // sameAgain records a report and says it was the same state said again. type sameAgain struct{} @@ -617,37 +229,6 @@ func (u *manyTimesBusy) called() int { // **A merge held more than five times is still handed over and recorded** (issue 973, novox/hq ADR 0407 decision // 3): the merges have a durable consumer of their own with no max-deliver, so they never go to DEAD_LETTERS to wait // for a person; the delay grows, and is capped. -func TestNatsAMergeHeldMoreThanFiveTimesIsStillHandedOverAndRecorded(t *testing.T) { - first, most := MergeHeldFirst, MergeHeldMost - MergeHeldFirst, MergeHeldMost = 5*time.Millisecond, 40*time.Millisecond - t.Cleanup(func() { MergeHeldFirst, MergeHeldMost = first, most }) - js := aBus(t) - up := &manyTimesBusy{busy: 8} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(up); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - eventually(t, "the merges consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) - return err == nil - }) - - body, _ := json.Marshal(SourceMoved{Owner: "novox", Repo: "one", Base: "main", Commit: "abcdef0123"}) - if _, err := js.Context().Publish(MergedSubject, body); err != nil { - t.Fatal(err) - } - eventually(t, "the ninth hand-over recording the merge", func() bool { return up.called() == 9 }) - eventually(t, "the merge being acknowledged", func() bool { - info, err := js.Context().ConsumerInfo("EVENTS", broker.MergesConsumerName) - return err == nil && info.NumAckPending == 0 && info.NumPending == 0 - }) - if info, err := js.Context().StreamInfo(broker.DeadLettersStream); err == nil && info.State.Msgs != 0 { - t.Fatalf("%d message(s) in dead letters: a merge was given up on", info.State.Msgs) - } -} // The delay a held merge waits grows from the first to the most and stays there. func TestAHeldMergesDelayGrowsAndIsCapped(t *testing.T) { diff --git a/internal/link/rekey_test.go b/internal/link/rekey_test.go index fc7be801..c6f295ca 100644 --- a/internal/link/rekey_test.go +++ b/internal/link/rekey_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link_test import ( diff --git a/internal/link/replay314_test.go b/internal/link/replay314_test.go index f7d57965..6ef63ded 100644 --- a/internal/link/replay314_test.go +++ b/internal/link/replay314_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/replay330_test.go b/internal/link/replay330_test.go index ac25b362..7e56ce64 100644 --- a/internal/link/replay330_test.go +++ b/internal/link/replay330_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/seat_as_test.go b/internal/link/seat_as_test.go index c25ca8d0..a6f7f8b2 100644 --- a/internal/link/seat_as_test.go +++ b/internal/link/seat_as_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/setuid_search_test.go b/internal/link/setuid_search_test.go index 4a05aa57..7ced258b 100644 --- a/internal/link/setuid_search_test.go +++ b/internal/link/setuid_search_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/standby_nats_test.go b/internal/link/standby_nats_test.go index ac5940f0..31e5ba8d 100644 --- a/internal/link/standby_nats_test.go +++ b/internal/link/standby_nats_test.go @@ -1,3 +1,5 @@ +//go:build integration + package link import ( diff --git a/internal/link/standing_integration_test.go b/internal/link/standing_integration_test.go new file mode 100644 index 00000000..68183867 --- /dev/null +++ b/internal/link/standing_integration_test.go @@ -0,0 +1,48 @@ +//go:build integration + +package link + +import ( + "context" + "testing" + + "github.com/novox/mesh-controller/internal/broker" +) + +func TestNatsAProvidersStandingReachesTheController(t *testing.T) { + js := aBus(t) + kept := &lockedStandings{} + s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} + if err := s.Follows(&toldAbout{}); err != nil { + t.Fatal(err) + } + if err := s.Watches(kept); err != nil { + t.Fatal(err) + } + ctx, stop := context.WithCancel(context.Background()) + defer stop() + go func() { _ = s.Serve(ctx) }() + eventually(t, "the controller's event consumer being made", func() bool { + _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil + }) + + for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} { + if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event, + []byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil { + t.Fatal(err) + } + } + // Somebody else's event under the same prefix is not the controller's to hear. + if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil { + t.Fatal(err) + } + eventually(t, "both standings being kept, in order, naming the emitter", func() bool { + got := kept.all() + return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing + }) + eventually(t, "both being acknowledged and nothing else delivered", func() bool { + info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) + return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 + }) +} diff --git a/internal/link/standing_test.go b/internal/link/standing_test.go index 4d7b3f34..1a4136af 100644 --- a/internal/link/standing_test.go +++ b/internal/link/standing_test.go @@ -150,43 +150,6 @@ func TestAStandingWithNothingKeepingItIsTaken(t *testing.T) { // Over a real bus: a provider's standing published under its own module's namespace reaches the // controller through the events consumer's filter — the one wildcard filter on it — names the emitter // from the subject, and is acknowledged. -func TestNatsAProvidersStandingReachesTheController(t *testing.T) { - js := aBus(t) - kept := &lockedStandings{} - s := &Server{inbound: Nats(js), bus: OverNATS{Conn: js.Conn(), JS: js.Context()}, log: quiet()} - if err := s.Follows(&toldAbout{}); err != nil { - t.Fatal(err) - } - if err := s.Watches(kept); err != nil { - t.Fatal(err) - } - ctx, stop := context.WithCancel(context.Background()) - defer stop() - go func() { _ = s.Serve(ctx) }() - eventually(t, "the controller's event consumer being made", func() bool { - _, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil - }) - - for _, event := range []string{broker.ProvisionerFailing, broker.ProvisionerRecovered} { - if _, err := js.Context().Publish("mesh.mod.keycloak.event."+event, - []byte(`{"consumer":"mesh_home_grafana","provider-node":"anchor","class":"credentials-rejected"}`)); err != nil { - t.Fatal(err) - } - } - // Somebody else's event under the same prefix is not the controller's to hear. - if _, err := js.Context().Publish("mesh.mod.keycloak.event.client.created", []byte(`{}`)); err != nil { - t.Fatal(err) - } - eventually(t, "both standings being kept, in order, naming the emitter", func() bool { - got := kept.all() - return len(got) == 2 && got[0].Module == "keycloak" && got[0].Failing && !got[1].Failing - }) - eventually(t, "both being acknowledged and nothing else delivered", func() bool { - info, err := js.Context().ConsumerInfo("EVENTS", broker.ControllerName) - return err == nil && info.NumAckPending == 0 && info.Delivered.Consumer == 2 - }) -} type lockedStandings struct { mu sync.Mutex diff --git a/internal/lint/mergecheck_unit_level_test.go b/internal/lint/mergecheck_unit_level_test.go new file mode 100644 index 00000000..b019ce54 --- /dev/null +++ b/internal/lint/mergecheck_unit_level_test.go @@ -0,0 +1,76 @@ +//go:build integration + +package lint + +import ( + "context" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +// judgeUnitRun runs merge-check.sh's judging of the unit level on a recorded `go test -v` output (issue 895), as it +// would judge the output of the untagged run: it starts the shell, so it is an integration test. +func judgeUnitRun(t *testing.T, recorded string, bound string) (string, error) { + t.Helper() + out := filepath.Join(t.TempDir(), "unit.out") + if err := os.WriteFile(out, []byte(recorded), 0o600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "sh", "merge-check.sh", "judge-unit-run", out, bound) + cmd.Dir = filepath.Join("..", "..") + said, err := cmd.CombinedOutput() + return string(said), err +} + +const recordedUnitRun = `=== RUN TestQuick +--- PASS: TestQuick (0.01s) +=== RUN TestSlowOne +--- PASS: TestSlowOne (2.50s) +=== RUN TestSlowTwo + --- PASS: TestSlowTwo/sub (1.00s) +--- PASS: TestSlowTwo (1.20s) +--- SKIP: TestNeedsAStore (0.00s) +PASS +ok example.test/internal/one 2.80s +=== RUN TestElsewhere +--- PASS: TestElsewhere (0.30s) +PASS +ok example.test/internal/two 0.35s +? example.test/internal/three [no test files] +` + +// **The unit level over its bound fails the check and names the slowest tests** (issue 895). +func TestAUnitRunOverItsBoundFailsAndNamesTheSlowestTests(t *testing.T) { + said, err := judgeUnitRun(t, recordedUnitRun, "1") + if err == nil { + t.Fatalf("a unit run of 2.80 s passed a bound of 1 s: %s", said) + } + for _, want := range []string{"5 tests (1 skipped)", "2.80 s (bound 1 s)", "TestSlowOne", "TestSlowTwo", "example.test/internal/one"} { + if !strings.Contains(said, want) { + t.Errorf("the verdict does not say %q: %s", want, said) + } + } + if strings.Index(said, "TestSlowOne") > strings.Index(said, "TestSlowTwo") { + t.Errorf("the slowest test is not named first: %s", said) + } + if strings.Contains(said, "TestSlowTwo/sub") { + t.Errorf("a subtest is named as if it were a test of its own: %s", said) + } +} + +// A unit run within its bound passes and says its count and time. +func TestAUnitRunWithinItsBoundPasses(t *testing.T) { + said, err := judgeUnitRun(t, recordedUnitRun, "3") + if err != nil { + t.Fatalf("a unit run of 2.80 s failed a bound of 3 s: %v: %s", err, said) + } + if !strings.Contains(said, "unit level: 5 tests (1 skipped), 2.80 s (bound 3 s)") { + t.Errorf("the level's count and time are not said: %s", said) + } +} diff --git a/internal/recordmirror/mirror_integration_test.go b/internal/recordmirror/mirror_integration_test.go new file mode 100644 index 00000000..b4a1bebd --- /dev/null +++ b/internal/recordmirror/mirror_integration_test.go @@ -0,0 +1,248 @@ +//go:build integration + +package recordmirror + +import ( + "errors" + "slices" + "testing" + + "github.com/nats-io/nats.go/jetstream" + "github.com/novox/mesh-controller/internal/broker" + "github.com/novox/mesh-controller/internal/testbus" +) + +func aMirrorBus(t *testing.T) (*broker.JetStream, jetstream.JetStream) { + t.Helper() + js, err := broker.Dial(testbus.URL(t)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(js.Close) + if err := broker.AssertMeshStreams(js); err != nil { + t.Fatal(err) + } + if err := js.EnsureControllerBuckets(); err != nil { + t.Fatal(err) + } + api, err := jetstream.New(js.Conn()) + if err != nil { + t.Fatal(err) + } + return js, api +} + +func TestNatsAMirrorChangesOneKeyAndRaisesOneEventPerChange(t *testing.T) { + for _, m := range []Mirror{Assignments, DeclaredConfig} { + t.Run(m.Name, func(t *testing.T) { + js, api := aMirrorBus(t) + events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+m.Event) + w := aWriter(t, js, api, m) + ctx := t.Context() + + first := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)} + if _, err := w.Sync(ctx, first); err != nil { + t.Fatal(err) + } + before := writes(t, api, m.Bucket) + heardBefore := len(settle(events, 1)) + + // One record changes, one is added: two keys, two events, nothing else. + second := map[string][]byte{"anchor.gitea": []byte(`{"a":2}`), "anchor.forgejo": []byte(`{"a":1}`)} + r, err := w.Sync(ctx, second) + if err != nil { + t.Fatal(err) + } + if r.Written != 2 || r.Removed != 0 { + t.Errorf("a pass over one changed and one new record wrote %+v", r) + } + if got := writes(t, api, m.Bucket) - before; got != 2 { + t.Errorf("the bucket took %d writes for two changed records", got) + } + said := settle(events, heardBefore+2)[heardBefore:] + if len(said) != 2 { + t.Fatalf("two changes raised %d events: %+v", len(said), said) + } + changes := map[string]Event{} + for _, e := range said { + changes[e.Key] = e + } + if e := changes["anchor.gitea"]; e.Change != Changed || !slices.Equal(e.Fields, []string{"a"}) || + e.Mirror != m.Name || e.Epoch != 7 || e.Revision == 0 { + t.Errorf("the change of anchor.gitea was said as %+v", e) + } + if e := changes["anchor.forgejo"]; e.Change != Added { + t.Errorf("the new anchor.forgejo was said as %+v", e) + } + + // **An unchanged pass writes nothing**, and says nothing (ADR 0289 §1). + heardBefore = len(events.all()) + before = writes(t, api, m.Bucket) + r, err = w.Sync(ctx, second) + if err != nil { + t.Fatal(err) + } + if r != (Result{}) { + t.Errorf("a pass over unchanged records did %+v", r) + } + if got := writes(t, api, m.Bucket) - before; got != 0 { + t.Errorf("a pass over unchanged records wrote the bucket %d times", got) + } + if said := settle(events, heardBefore+1)[heardBefore:]; len(said) != 0 { + t.Errorf("a pass over unchanged records raised %+v", said) + } + + // **A record gone is a purge**, one write and one `removed`. + before = writes(t, api, m.Bucket) + r, err = w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`)}) + if err != nil { + t.Fatal(err) + } + if r.Removed != 1 || r.Written != 0 { + t.Errorf("a record gone did %+v", r) + } + if got := writes(t, api, m.Bucket) - before; got != 1 { + t.Errorf("a record gone wrote the bucket %d times", got) + } + if _, err := w.KV.Get(ctx, "anchor.forgejo"); !errors.Is(err, jetstream.ErrKeyNotFound) { + t.Errorf("the key of a record gone is still read: %v", err) + } + said = settle(events, heardBefore+1)[heardBefore:] + if len(said) != 1 || said[0].Change != Removed || said[0].Key != "anchor.forgejo" { + t.Errorf("a record gone was said as %+v", said) + } + }) + } +} + +func TestNatsAWriterStartedAgainWritesOnlyWhatDiffers(t *testing.T) { + js, api := aMirrorBus(t) + ctx := t.Context() + first := aWriter(t, js, api, Assignments) + held := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`), "anchor.forgejo": []byte(`{"a":1}`)} + if _, err := first.Sync(ctx, held); err != nil { + t.Fatal(err) + } + before := writes(t, api, Assignments.Bucket) + again := aWriter(t, js, api, Assignments) + r, err := again.Sync(ctx, held) + if err != nil { + t.Fatal(err) + } + if r != (Result{}) || writes(t, api, Assignments.Bucket) != before { + t.Errorf("a writer started again over unchanged records did %+v", r) + } +} + +func TestNatsAWriteRefusedByCompareAndSetReadsAgain(t *testing.T) { + js, api := aMirrorBus(t) + ctx := t.Context() + w := aWriter(t, js, api, Assignments) + if _, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil { + t.Fatal(err) + } + // Behind its back, as a former lease holder would. + if _, err := w.KV.Put(ctx, "anchor.gitea", []byte(`{"a":9}`)); err != nil { + t.Fatal(err) + } + r, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`)}) + if err != nil { + t.Fatal(err) + } + if r.Written != 1 { + t.Errorf("a refused write was not made again: %+v", r) + } + e, err := w.KV.Get(ctx, "anchor.gitea") + if err != nil || string(e.Value()) != `{"a":2}` { + t.Fatalf("the bucket holds %v, %v", e, err) + } +} + +func TestNatsAForeignWriteIsCorrectedOnTheNextPass(t *testing.T) { + js, api := aMirrorBus(t) + ctx := t.Context() + events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event) + w := aWriter(t, js, api, Assignments) + store := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)} + if _, err := w.Sync(ctx, store); err != nil { + t.Fatal(err) + } + heardBefore := len(settle(events, 1)) + if _, err := w.KV.Put(ctx, "anchor.gitea", []byte(`{"a":9}`)); err != nil { + t.Fatal(err) + } + r, err := w.Sync(ctx, store) + if err != nil { + t.Fatal(err) + } + if r != (Result{Written: 1}) { + t.Errorf("a pass after a foreign write did %+v", r) + } + e, err := w.KV.Get(ctx, "anchor.gitea") + if err != nil || string(e.Value()) != `{"a":1}` { + t.Fatalf("the bucket still holds %v, %v", e, err) + } + if said := settle(events, heardBefore+1)[heardBefore:]; len(said) != 1 || said[0].Change != Changed { + t.Errorf("the correction was said as %+v", said) + } + // And put right, it stays: the next pass writes nothing. + if r, err := w.Sync(ctx, store); err != nil || r != (Result{}) { + t.Errorf("a pass after the correction did %+v, %v", r, err) + } +} + +func TestNatsAWriterWithoutTheLeaseWritesNothing(t *testing.T) { + js, api := aMirrorBus(t) + ctx := t.Context() + held := aWriter(t, js, api, Assignments) + if _, err := held.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil { + t.Fatal(err) + } + before := writes(t, api, Assignments.Bucket) + for _, holds := range []func() bool{nil, func() bool { return false }} { + w := aWriter(t, js, api, Assignments) + w.Holds = holds + r, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`), "anchor.forgejo": []byte(`{"a":1}`)}) + if !errors.Is(err, ErrNotHeld) || r != (Result{}) { + t.Errorf("a writer without the lease did %+v, %v", r, err) + } + r, err = w.Sync(ctx, map[string][]byte{}) + if !errors.Is(err, ErrNotHeld) || r != (Result{}) { + t.Errorf("a writer without the lease removed: %+v, %v", r, err) + } + } + // And one that loses it mid-pass stops before its next write. + asked := 0 + w := aWriter(t, js, api, Assignments) + w.Holds = func() bool { asked++; return asked <= 2 } + r, err := w.Sync(ctx, map[string][]byte{"anchor.a": []byte(`{}`), "anchor.b": []byte(`{}`), "anchor.gitea": []byte(`{"a":1}`)}) + if !errors.Is(err, ErrNotHeld) || r.Written != 1 { + t.Errorf("a writer that lost the lease mid-pass did %+v, %v", r, err) + } + if got := writes(t, api, Assignments.Bucket) - before; got != 1 { + t.Errorf("writers without the lease wrote the bucket %d times", got) + } +} + +func TestNatsARemovalWithoutTheLeaseRemovesNothing(t *testing.T) { + js, api := aMirrorBus(t) + ctx := t.Context() + held := aWriter(t, js, api, Assignments) + if _, err := held.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil { + t.Fatal(err) + } + before := writes(t, api, Assignments.Bucket) + asked := 0 + w := aWriter(t, js, api, Assignments) + w.Holds = func() bool { asked++; return asked == 1 } + r, err := w.Sync(ctx, map[string][]byte{}) + if !errors.Is(err, ErrNotHeld) || r != (Result{}) || asked != 2 { + t.Errorf("a removal without the lease did %+v, %v (the lease asked %d times)", r, err, asked) + } + if got := writes(t, api, Assignments.Bucket) - before; got != 0 { + t.Errorf("a removal without the lease wrote the bucket %d times", got) + } + if _, err := w.KV.Get(ctx, "anchor.gitea"); err != nil { + t.Errorf("the key was removed without the lease: %v", err) + } +} diff --git a/internal/recordmirror/mirror_store_test.go b/internal/recordmirror/mirror_store_test.go index 304f8e33..34d196df 100644 --- a/internal/recordmirror/mirror_store_test.go +++ b/internal/recordmirror/mirror_store_test.go @@ -1,3 +1,5 @@ +//go:build integration + package recordmirror import ( diff --git a/internal/recordmirror/mirror_test.go b/internal/recordmirror/mirror_test.go index 9d987e37..71cec884 100644 --- a/internal/recordmirror/mirror_test.go +++ b/internal/recordmirror/mirror_test.go @@ -2,7 +2,6 @@ package recordmirror import ( "encoding/json" - "errors" "slices" "sync" "testing" @@ -13,7 +12,6 @@ import ( "github.com/novox/mesh-controller/internal/broker" "github.com/novox/mesh-controller/internal/link" - "github.com/novox/mesh-controller/internal/testbus" ) // A record mirror against a real bus (novox/hq design 54 §2, §3; ADR 0289): a change of a record changes @@ -22,25 +20,6 @@ import ( // aMirrorBus is a bus of this test alone with the mesh's streams and the controller's buckets, as the // controller asserts them at its start, and a counter of the writes made to one bucket. -func aMirrorBus(t *testing.T) (*broker.JetStream, jetstream.JetStream) { - t.Helper() - js, err := broker.Dial(testbus.URL(t)) - if err != nil { - t.Fatal(err) - } - t.Cleanup(js.Close) - if err := broker.AssertMeshStreams(js); err != nil { - t.Fatal(err) - } - if err := js.EnsureControllerBuckets(); err != nil { - t.Fatal(err) - } - api, err := jetstream.New(js.Conn()) - if err != nil { - t.Fatal(err) - } - return js, api -} // heard counts what is said on one subject, each body kept. type heard struct { @@ -112,230 +91,20 @@ func aWriter(t *testing.T, js *broker.JetStream, api jetstream.JetStream, m Mirr Epoch: func() uint64 { return 7 }, Holds: func() bool { return true }} } -func TestNatsAMirrorChangesOneKeyAndRaisesOneEventPerChange(t *testing.T) { - for _, m := range []Mirror{Assignments, DeclaredConfig} { - t.Run(m.Name, func(t *testing.T) { - js, api := aMirrorBus(t) - events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+m.Event) - w := aWriter(t, js, api, m) - ctx := t.Context() - - first := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)} - if _, err := w.Sync(ctx, first); err != nil { - t.Fatal(err) - } - before := writes(t, api, m.Bucket) - heardBefore := len(settle(events, 1)) - - // One record changes, one is added: two keys, two events, nothing else. - second := map[string][]byte{"anchor.gitea": []byte(`{"a":2}`), "anchor.forgejo": []byte(`{"a":1}`)} - r, err := w.Sync(ctx, second) - if err != nil { - t.Fatal(err) - } - if r.Written != 2 || r.Removed != 0 { - t.Errorf("a pass over one changed and one new record wrote %+v", r) - } - if got := writes(t, api, m.Bucket) - before; got != 2 { - t.Errorf("the bucket took %d writes for two changed records", got) - } - said := settle(events, heardBefore+2)[heardBefore:] - if len(said) != 2 { - t.Fatalf("two changes raised %d events: %+v", len(said), said) - } - changes := map[string]Event{} - for _, e := range said { - changes[e.Key] = e - } - if e := changes["anchor.gitea"]; e.Change != Changed || !slices.Equal(e.Fields, []string{"a"}) || - e.Mirror != m.Name || e.Epoch != 7 || e.Revision == 0 { - t.Errorf("the change of anchor.gitea was said as %+v", e) - } - if e := changes["anchor.forgejo"]; e.Change != Added { - t.Errorf("the new anchor.forgejo was said as %+v", e) - } - - // **An unchanged pass writes nothing**, and says nothing (ADR 0289 §1). - heardBefore = len(events.all()) - before = writes(t, api, m.Bucket) - r, err = w.Sync(ctx, second) - if err != nil { - t.Fatal(err) - } - if r != (Result{}) { - t.Errorf("a pass over unchanged records did %+v", r) - } - if got := writes(t, api, m.Bucket) - before; got != 0 { - t.Errorf("a pass over unchanged records wrote the bucket %d times", got) - } - if said := settle(events, heardBefore+1)[heardBefore:]; len(said) != 0 { - t.Errorf("a pass over unchanged records raised %+v", said) - } - - // **A record gone is a purge**, one write and one `removed`. - before = writes(t, api, m.Bucket) - r, err = w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`)}) - if err != nil { - t.Fatal(err) - } - if r.Removed != 1 || r.Written != 0 { - t.Errorf("a record gone did %+v", r) - } - if got := writes(t, api, m.Bucket) - before; got != 1 { - t.Errorf("a record gone wrote the bucket %d times", got) - } - if _, err := w.KV.Get(ctx, "anchor.forgejo"); !errors.Is(err, jetstream.ErrKeyNotFound) { - t.Errorf("the key of a record gone is still read: %v", err) - } - said = settle(events, heardBefore+1)[heardBefore:] - if len(said) != 1 || said[0].Change != Removed || said[0].Key != "anchor.forgejo" { - t.Errorf("a record gone was said as %+v", said) - } - }) - } -} - // **A controller started again writes nothing for what the bucket already holds** (design 54 §2: the writer's // memory is filled from the bucket), and a key nobody holds a record for is removed. -func TestNatsAWriterStartedAgainWritesOnlyWhatDiffers(t *testing.T) { - js, api := aMirrorBus(t) - ctx := t.Context() - first := aWriter(t, js, api, Assignments) - held := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`), "anchor.forgejo": []byte(`{"a":1}`)} - if _, err := first.Sync(ctx, held); err != nil { - t.Fatal(err) - } - before := writes(t, api, Assignments.Bucket) - again := aWriter(t, js, api, Assignments) - r, err := again.Sync(ctx, held) - if err != nil { - t.Fatal(err) - } - if r != (Result{}) || writes(t, api, Assignments.Bucket) != before { - t.Errorf("a writer started again over unchanged records did %+v", r) - } -} // **Another writer in between is no lost value** (design 54 §2, compare-and-set): a write refused because the // key moved reads it again, takes the revision when it already holds the value, and otherwise writes again. -func TestNatsAWriteRefusedByCompareAndSetReadsAgain(t *testing.T) { - js, api := aMirrorBus(t) - ctx := t.Context() - w := aWriter(t, js, api, Assignments) - if _, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil { - t.Fatal(err) - } - // Behind its back, as a former lease holder would. - if _, err := w.KV.Put(ctx, "anchor.gitea", []byte(`{"a":9}`)); err != nil { - t.Fatal(err) - } - r, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`)}) - if err != nil { - t.Fatal(err) - } - if r.Written != 1 { - t.Errorf("a refused write was not made again: %+v", r) - } - e, err := w.KV.Get(ctx, "anchor.gitea") - if err != nil || string(e.Value()) != `{"a":2}` { - t.Fatalf("the bucket holds %v, %v", e, err) - } -} // **A value written behind the writer's back is put right on the next pass** (ADR 0289 §10): a former lease // holder's write differs from the store, and the store's value is written again, said as one change. -func TestNatsAForeignWriteIsCorrectedOnTheNextPass(t *testing.T) { - js, api := aMirrorBus(t) - ctx := t.Context() - events := listen(t, js.Conn(), "mesh.seat."+broker.ControllerSeat+".event."+Assignments.Event) - w := aWriter(t, js, api, Assignments) - store := map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)} - if _, err := w.Sync(ctx, store); err != nil { - t.Fatal(err) - } - heardBefore := len(settle(events, 1)) - if _, err := w.KV.Put(ctx, "anchor.gitea", []byte(`{"a":9}`)); err != nil { - t.Fatal(err) - } - r, err := w.Sync(ctx, store) - if err != nil { - t.Fatal(err) - } - if r != (Result{Written: 1}) { - t.Errorf("a pass after a foreign write did %+v", r) - } - e, err := w.KV.Get(ctx, "anchor.gitea") - if err != nil || string(e.Value()) != `{"a":1}` { - t.Fatalf("the bucket still holds %v, %v", e, err) - } - if said := settle(events, heardBefore+1)[heardBefore:]; len(said) != 1 || said[0].Change != Changed { - t.Errorf("the correction was said as %+v", said) - } - // And put right, it stays: the next pass writes nothing. - if r, err := w.Sync(ctx, store); err != nil || r != (Result{}) { - t.Errorf("a pass after the correction did %+v, %v", r, err) - } -} // **A writer without the lease writes nothing** (ADR 0289 §9, design 54 §2): not a new key, not a change, not // a removal, and it says so. -func TestNatsAWriterWithoutTheLeaseWritesNothing(t *testing.T) { - js, api := aMirrorBus(t) - ctx := t.Context() - held := aWriter(t, js, api, Assignments) - if _, err := held.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil { - t.Fatal(err) - } - before := writes(t, api, Assignments.Bucket) - for _, holds := range []func() bool{nil, func() bool { return false }} { - w := aWriter(t, js, api, Assignments) - w.Holds = holds - r, err := w.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":2}`), "anchor.forgejo": []byte(`{"a":1}`)}) - if !errors.Is(err, ErrNotHeld) || r != (Result{}) { - t.Errorf("a writer without the lease did %+v, %v", r, err) - } - r, err = w.Sync(ctx, map[string][]byte{}) - if !errors.Is(err, ErrNotHeld) || r != (Result{}) { - t.Errorf("a writer without the lease removed: %+v, %v", r, err) - } - } - // And one that loses it mid-pass stops before its next write. - asked := 0 - w := aWriter(t, js, api, Assignments) - w.Holds = func() bool { asked++; return asked <= 2 } - r, err := w.Sync(ctx, map[string][]byte{"anchor.a": []byte(`{}`), "anchor.b": []byte(`{}`), "anchor.gitea": []byte(`{"a":1}`)}) - if !errors.Is(err, ErrNotHeld) || r.Written != 1 { - t.Errorf("a writer that lost the lease mid-pass did %+v, %v", r, err) - } - if got := writes(t, api, Assignments.Bucket) - before; got != 1 { - t.Errorf("writers without the lease wrote the bucket %d times", got) - } -} // **A removal asks for the lease too** (ADR 0289 §9): a writer that held the lease when its pass began and lost // it before its first removal purges nothing, and says it does not hold the lease. -func TestNatsARemovalWithoutTheLeaseRemovesNothing(t *testing.T) { - js, api := aMirrorBus(t) - ctx := t.Context() - held := aWriter(t, js, api, Assignments) - if _, err := held.Sync(ctx, map[string][]byte{"anchor.gitea": []byte(`{"a":1}`)}); err != nil { - t.Fatal(err) - } - before := writes(t, api, Assignments.Bucket) - asked := 0 - w := aWriter(t, js, api, Assignments) - w.Holds = func() bool { asked++; return asked == 1 } - r, err := w.Sync(ctx, map[string][]byte{}) - if !errors.Is(err, ErrNotHeld) || r != (Result{}) || asked != 2 { - t.Errorf("a removal without the lease did %+v, %v (the lease asked %d times)", r, err, asked) - } - if got := writes(t, api, Assignments.Bucket) - before; got != 0 { - t.Errorf("a removal without the lease wrote the bucket %d times", got) - } - if _, err := w.KV.Get(ctx, "anchor.gitea"); err != nil { - t.Errorf("the key was removed without the lease: %v", err) - } -} // **A key the bus cannot hold is refused alone**, and the others are written. func TestAKeyTheBusCannotHoldIsRefusedAlone(t *testing.T) { diff --git a/internal/recordmirror/provisioning_live_test.go b/internal/recordmirror/provisioning_live_test.go index d80f4bc9..52f30171 100644 --- a/internal/recordmirror/provisioning_live_test.go +++ b/internal/recordmirror/provisioning_live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package recordmirror import ( diff --git a/internal/store/live_test.go b/internal/store/live_test.go index a03a03ae..9fc67ba4 100644 --- a/internal/store/live_test.go +++ b/internal/store/live_test.go @@ -1,3 +1,5 @@ +//go:build integration + package store import ( diff --git a/internal/testbus/testbus_integration_test.go b/internal/testbus/testbus_integration_test.go new file mode 100644 index 00000000..f189891f --- /dev/null +++ b/internal/testbus/testbus_integration_test.go @@ -0,0 +1,45 @@ +//go:build integration + +package testbus + +import ( + "strings" + "testing" + + "github.com/nats-io/nats.go" +) + +func TestEveryTestHasABusOfItsOwn(t *testing.T) { + seen := map[string]bool{} + for i := 0; i < 2; i++ { + t.Run("one", func(t *testing.T) { + url := URL(t) + if seen[url] { + t.Fatalf("two tests were handed %s", url) + } + seen[url] = true + conn, err := nats.Connect(url) + if err != nil { + t.Fatal(err) + } + defer conn.Close() + js, err := conn.JetStream() + if err != nil { + t.Fatal(err) + } + if _, err := js.StreamInfo("EVENTS"); err == nil { + t.Fatal("a stream an earlier test made is here") + } + if _, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.>"}}); err != nil { + t.Fatal(err) + } + }) + } + // Asked twice, one test reaches one bus. + if a, b := URL(t), URL(t); a != b { + t.Errorf("one test was handed two buses: %s and %s", a, b) + } + if v := strings.Count(Version, "."); v != 2 { + t.Errorf("the server's release reads %q", Version) + } +} diff --git a/internal/testbus/testbus_test.go b/internal/testbus/testbus_test.go index 5c261b9e..c851d459 100644 --- a/internal/testbus/testbus_test.go +++ b/internal/testbus/testbus_test.go @@ -9,8 +9,6 @@ import ( "encoding/json" - "github.com/nats-io/nats.go" - "github.com/novox/mesh-controller/internal/beside" ) @@ -59,37 +57,3 @@ func TestTheBusTheTestsRunIsTheBusTheMeshRuns(t *testing.T) { } // A test's bus is its own: what one test makes, the next never sees. -func TestEveryTestHasABusOfItsOwn(t *testing.T) { - seen := map[string]bool{} - for i := 0; i < 2; i++ { - t.Run("one", func(t *testing.T) { - url := URL(t) - if seen[url] { - t.Fatalf("two tests were handed %s", url) - } - seen[url] = true - conn, err := nats.Connect(url) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - js, err := conn.JetStream() - if err != nil { - t.Fatal(err) - } - if _, err := js.StreamInfo("EVENTS"); err == nil { - t.Fatal("a stream an earlier test made is here") - } - if _, err := js.AddStream(&nats.StreamConfig{Name: "EVENTS", Subjects: []string{"mesh.>"}}); err != nil { - t.Fatal(err) - } - }) - } - // Asked twice, one test reaches one bus. - if a, b := URL(t), URL(t); a != b { - t.Errorf("one test was handed two buses: %s and %s", a, b) - } - if v := strings.Count(Version, "."); v != 2 { - t.Errorf("the server's release reads %q", Version) - } -} diff --git a/merge-check.sh b/merge-check.sh index 9eb92ba4..4c5f0a52 100755 --- a/merge-check.sh +++ b/merge-check.sh @@ -27,12 +27,12 @@ set -eu export GOFLAGS=-mod=vendor GOPROXY=off # **The unit level's bound, in seconds** (issue 895): the slowest package's own test time of the untagged run, the -# compile not counted. It is NOT enforced yet: while this script is main's it judges the pull requests that put the -# `integration` tag on the tests, and on main the untagged run still holds them (about a minute), so the level's time -# is said and not judged. The release that tags the tests sets this to a number of seconds (and then to 1, how-we-build -# section 5: the whole unit suite under a second); with a number set, a run over it fails the check and names the -# slowest tests. -unit_bound= +# compile not counted. A run over it fails the check and names the slowest tests. It is 5 s since the tests carry the +# `integration` tag (design 47 milestone 3): the slowest packages, cmd/mesh-controller and internal/catalogue, measured +# 2.6-2.8 s on a quiet machine in about 2000 small tests each, and the build node runs the store tests beside this run +# and other work besides, so twice the measure. How-we-build section 5 asks 1 s for the whole unit suite: issue 1055 +# (a catalogue loaded once per package run, fewer source-scanning tests, the timed run started alone). +unit_bound=5 # judge_store_tests says each listed test of one package as `go test -v` reported it, and counts in $missing each # that did not pass: $1 the package, $2 go test's exit status, $3 the file holding its output, then the names. diff --git a/modules/build-agent/module.json b/modules/build-agent/module.json index f9b789af..4d74764d 100644 --- a/modules/build-agent/module.json +++ b/modules/build-agent/module.json @@ -47,7 +47,7 @@ "id": "judge-cache", "path": "${dir:judge-cache}", "class": "cache", - "why": "the Go cache every merge check's judge is built with and the release-branch judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)" + "why": "the Go cache every merge check's judge is built with and the group judges kept per head; mounted into judge builds only, never into a check's containers (novox/hq ADR 0358 §6)" } ] }, diff --git a/testdata/store-tests b/testdata/store-tests index 9666cc0d..589272ff 100644 --- a/testdata/store-tests +++ b/testdata/store-tests @@ -126,8 +126,6 @@ ./cmd/mesh-controller TestReplay867 ./cmd/mesh-controller TestReplay867Recheck ./cmd/mesh-controller TestTheRecheckVerbAsksMeshDeliveryOnABusOfItsOwn -./cmd/mesh-controller TestAGroupsCompositionIntoAReleaseBranchIsJudgedByItsOwnController -./cmd/mesh-controller TestAGroupsCompositionJudgedWithTheValidatorIntoAReleaseBranchIsJudgedByItsOwnController ./cmd/mesh-controller TestAssigningTheClientIssuesItsAccount ./cmd/mesh-controller TestAnUnrelatedBatchWalksBesideAnOpenWalk ./cmd/mesh-controller TestARelatedBatchWaitsBehindTheOpenWalk