diff --git a/go/catalogue/bus_snapshot_test.go b/go/catalogue/bus_snapshot_test.go index e767d48..5f8edb3 100644 --- a/go/catalogue/bus_snapshot_test.go +++ b/go/catalogue/bus_snapshot_test.go @@ -1,6 +1,7 @@ package catalogue import ( + "encoding/json" "os" "path/filepath" "strings" @@ -27,14 +28,28 @@ func TestTheBussAccountSaysNothingOnTheBus(t *testing.T) { "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, "emits":["bus.slow-consumer"],"tools":["bus_streams"], "build":{"artifacts":[{"name":"tools","kind":"bundle","language":"go","system":"arch","from":"cmd/t","binary":"t"}]}}`) - if _, err := ParseManifest(bundled); err != nil { + m, err := ParseManifest(bundled) + if err != nil { t.Fatalf("a bus module whose tools bundle declares what it emits was refused: %v", err) } + // And so does its built manifest, its bundle in Bundles and its build read away (issue 774: nats's built manifest + // was refused, and with it every plan of the control-node). + built, err := m.Resolve([]Built{{Name: "tools", Kind: ArtifactBundle, Reference: "store/tools:1", Digest: "sha256:1"}}) + if err != nil { + t.Fatal(err) + } + raw, err = json.Marshal(built) + if err != nil { + t.Fatal(err) + } + if _, err := ParseManifestForInputs(raw); err != nil { + t.Fatalf("the built manifest of a bus module whose tools bundle declares what it emits was refused: %v", err) + } quiet := []byte(`{"module":"bus","version":"1","provides":[{"name":"mesh-bus","scope":"mesh"}], "claims":[{"name":"mesh-broker","scope":"mesh"}],"own-secrets":{"broker":"/run/broker"}, "tools":["bus_streams"]}`) - m, err := ParseManifest(quiet) + m, err = ParseManifest(quiet) if err != nil { t.Fatalf("a bus module with an account and tools was refused: %v", err) } diff --git a/go/catalogue/manifest.go b/go/catalogue/manifest.go index 04e86b8..5db0e07 100644 --- a/go/catalogue/manifest.go +++ b/go/catalogue/manifest.go @@ -3378,6 +3378,11 @@ func whileStoppedProblems(m Manifest, r map[string]any, scheduled bool) []string // shipsABundle says the module builds a tools bundle, which the node's runtime carries and speaks for. func (m Manifest) shipsABundle() bool { + // A built manifest says its bundles in Bundles, its build read away (Resolve; issue 774: nats's built manifest was + // refused for the emits its tools bundle says). + if len(m.Bundles) > 0 { + return true + } if m.Build == nil { return false } diff --git a/go/catalogue/secrets_into_files.go b/go/catalogue/secrets_into_files.go index 0ed07c3..7baca99 100644 --- a/go/catalogue/secrets_into_files.go +++ b/go/catalogue/secrets_into_files.go @@ -59,6 +59,10 @@ func sealedFor(m Manifest, needs []Needed, with Rendering) (map[string]string, e for name := range m.OwnSecrets.Plain() { if value := with.Needed[m.Module][name]; value != "" { sealed[name] = value + } else if with.Foreseen[m.Module][name] { + // A stand-in where its own file has one (the Foreseeing and Planning modes, issue 774): a file naming it + // composes as its own file does, never refused for a value no such composition reads. + sealed[name] = ForeseenSealed } } for _, to := range m.SecretRequirements() { diff --git a/go/catalogue/secrets_into_files_test.go b/go/catalogue/secrets_into_files_test.go index 1a92a7e..f8d9bb3 100644 --- a/go/catalogue/secrets_into_files_test.go +++ b/go/catalogue/secrets_into_files_test.go @@ -21,34 +21,42 @@ func fileNamed(out []map[string]any, id string) map[string]any { return nil } -// A module's own credential, put into a file it wrote. +// A module's own credential, put into a file it wrote: the value sealed to the node, or, in a composition asked ahead of +// the send or for a plan alone, the stand-in its own file has (issue 774: baserow's config.json refused ace's plans). func TestAFileGetsTheSecretItsContentAsksFor(t *testing.T) { - r := Resolution{Node: "anchor", Modules: []Manifest{{ - Module: "gitea", - OwnSecrets: OwnSecrets{"admin": {Path: "/var/lib/gitea/admin.env"}}, - Resources: []map[string]any{{ - "id": "conf", "type": "file", "path": "/etc/gitea/app.ini", - "content": "[security]\nSECRET_KEY = ${secret:admin}\n", - }}, - }}} - out, err := r.Declaration(Rendering{ - Needed: map[string]map[string]string{"gitea": {"admin": "sealed-admin"}}, - }) - if err != nil { - t.Fatal(err) - } - file := fileNamed(out, "gitea.conf") - if file == nil { - t.Fatalf("no file in %v", out) - } - got, _ := file["secrets"].(map[string]any) - if got["admin"] != "sealed-admin" { - t.Errorf("the file was sent without its secret: %v", file) - } - // The hole is still a hole on the wire. The mesh does not fill it in; the host does, on the - // machine, which is the only place both halves exist. - if !strings.Contains(file["content"].(string), "${secret:admin}") { - t.Errorf("the mesh substituted it itself: %v", file["content"]) + for _, row := range []struct { + name string + with Rendering + want string + }{ + {"sealed", Rendering{Needed: map[string]map[string]string{"gitea": {"admin": "sealed-admin"}}}, "sealed-admin"}, + {"foreseen", Rendering{Foreseen: map[string]map[string]bool{"gitea": {"admin": true}}}, ForeseenSealed}, + } { + r := Resolution{Node: "anchor", Modules: []Manifest{{ + Module: "gitea", + OwnSecrets: OwnSecrets{"admin": {Path: "/var/lib/gitea/admin.env"}}, + Resources: []map[string]any{{ + "id": "conf", "type": "file", "path": "/etc/gitea/app.ini", + "content": "[security]\nSECRET_KEY = ${secret:admin}\n", + }}, + }}} + out, err := r.Declaration(row.with) + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + file := fileNamed(out, "gitea.conf") + if file == nil { + t.Fatalf("%s: no file in %v", row.name, out) + } + got, _ := file["secrets"].(map[string]any) + if got["admin"] != row.want { + t.Errorf("%s: the file was sent without its secret: %v", row.name, file) + } + // The hole is still a hole on the wire. The mesh does not fill it in; the host does, on the + // machine, which is the only place both halves exist. + if !strings.Contains(file["content"].(string), "${secret:admin}") { + t.Errorf("%s: the mesh substituted it itself: %v", row.name, file["content"]) + } } } diff --git a/go/compose/frombus/frombus.go b/go/compose/frombus/frombus.go index 2fedc4c..ac67876 100644 --- a/go/compose/frombus/frombus.go +++ b/go/compose/frombus/frombus.go @@ -439,6 +439,68 @@ func Plans(ctx context.Context, r Reader, mesh Mesh, s Scope) (map[string]catalo return PlansOf(in, s.Node) } +// PlansHeld is Plans composed module by module (issue 774: one bad module must not drop its node): a module placed on +// the scope's node whose build cannot be read or taken in, or whose plan refuses the node's, is held, by the one rule +// Compose follows (composeEach, culprit): composed as its stub so what it provides and requires still shapes the rest, +// named in held with why, and none of its plan in plans. Only a refusal that is no single module's refuses the node, +// err. The seat mesh-provisioner's holder composes so: a node with one bad manifest still has every other module's plan. +func PlansHeld(ctx context.Context, r Reader, mesh Mesh, s Scope) (plans map[string]catalogue.Plan, held map[string]string, err error) { + held = map[string]string{} + in, err := input(ctx, r, mesh, s, func(module string, err error) { held[module] = err.Error() }) + if err != nil { + return nil, held, err + } + plans, err = plansHeldOf(in, s.Node, held) + return plans, held, err +} + +// plansHeldOf is the plans of node over in, every module in held held, and each module a refusal is of held too until +// the plans compose (culprit's rule over the Planning mode: its stub first, then its bare stub). +func plansHeldOf(in compose.Input, node string, held map[string]string) (map[string]catalogue.Plan, error) { + for _, m := range sortedKeys(held) { + in = stubbed(in, node, m, "") + } + compose1 := func(in compose.Input) (map[string]catalogue.Plan, error) { return plansIn(in, node, compose.Planning) } + // Each try holds one module more or leaves out all of one held module's offer: bounded by twice the modules placed. + for tries := 0; tries <= 2*len(assignedOn(in, node))+1; tries++ { + plans, err := compose1(in) + if err == nil { + for m := range held { + delete(plans, m) + } + return plans, nil + } + var order []string + for _, m := range assignedOn(in, node) { + if strings.Contains(err.Error(), m) && len(order) < maxCulprits { + order = append(order, m) + } + } + module, leave := "", "" + find: + for _, l := range []string{"", leaveAll} { + for _, m := range order { + if _, done := held[m]; done && l == "" { + continue + } + if _, again := compose1(stubbed(in, node, m, l)); again == nil || again.Error() != err.Error() { + module, leave = m, l + break find + } + } + } + if module == "" { + return nil, err + } + if _, done := held[module]; !done { + held[module] = err.Error() + } + in = stubbed(in, node, module, leave) + } + return nil, fmt.Errorf("frombus: the plans of %s do not compose with %d module(s) held: %s", node, len(held), + strings.Join(sortedKeys(held), ", ")) +} + // PlansOf is Plans over an input already filled, rendered in the Planning mode (compose.Planning): every own secret the // input holds none for stands in, so no value and no record of one is needed. func PlansOf(in compose.Input, node string) (map[string]catalogue.Plan, error) { diff --git a/go/compose/frombus/frombustest/frombustest_test.go b/go/compose/frombus/frombustest/frombustest_test.go index edb8388..476bb1d 100644 --- a/go/compose/frombus/frombustest/frombustest_test.go +++ b/go/compose/frombus/frombustest/frombustest_test.go @@ -2,25 +2,75 @@ package frombustest import ( "context" + "strings" "testing" "git.novox.be/novox/mesh-sdk/go/compose/frombus" + "git.novox.be/novox/mesh-sdk/go/delivery" ) // ADR 0570 clause 2 (issue 1954; a3's H5): the in-memory mesh composes a plan from the bus that hands restic to the -// provisioner and names its postgres-database, provided by postgres on n1. +// provisioner and names its postgres-database, provided by postgres on n1. And one bad module must not drop its node +// (issue 774): with a module placed beside it whose built manifest cannot be used, or whose plan refuses, PlansHeld +// still hands restic's plan and names the bad one with why, where Plans refuses the whole node. func TestTheProvisionedMeshHandsAPlanWithItsProvision(t *testing.T) { - b, err := Provisioned() - if err != nil { - t.Fatal(err) - } - plans, err := frombus.Plans(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}) - if err != nil { - t.Fatal(err) - } - restic, handed := plans["restic"] - if !handed || len(restic.Provisions) != 1 || restic.Provisions[0].Provision != "postgres-database" || - restic.Provisions[0].Provider != Node || restic.Provisions[0].ProviderModule != "postgres" { - t.Fatalf("the plans are %+v", plans) + fp := "src2:" + strings.Repeat("9", 64) + for _, row := range []struct { + name string + bad string // the indexer's manifest of a module "backup" placed on n1; empty places none + // built is the manifest backup's done build names; empty gives it no build + built string + why string // words backup's hold says + }{ + {name: "every module composes"}, + {name: "a built manifest that cannot be used", + bad: `{"module":"backup","version":"1","build":{"artifacts":[{"name":"files","kind":"archive","from":"files"}]}}`, + built: `{"module":"backup","version":"1","requires":["amqp"]}`, why: "backup"}, + {name: "a plan that refuses", + bad: `{"module":"backup","version":"1","resources":[{"id":"conf","type":"file","path":"/etc/backup.conf",` + + `"mode":"0600","content":"key=${secret:nope}"}]}`, why: "${secret:nope}"}, + } { + b, err := Provisioned() + if err != nil { + t.Fatal(err) + } + if row.bad != "" { + p := b.Placement[Node] + p.Modules = append(p.Modules, delivery.PlacedModule{Module: "backup"}) + b.Placement[Node] = p + digest := delivery.ManifestDigest([]byte(row.bad)) + b.Manifests[digest] = []byte(row.bad) + b.Module["backup"] = delivery.Module{FP2: fp, Manifest: digest} + if row.built != "" { + builtDigest := delivery.ManifestDigest([]byte(row.built)) + b.Manifests[builtDigest] = []byte(row.built) + b.Builds["backup "+fp] = &delivery.BuilderBuild{State: delivery.BuildDone, Manifest: builtDigest, + Artifacts: []delivery.BuiltArtifact{{Name: "files", Kind: "archive", Reference: "artifact-store://backup/files/blobs/x", Digest: "sha256:x"}}} + } + if _, err := frombus.Plans(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}); err == nil { + t.Errorf("%s: Plans composed the node; the row does not refuse it", row.name) + } + } + plans, held, err := frombus.PlansHeld(context.Background(), b, frombus.Mesh{}, frombus.Scope{Node: Node}) + if err != nil { + t.Fatalf("%s: %v", row.name, err) + } + restic, handed := plans["restic"] + if !handed || len(restic.Provisions) != 1 || restic.Provisions[0].Provision != "postgres-database" || + restic.Provisions[0].Provider != Node || restic.Provisions[0].ProviderModule != "postgres" { + t.Fatalf("%s: the plans are %+v (held %v)", row.name, plans, held) + } + if row.bad == "" { + if len(held) > 0 { + t.Errorf("%s: held %v", row.name, held) + } + continue + } + if why, ok := held["backup"]; !ok || !strings.Contains(why, row.why) || len(held) != 1 { + t.Errorf("%s: held %v, want backup alone, saying %q", row.name, held, row.why) + } + if _, planned := plans["backup"]; planned { + t.Errorf("%s: the held module has a plan", row.name) + } } }