diff --git a/cmd/mesh-host/filling.go b/cmd/mesh-host/filling.go index 5257e3e..373aef2 100644 --- a/cmd/mesh-host/filling.go +++ b/cmd/mesh-host/filling.go @@ -3,22 +3,25 @@ package main import ( "context" "crypto/ed25519" + "encoding/json" "errors" "fmt" "path/filepath" "sync/atomic" "git.novox.be/novox/mesh-sdk/go/declaration" + "git.novox.be/novox/mesh-sdk/go/provisions" "github.com/novox/mesh-host/internal/apply" "github.com/novox/mesh-host/internal/filling" + "github.com/novox/mesh-host/internal/link" "github.com/novox/mesh-host/internal/store" "github.com/novox/mesh-host/internal/system" ) // serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq // ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying. -func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) { - e := fillingEngine(opts, signer, say) +func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, queue *link.Session, node string, say func(string)) { + e := fillingEngine(opts, signer, provisionSecretsOn(queue, node, opts.state), say) fillingNow.Store(e) defer fillingNow.CompareAndSwap(e, nil) if err := filling.Serve(ctx, filling.Socket, e, say); err != nil { @@ -50,7 +53,10 @@ func knockFilling(after string, say func(string)) { // fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under // the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file // written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is. -func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine { +// +// A provision's `${secret:}` the client left in the content is filled first (secrets, novox/hq ADR 0570 clause 6), and a +// file so filled is written 0600 when the declaration gives no mode. +func fillingEngine(opts options, signer ed25519.PublicKey, secrets provisionSecrets, say func(string)) *filling.Engine { return &filling.Engine{ Declared: func() (*declaration.Declaration, error) { return declaredNow(opts.state, signer) @@ -74,8 +80,13 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi if err != nil { return "", "", err } - report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner, - announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state))) + content, secret, err := secrets.fill(ctx, filledModule(d, path), content) + if err != nil { + return "", "", fmt.Errorf("%s was not written: %w", path, err) + } + report, updated, writeErr := apply.WriteFilledWith(ctx, built, d, known, path, content, apply.ExecRunner, + announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)), + apply.Filled{Secret: secret}) // Saved whichever way it went, as an apply's: what was written is on the machine either way. if err := store.Save(opts.state, updated); err != nil { return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+ @@ -94,3 +105,39 @@ func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *fi }, } } + +// filledModule is the module whose file the declaration declares at path, or "" when none or several do. +func filledModule(d *declaration.Declaration, path string) string { + module := "" + for _, r := range d.Resources { + if f, ok := r.(*declaration.File); ok && f.Path == path { + if module != "" { + return "" + } + module, _ = d.ModuleOfID(f.ID) + } + } + return module +} + +// provisionSecretsOn reads provisions and asks the vault over the link, and opens with this machine's sealing key. +func provisionSecretsOn(queue *link.Session, node, statePath string) provisionSecrets { + p := provisionSecrets{node: node, open: sealOpener(statePath)} + if queue == nil { + return p + } + p.read = func(ctx context.Context, key string) ([]byte, bool, error) { + h, err := queue.KVHistory(ctx, provisions.ProvisionsBucket, key) + if err != nil { + return nil, false, err + } + if len(h) == 0 || h[len(h)-1].Deleted { + return nil, false, nil + } + return h[len(h)-1].Value, true, nil + } + p.own = func(ctx context.Context, args map[string]any) (json.RawMessage, error) { + return queue.AskVault(ctx, "own", args) + } + return p +} diff --git a/cmd/mesh-host/filling_test.go b/cmd/mesh-host/filling_test.go index 828682a..a25cc55 100644 --- a/cmd/mesh-host/filling_test.go +++ b/cmd/mesh-host/filling_test.go @@ -10,6 +10,8 @@ import ( "testing" "time" + "git.novox.be/novox/mesh-sdk/go/provisions" + "github.com/novox/mesh-host/internal/filling" "github.com/novox/mesh-host/internal/store" ) @@ -45,7 +47,28 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil { t.Fatal(err) } - e := fillingEngine(opts, public, func(string) {}) + // The provision m reads, as mesh-provisioner keeps it, and the vault's value of it, sealed to this machine (faked: + // the opener reads the box's "v"). + provision := map[string]string{} + secrets := provisionSecrets{node: "n1", + read: func(_ context.Context, key string) ([]byte, bool, error) { + v, ok := provision[key] + return []byte(v), ok, nil + }, + own: func(_ context.Context, args map[string]any) (json.RawMessage, error) { + raw, _ := json.Marshal(args) + var a struct{ Credentials []map[string]any } + _ = json.Unmarshal(raw, &a) + c := a.Credentials[0] + c["generation"], c["sealed"] = 2, map[string]string{"v": "s3cret"} + return json.Marshal(map[string]any{"node": "n1", "credentials": []any{c}}) + }, + open: func(sealed string) ([]byte, error) { + var b struct{ V string } + err := json.Unmarshal([]byte(sealed), &b) + return []byte(b.V), err + }} + e := fillingEngine(opts, public, secrets, func(string) {}) uid := uint32(os.Getuid()) if err := e.Accepts(uid); err != nil { t.Fatal(err) @@ -102,4 +125,52 @@ func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) { if a := <-done; a.Action != "updated" { t.Fatalf("the second write: %+v", a) } + + // ADR 0570 clause 6: the client leaves a provision's `${secret:}` to the node-engine, which fills it at the write + // from the vault's value of a provided provision, and writes the file 0600; a provision not provided refuses it. + keyed := func(state string, ref bool) string { + p := provisions.Provision{Provision: "db", Consumer: provisions.End{Module: "m", Node: "n1"}, + Provider: provisions.End{Module: "postgres", Node: "n2"}, State: state, At: time.Now()} + if state != provisions.Provided { + p.Why = "the provider has not answered" + } + if ref { + p.Reference = &provisions.Reference{ID: "provision/postgres/db", Generation: 2} + } + raw, err := provisions.Encode(p) + if err != nil { + t.Fatal(err) + } + return string(raw) + } + for _, tc := range []struct { + name, state string + ref bool + content string + want string // the file after; "" keeps what was there + mode os.FileMode + }{ + {name: "a provision not provided is refused", state: provisions.Asked, content: "password=${secret:db}\n"}, + {name: "provided with no reference is refused", state: provisions.Provided, content: "password=${secret:db}\n"}, + {name: "provided: filled at the write, 0600", state: provisions.Provided, ref: true, + content: "password=${secret:db}\n", want: "password=s3cret\n", mode: 0o600}, + {name: "no placeholder is written as it came, as the declaration says", content: "plain\n", want: "plain\n", mode: 0o644}, + } { + before, _ := os.ReadFile(path) + if tc.state != "" { + provision["n1.m.db"] = keyed(tc.state, tc.ref) + } + a := e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte(tc.content)}) + got, _ := os.ReadFile(path) + if tc.want == "" { + if a.Failed == "" && a.Refused == "" || string(got) != string(before) { + t.Errorf("%s: answered %+v, the file holds %q", tc.name, a, got) + } + continue + } + info, _ := os.Stat(path) + if a.Failed != "" || a.Refused != "" || string(got) != tc.want || info.Mode().Perm() != tc.mode { + t.Errorf("%s: answered %+v, the file holds %q, mode %v", tc.name, a, got, info.Mode()) + } + } } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index bcff6e6..8e52318 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -1200,7 +1200,7 @@ func runLink(ctx context.Context, opts options) error { go serveMeshCLI(aside, queue, say) // And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR // 0349 §4). - go serveFilling(aside, opts, mine.Membership.Signer, say) + go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say) held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx)) // **Cleanly**, or the launcher counts standing aside as a crash and rolls the new host back diff --git a/cmd/mesh-host/provision_secrets.go b/cmd/mesh-host/provision_secrets.go new file mode 100644 index 0000000..e9c32f7 --- /dev/null +++ b/cmd/mesh-host/provision_secrets.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "strings" + + "git.novox.be/novox/mesh-sdk/go/provisions" +) + +// provisionSecrets fills a provision's value into a file the provisioner's client hands the node-engine to write +// (novox/hq ADR 0570 clause 6): the client fills the file but for each `${secret:}`, and the node-engine +// fills those at the write, so the client never holds the value. A whole `secrets` file of a handed consumer is the +// placeholder alone. +// +// For each placeholder it reads the provision's key `..` of the seat mesh-provisioner's state +// `provisions` (clause 1), which must be `provided` and carry the vault's reference; asks the vault for that pair +// credential sealed to this machine's key (the verb `own`, ADR 0349 §11 with the node-engine standing for the client, +// ADR 0550 §5); and opens it with this machine's sealing key. Anything short of that refuses the write, naming the +// provision and why: a file written with a placeholder left in it, or with no value, is a module that starts and cannot +// connect. +type provisionSecrets struct { + node string + // read is the provision's key from mesh-provisioner's `provisions`: its value and whether it is there. + read func(ctx context.Context, key string) ([]byte, bool, error) + // own asks the vault's verb `own` in this engine's name. + own func(ctx context.Context, args map[string]any) (json.RawMessage, error) + // open opens a value sealed to this machine. + open func(sealed string) ([]byte, error) +} + +// secretPlaceholder is `${secret:}`, or `${secret:.}` where the consumer keeps several values +// of one provision (ADR 0094). +var secretPlaceholder = regexp.MustCompile(`\$\{secret:([A-Za-z0-9_-]+)(?:\.([A-Za-z0-9_-]+))?\}`) + +// fill is content with each provision placeholder of module's filled, and whether any was. +func (p provisionSecrets) fill(ctx context.Context, module string, content []byte) ([]byte, bool, error) { + found := secretPlaceholder.FindAllSubmatch(content, -1) + if len(found) == 0 { + return content, false, nil + } + if module == "" { + return nil, false, fmt.Errorf("the file names %d provision secret(s) and belongs to no module, so whose they are is not known", len(found)) + } + out := string(content) + done := map[string]bool{} + for _, f := range found { + whole, provision, local := string(f[0]), string(f[1]), string(f[2]) + if done[whole] { + continue + } + done[whole] = true + value, err := p.value(ctx, module, provision, local) + if err != nil { + return nil, false, err + } + out = strings.ReplaceAll(out, whole, string(value)) + } + return []byte(out), true, nil +} + +// value is one provision's value for module on this node, opened. +func (p provisionSecrets) value(ctx context.Context, module, provision, local string) ([]byte, error) { + key, err := provisions.Key(p.node, module, provision, local) + if err != nil { + return nil, err + } + if p.read == nil || p.own == nil || p.open == nil { + return nil, fmt.Errorf("the provision %s cannot be read on this machine: the node-engine is not linked to the mesh", key) + } + raw, ok, err := p.read(ctx, key) + switch { + case err != nil: + return nil, fmt.Errorf("the provision %s cannot be read: %w", key, err) + case !ok: + return nil, fmt.Errorf("the provision %s is not in %s: nothing asks for it", key, provisions.ProvisionsRead) + } + pr, err := provisions.Decode(raw) + if err != nil { + return nil, fmt.Errorf("the provision %s: %w", key, err) + } + if pr.State != provisions.Provided || pr.Reference == nil { + why := pr.State + if pr.Why != "" { + why += ": " + pr.Why + } + if pr.State == provisions.Provided { + why = "provided with no reference to the vault's value" + } + return nil, fmt.Errorf("the provision %s is %s, so its value is not written yet", key, why) + } + type asked struct { + Role string `json:"role"` + Provision string `json:"provision"` + Consumer string `json:"consumer"` + ConsumerModule string `json:"consumer_module"` + Provider string `json:"provider"` + Local string `json:"local,omitempty"` + } + ask := asked{Role: "consumer", Provision: provision, Consumer: p.node, ConsumerModule: module, Provider: pr.Provider.Node, Local: local} + answer, err := p.own(ctx, map[string]any{"credentials": []asked{ask}}) + if err != nil { + return nil, fmt.Errorf("the vault's value of %s: %w", key, err) + } + var a struct { + Node string `json:"node"` + Credentials []struct { + asked + Generation int `json:"generation"` + Sealed json.RawMessage `json:"sealed"` + } `json:"credentials"` + } + if err := json.Unmarshal(answer, &a); err != nil { + return nil, fmt.Errorf("the vault's answer for %s is not one: %w", key, err) + } + if a.Node != p.node { + return nil, fmt.Errorf("the vault answered %s's credentials for %s on %s", a.Node, key, p.node) + } + for _, c := range a.Credentials { + if c.asked != ask { + continue + } + if c.Generation < pr.Reference.Generation { + return nil, fmt.Errorf("the vault holds %s at generation %d and the provision names %d: not written until it has it", + key, c.Generation, pr.Reference.Generation) + } + value, err := p.open(string(c.Sealed)) + if err != nil { + return nil, fmt.Errorf("the vault's value of %s does not open on this machine: %w", key, err) + } + return value, nil + } + return nil, fmt.Errorf("the vault answered no value of %s for this machine", key) +} diff --git a/go.mod b/go.mod index 60fe963..675bca2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index f53fc84..3c95de2 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635 h1:ImOyj4WqH9lwxOr95aH9Wq2iJkUwdtnEyPrASm9TrYA= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029211750-2e5cc4f16635/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368 h1:LaBxsyU3HeTIPKYmvpJuw+Wm2plpf6O7qHO91O+mAi0= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261029213215-4a967933b368/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/apply/filled.go b/internal/apply/filled.go index 5d4d2ca..6bd9b7a 100644 --- a/internal/apply/filled.go +++ b/internal/apply/filled.go @@ -45,6 +45,30 @@ func WriteFilled( log func(string), keep Keep, windows *Windows, +) (report Report, _ store.State, _ error) { + return WriteFilledWith(ctx, sys, d, known, path, content, run, log, keep, windows, Filled{}) +} + +// Filled is what the node-engine did to a filled file's content before its write. +type Filled struct { + // Secret says the node-engine filled a value into the content, a provision's `${secret:}` (novox/hq ADR 0570 + // clause 6): the file is then written 0600 when the declaration gives no mode, as a file carrying a secret is. + Secret bool +} + +// WriteFilledWith is WriteFilled for content the node-engine filled before the write (Filled). +func WriteFilledWith( + ctx context.Context, + sys system.System, + d *declaration.Declaration, + known store.State, + path string, + content []byte, + run Runner, + log func(string), + keep Keep, + windows *Windows, + filled Filled, ) (report Report, _ store.State, _ error) { if log == nil { log = func(string) {} @@ -65,7 +89,7 @@ func WriteFilled( return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+ "is written for exactly one, so nothing was written", len(declared), path)} } - file := filledAs(declared[0], content) + file := filledAs(declared[0], content, filled.Secret) if held, isHeld := known.HeldAt(file.ID); isHeld { return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+ "yet taken (%s), so nothing was written", path, held.Why)} @@ -163,10 +187,10 @@ func WriteFilled( } // filledAs is the declared file with the filled content in place of the declaration's. -func filledAs(declared *declaration.File, content []byte) *declaration.File { +func filledAs(declared *declaration.File, content []byte, secret bool) *declaration.File { f := *declared f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil - if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) { + if f.Mode == "" && (secret || declared.Secret() || len(declared.Secrets) > 0) { // Written as the declaration would have written it: a file carrying a credential is not world-readable. f.Mode = "0600" }