diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 1aa17cb..da18fc1 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -331,11 +331,12 @@ func runApply(ctx context.Context, opts options, d *declaration.Declaration, sou return err } - report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, apply.ExecRunner, func(line string) { - if !opts.json { - fmt.Println(line) - } - }) + report, updated, applyErr := apply.Apply(ctx, sys, d, known, store.OriginCarried, + apply.ExecRunner, func(line string) { + if !opts.json { + fmt.Println(line) + } + }, sealOpener(opts.state)) // Saved whichever way it went. Recording only on success would lose the footprint of a // failed apply, and that footprint is on the machine either way. @@ -449,6 +450,15 @@ func enrol(ctx context.Context, opts options) error { } fmt.Printf("generated this node's overlay key: %s\n", mine.Overlay.Public) + // And the key secrets are sealed to. Here, with the others, because the mesh cannot seal + // anything to a key it has not been told about — a key made later would leave a node that + // looks enrolled and can receive no credential. + sealing, err := identity.GenerateSealingKey() + if err != nil { + return err + } + fmt.Printf("generated this node's sealing key: %s\n", sealing.Public) + // What this machine can be asked to do, gathered before joining rather than after. The // control plane cannot decide what a node should run without it, so it travels with the // request instead of being asked for in a second round trip. @@ -459,7 +469,7 @@ func enrol(ctx context.Context, opts options) error { } reply, err := link.Enrol(ctx, token.Broker, token.Fingerprint, *name, token.Secret, - mine.Public, mine.Overlay.Public, reported, opts.timeout) + mine.Public, mine.Overlay.Public, sealing.Public, reported, opts.timeout) if err != nil { return err } @@ -504,6 +514,10 @@ func enrol(ctx context.Context, opts options) error { []byte(mine.Overlay.Private+"\n"), 0o600); err != nil { return fmt.Errorf("cannot write this node's overlay key: %w", err) } + if err := os.WriteFile(identity.SealingKeyPath(opts.state), + []byte(sealing.Private+"\n"), 0o600); err != nil { + return fmt.Errorf("cannot write this node's sealing key: %w", err) + } fmt.Printf("\nenrolled as %s\n", reply.Node) fmt.Printf(" identity %s\n", identityPath) @@ -649,7 +663,7 @@ func applyAndKeep(ctx context.Context, opts options, raw []byte, signed *store.D // Declared, not carried. A declaration from the mesh removes only what the mesh previously // declared — never what this machine raised for itself from its bundle (04-ISSUES/010). outcome, updated, applyErr := apply.Apply(ctx, built, declared, known, store.OriginDeclared, - apply.ExecRunner, nil) + apply.ExecRunner, nil, sealOpener(opts.state)) // Saved whichever way it went. Recording only on success would lose the footprint of a // failed apply, and that footprint is on the machine either way. @@ -709,3 +723,17 @@ func waitForEnrolment(ctx context.Context, opts options) (identity.Identity, err } } } + +// sealOpener is how a sealed file is opened. +// +// Looked up per file rather than held, because most declarations contain no sealed file at all +// and a node with no key must fail on the one that needs it rather than on every apply. +func sealOpener(statePath string) apply.Unseal { + return func(sealed string) ([]byte, error) { + key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath)) + if err != nil { + return nil, err + } + return key.Unseal(sealed) + } +} diff --git a/go.mod b/go.mod index 8ae86a4..c3444a4 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,9 @@ module github.com/novox/mesh-host -go 1.24 +go 1.25.0 -require github.com/rabbitmq/amqp091-go v1.14.0 // indirect +require ( + github.com/rabbitmq/amqp091-go v1.14.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/sys v0.47.0 // indirect +) diff --git a/go.sum b/go.sum index c9d50b5..661ae93 100644 --- a/go.sum +++ b/go.sum @@ -1,2 +1,6 @@ github.com/rabbitmq/amqp091-go v1.14.0 h1:RSaT7aOKt/OrkVUyswPDW29lnRz9psuGmfZFBmLqLek= github.com/rabbitmq/amqp091-go v1.14.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/internal/apply/apply.go b/internal/apply/apply.go index 46a19a2..b3f6a36 100644 --- a/internal/apply/apply.go +++ b/internal/apply/apply.go @@ -98,6 +98,7 @@ func Apply( origin string, run Runner, log func(string), + unseal Unseal, ) (Report, store.State, error) { if log == nil { log = func(string) {} @@ -129,7 +130,7 @@ func Apply( for _, resource := range d.Resources { was, _ := known.Find(resource.Identity()) - outcome, err := applyOne(ctx, sys, resource, run, changed, was) + outcome, err := applyOne(ctx, sys, resource, run, changed, was, unseal) if err != nil { return report, known, &Error{Resource: resource.Identity(), Err: err, Done: report} } @@ -150,13 +151,17 @@ func Apply( return report, known, nil } +// Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which +// makes every sealed file an error rather than a silently skipped one. +type Unseal func(sealed string) ([]byte, error) + func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner, - changed map[string]bool, previous store.Applied) (Outcome, error) { + changed map[string]bool, previous store.Applied, unseal Unseal) (Outcome, error) { switch res := r.(type) { case *declaration.Directory: return applyDirectory(res) case *declaration.File: - return applyFile(res, previous) + return applyFile(res, previous, unseal) case *declaration.Service: return applyService(ctx, sys, res, run, changed) case *declaration.Package: @@ -239,10 +244,32 @@ func applyDirectory(r *declaration.Directory) (Outcome, error) { return out, nil } -func applyFile(r *declaration.File, previous store.Applied) (Outcome, error) { +func applyFile(r *declaration.File, previous store.Applied, unseal Unseal) (Outcome, error) { out := begin(r) - out.wrote = digestOf(r.Content) - mode, err := modeOf(r.Mode, 0o644) + + // What actually goes on disk. For a sealed file the mesh never had this, and neither did + // whatever carried the declaration here. + content := r.Content + // A secret written world-readable is a secret. The default differs from an ordinary file's + // for that reason alone; an explicit mode still wins, because a module may need its own user + // to read it and only the module knows which. + fallback := os.FileMode(0o644) + if r.Secret() { + fallback = 0o600 + if unseal == nil { + // Refused rather than skipped. A machine that quietly does not apply the one resource + // carrying a credential is a machine that looks configured and cannot connect. + return out, fmt.Errorf( + "%s is sealed to this node and this node has no sealing key", r.Path) + } + opened, err := unseal(r.Sealed) + if err != nil { + return out, fmt.Errorf("cannot open %s: %w", r.Path, err) + } + content = string(opened) + } + out.wrote = digestOf(content) + mode, err := modeOf(r.Mode, fallback) if err != nil { return out, err } @@ -260,7 +287,7 @@ func applyFile(r *declaration.File, previous store.Applied) (Outcome, error) { } } - contentSame := existed && string(existing) == r.Content + contentSame := existed && string(existing) == content // Whether the machine still holds what this host last put there. When it does not, and the // declaration has not changed either, somebody edited it — and saying so is the whole @@ -272,7 +299,7 @@ func applyFile(r *declaration.File, previous store.Applied) (Outcome, error) { if err := os.MkdirAll(filepath.Dir(r.Path), 0o755); err != nil { return out, err } - if err := writeAtomically(r.Path, []byte(r.Content), mode); err != nil { + if err := writeAtomically(r.Path, []byte(content), mode); err != nil { return out, err } } else if !modeSame { @@ -286,7 +313,7 @@ func applyFile(r *declaration.File, previous store.Applied) (Outcome, error) { if err != nil { return out, fmt.Errorf("wrote %s and cannot read it back: %w", r.Path, err) } - if string(written) != r.Content { + if string(written) != content { return out, fmt.Errorf("%s does not contain what was declared after writing it", r.Path) } info, err := os.Stat(r.Path) diff --git a/internal/apply/apply_test.go b/internal/apply/apply_test.go index c2c0d03..3415849 100644 --- a/internal/apply/apply_test.go +++ b/internal/apply/apply_test.go @@ -40,7 +40,7 @@ func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) { {"id":"f","type":"file","path":"`+dir+`/etc/a.conf","content":"hello\n","mode":"0640"} ]}`) - first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil) + first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -48,7 +48,7 @@ func TestApplyingTwiceChangesNothingTheSecondTime(t *testing.T) { t.Fatal("the first apply on an empty machine changed nothing") } - second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil) + second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -66,7 +66,7 @@ func TestADriftedMachineIsReturned(t *testing.T) { {"id":"f","type":"file","path":"`+path+`","content":"correct\n","mode":"0644"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -74,7 +74,7 @@ func TestADriftedMachineIsReturned(t *testing.T) { t.Fatal(err) } - report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil) + report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -98,7 +98,7 @@ func TestADroppedResourceIsRemoved(t *testing.T) { {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"}, {"id":"drop","type":"file","path":"`+drop+`","content":"b\n"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil) + _, state, err := Apply(context.Background(), archHost(t), both, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -106,7 +106,7 @@ func TestADroppedResourceIsRemoved(t *testing.T) { one := parse(t, `{"declaration":1,"resources":[ {"id":"keep","type":"file","path":"`+keep+`","content":"a\n"} ]}`) - report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil) + report, state, err := Apply(context.Background(), archHost(t), one, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -138,7 +138,7 @@ func TestNothingTheHostDidNotCreateIsTouched(t *testing.T) { d := parse(t, `{"declaration":1,"resources":[ {"id":"ours","type":"file","path":"`+filepath.Join(dir, "ours.conf")+`","content":"a\n"} ]}`) - if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil); err != nil { + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil); err != nil { t.Fatal(err) } @@ -157,7 +157,7 @@ func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) { before := parse(t, `{"declaration":1,"resources":[ {"id":"old","type":"file","path":"`+path+`","content":"old\n"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil) + _, state, err := Apply(context.Background(), archHost(t), before, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -165,7 +165,7 @@ func TestARenameToTheSamePathDoesNotDeleteTheNewFile(t *testing.T) { after := parse(t, `{"declaration":1,"resources":[ {"id":"new","type":"file","path":"`+path+`","content":"new\n"} ]}`) - if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil); err != nil { + if _, _, err := Apply(context.Background(), archHost(t), after, state, store.OriginCarried, noServices, nil, nil); err != nil { t.Fatal(err) } @@ -193,7 +193,7 @@ func TestAFailedStepFailsTheApply(t *testing.T) { {"id":"never","type":"file","path":"`+filepath.Join(dir, "never.conf")+`","content":"b\n"} ]}`) - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err == nil { t.Fatal("an impossible resource did not fail the apply") } @@ -226,7 +226,7 @@ func TestNothingIsRecordedUntilItWorked(t *testing.T) { {"id":"doomed","type":"directory","path":"`+blocker+`"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err == nil { t.Fatal("expected a failure") } @@ -245,7 +245,7 @@ func TestAModeIsMaintainedNotJustSet(t *testing.T) { {"id":"f","type":"file","path":"`+path+`","content":"s\n","mode":"0600"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -253,7 +253,7 @@ func TestAModeIsMaintainedNotJustSet(t *testing.T) { t.Fatal(err) } - report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil) + report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -284,7 +284,7 @@ func TestAServiceIsReadBackNotAssumed(t *testing.T) { d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"doomed.service","state":"running"} ]}`) - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a service that died immediately was reported as running") } @@ -300,7 +300,7 @@ func TestAnUnknownServiceStateIsRefusedNotGuessed(t *testing.T) { d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"odd.service","state":"running"} ]}`) - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil || !strings.Contains(err.Error(), "neither running nor stopped") { t.Errorf("an unrecognised service state was not refused: %v", err) } @@ -324,7 +324,7 @@ func TestADroppedServiceIsStoppedNotDeleted(t *testing.T) { {"id":"other","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) - if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil); err != nil { + if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } joined := strings.Join(commands, "; ") @@ -350,7 +350,7 @@ func TestAUnitThatDoesNotExistIsNotStopped(t *testing.T) { {"id":"s","type":"service","unit":"never-installed.service","state":"stopped"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, absent, nil, nil) if err == nil { t.Fatal("a unit that does not exist was reported as satisfactorily stopped") } @@ -371,7 +371,7 @@ func TestAMaskedUnitIsRefused(t *testing.T) { d := parse(t, `{"declaration":1,"resources":[ {"id":"s","type":"service","unit":"masked.service","state":"running"} ]}`) - if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil); err == nil { + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, masked, nil, nil); err == nil { t.Fatal("a masked unit was accepted") } } @@ -399,7 +399,7 @@ func TestForgettingAUnitThatIsGoneDoesNotStrandTheNode(t *testing.T) { {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) - report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil) + report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("a vanished unit stranded the apply: %v", err) } @@ -443,7 +443,7 @@ func TestABrokenPackageDatabaseIsNotReadAsNotInstalled(t *testing.T) { {"id":"rt","type":"package","package":"docker"} ]}`) - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a broken package database was read as 'not installed'") } @@ -464,7 +464,7 @@ func TestAnInstalledPackageIsNotReinstalled(t *testing.T) { {"id":"rt","type":"package","package":"docker"} ]}`) - report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } @@ -494,7 +494,7 @@ func TestAPackageIsNeverUninstalled(t *testing.T) { {"id":"f","type":"file","path":"`+filepath.Join(t.TempDir(), "a")+`","content":"a\n"} ]}`) - report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil) + report, state, err := Apply(context.Background(), archHost(t), d, known, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("dropping a package stranded the apply: %v", err) } @@ -524,7 +524,7 @@ func TestAnActionThatIsAlreadyTrueDoesNotRun(t *testing.T) { {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} ]}`) - report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } @@ -550,7 +550,7 @@ func TestAnActionThatSucceedsAndDoesNothingFails(t *testing.T) { {"id":"db","type":"action","command":["create-db","mesh"],"verify":["has-db","mesh"]} ]}`) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("an action that reported success and did nothing was accepted") } @@ -577,7 +577,7 @@ func TestAnActionRunsInsideTheContainerItNames(t *testing.T) { {"id":"db","type":"action","in":"store","command":["createdb","mesh"],"verify":["psql","-lqt"]} ]}`) - if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil); err != nil { + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil); err != nil { t.Fatalf("apply failed: %v", err) } if !sawExec { @@ -604,7 +604,7 @@ func TestAContainerThatExitsImmediatelyFailsTheApply(t *testing.T) { {"id":"store","type":"container","name":"store","image":"`+pinned+`"} ]}`) - _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a container that exited immediately was reported as applied") } @@ -646,7 +646,7 @@ func TestAContainerWhoseDeclarationChangedIsReplaced(t *testing.T) { return "", nil } - report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } @@ -676,7 +676,7 @@ func TestAContainerThatMatchesIsLeftAlone(t *testing.T) { return "", nil } - report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } @@ -736,7 +736,7 @@ func TestAServiceIsEnabledAtBootWhenAsked(t *testing.T) { ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, - systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil) + systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } @@ -756,7 +756,7 @@ func TestBootIsEnabledBeforeTheUnitIsStarted(t *testing.T) { {"id":"rt","type":"service","unit":"docker.service","state":"running","boot":"enabled"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, - systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil); err != nil { + systemctlStub(t, "loaded", "inactive", "disabled", &verbs), nil, nil); err != nil { t.Fatal(err) } if len(verbs) < 2 || verbs[0] != "enable" { @@ -771,7 +771,7 @@ func TestAlreadyEnabledAndRunningIsUnchanged(t *testing.T) { ]}`) report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, - systemctlStub(t, "loaded", "active", "enabled", &verbs), nil) + systemctlStub(t, "loaded", "active", "enabled", &verbs), nil, nil) if err != nil { t.Fatalf("apply failed: %v", err) } @@ -791,7 +791,7 @@ func TestOmittingBootLeavesItAlone(t *testing.T) { {"id":"rt","type":"service","unit":"docker.service","state":"running"} ]}`) if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, - systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil); err != nil { + systemctlStub(t, "loaded", "inactive", "enabled", &verbs), nil, nil); err != nil { t.Fatal(err) } for _, v := range verbs { @@ -811,7 +811,7 @@ func TestAStaticUnitCannotBeEnabled(t *testing.T) { ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, - systemctlStub(t, "loaded", "active", "static", &verbs), nil) + systemctlStub(t, "loaded", "active", "static", &verbs), nil, nil) if err == nil { t.Fatal("a static unit was accepted as enable-able") } @@ -826,7 +826,7 @@ func TestAnUnknownBootStateIsRefusedNotGuessed(t *testing.T) { {"id":"rt","type":"service","unit":"x.service","state":"running","boot":"enabled"} ]}`) _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, - systemctlStub(t, "loaded", "active", "indirect", &verbs), nil) + systemctlStub(t, "loaded", "active", "indirect", &verbs), nil, nil) if err == nil { t.Fatal("an unrecognised boot state was guessed at instead of refused") } @@ -901,7 +901,7 @@ func TestAContainerUsesTheRuntimeTheMachineHas(t *testing.T) { // It will fail at read-back — the stub never reports it running — and what matters is // WHICH binary it used getting there. - _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, _, _ = Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) for _, c := range calledWith { if c != "podman" { @@ -923,7 +923,7 @@ func TestNoRuntimeIsSaidPlainly(t *testing.T) { {"id":"store","type":"container","name":"store","image":"`+pinned+`"} ]}`) - _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginCarried, run, nil, nil) if err == nil { t.Fatal("a machine with no container runtime applied a container") } @@ -964,14 +964,14 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) { run := recordingServices(&commands) if _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, - store.OriginCarried, run, nil); err != nil { + store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } else { // Second apply with the same content: nothing moved, so nothing restarts. A machine that // restarted its services on every reconcile would never be steady. commands = nil if _, _, err := Apply(context.Background(), archHost(t), d, state, - store.OriginCarried, run, nil); err != nil { + store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } for _, c := range commands { @@ -987,7 +987,7 @@ func TestAServiceIsRestartedWhenWhatItReflectsChanges(t *testing.T) { ]}`, path)) commands = nil if _, _, err := Apply(context.Background(), archHost(t), changedDecl, state, - store.OriginCarried, run, nil); err != nil { + store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } var stopped, started bool @@ -1021,7 +1021,7 @@ func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) { var commands []string run := recordingServices(&commands) _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, - store.OriginCarried, run, nil) + store.OriginCarried, run, nil, nil) if err != nil { t.Fatal(err) } @@ -1033,7 +1033,7 @@ func TestAServiceIsNotRestartedByAChangeItDoesNotName(t *testing.T) { ]}`, conf, other)) commands = nil if _, _, err := Apply(context.Background(), archHost(t), second, state, - store.OriginCarried, run, nil); err != nil { + store.OriginCarried, run, nil, nil); err != nil { t.Fatal(err) } for _, c := range commands { @@ -1072,7 +1072,7 @@ func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) { ]}`, path)) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, - store.OriginCarried, noServices, nil) + store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -1083,7 +1083,7 @@ func TestAFileChangedOnTheMachineIsCorrectedAndSaidSo(t *testing.T) { } report, state, err := Apply(context.Background(), archHost(t), d, state, - store.OriginCarried, noServices, nil) + store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -1115,12 +1115,12 @@ func TestTheMeshChangingItsMindIsNotDrift(t *testing.T) { ]}`, path)) _, state, err := Apply(context.Background(), archHost(t), first, store.State{}, - store.OriginCarried, noServices, nil) + store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), archHost(t), second, state, - store.OriginCarried, noServices, nil) + store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } @@ -1138,12 +1138,12 @@ func TestAnUntouchedFileIsStillUnchanged(t *testing.T) { ]}`, path)) _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, - store.OriginCarried, noServices, nil) + store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } report, _, err := Apply(context.Background(), archHost(t), d, state, - store.OriginCarried, noServices, nil) + store.OriginCarried, noServices, nil, nil) if err != nil { t.Fatal(err) } diff --git a/internal/apply/sealed_test.go b/internal/apply/sealed_test.go new file mode 100644 index 0000000..8f6db5a --- /dev/null +++ b/internal/apply/sealed_test.go @@ -0,0 +1,187 @@ +package apply + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + + "github.com/novox/mesh-host/internal/declaration" + "github.com/novox/mesh-host/internal/identity" + "github.com/novox/mesh-host/internal/store" +) + +// A file the mesh delivers without being able to read. +// +// Everything else in a declaration is visible to whatever carried it: the message is signed, so +// it cannot be forged, and signing does not make it unreadable. A password in `content` is a +// password the broker sees — the transitive trust this design refuses everywhere else. + +func sealedTo(t *testing.T, key identity.SealingKey, value string) string { + t.Helper() + sealed, err := identity.Seal(key.Public, []byte(value)) + if err != nil { + t.Fatal(err) + } + return sealed +} + +func opener(key identity.SealingKey) Unseal { + return func(sealed string) ([]byte, error) { return key.Unseal(sealed) } +} + +func sealedFile(t *testing.T, path, sealed string) *declaration.Declaration { + t.Helper() + raw := map[string]any{"declaration": 1, "resources": []map[string]any{ + {"id": "creds", "type": "file", "path": path, "sealed": sealed}, + }} + body, _ := json.Marshal(raw) + d, err := declaration.Parse(body) + if err != nil { + t.Fatal(err) + } + return d +} + +func TestASealedFileIsOpenedAndWritten(t *testing.T) { + key, err := identity.GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, key, `{"password":"hunter2"}`)) + + report, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + if !report.Changed() { + t.Fatal("nothing changed") + } + on, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(on) != `{"password":"hunter2"}` { + t.Fatalf("the file holds %q", on) + } +} + +func TestASecretIsNotWorldReadableByDefault(t *testing.T) { + // An ordinary file defaults to 0644, which for a credential is the whole problem. The default + // differs because the consequence differs; an explicit mode still wins, since a module may + // need its own user to read it and only the module knows which. + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, key, "secret")) + if _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("a credential landed mode %o", info.Mode().Perm()) + } +} + +func TestSomethingSealedToAnotherNodeIsRefused(t *testing.T) { + // Refused, not skipped, and refused before anything is written. A machine that quietly does + // not apply the one resource carrying a credential looks configured and cannot connect. + mine, _ := identity.GenerateSealingKey() + theirs, _ := identity.GenerateSealingKey() + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, theirs, "not for you")) + + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(mine)) + if err == nil { + t.Fatal("a file sealed to another node was applied") + } + if _, statErr := os.Stat(path); statErr == nil { + t.Fatal("something was written before the failure") + } +} + +func TestANodeWithNoSealingKeyRefusesRatherThanSkipping(t *testing.T) { + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + d := sealedFile(t, dir+"/db.json", sealedTo(t, key, "secret")) + _, _, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, nil) + if err == nil { + t.Fatal("a sealed file was skipped by a node that cannot open one") + } + if !strings.Contains(err.Error(), "sealing key") { + t.Fatalf("the failure does not say why: %v", err) + } +} + +func TestTheSecretIsNeverInWhatTheMeshIsToldBack(t *testing.T) { + // The node reports what it applied, and that report goes over the same broker the sealing was + // for. A digest is a fact about the file; the file is not. + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + d := sealedFile(t, dir+"/db.json", sealedTo(t, key, "hunter2")) + report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + said, _ := json.Marshal(report) + kept, _ := json.Marshal(state) + for what, blob := range map[string][]byte{"the report": said, "the node's state": kept} { + if strings.Contains(string(blob), "hunter2") { + t.Fatalf("%s carries the secret in plain text:\n%s", what, blob) + } + } +} + +func TestASealedFileStillNoticesAHandEdit(t *testing.T) { + // Drift detection must survive not holding the plaintext. It does, because what is recorded + // is a digest of what was written rather than what was written. + key, _ := identity.GenerateSealingKey() + dir := t.TempDir() + path := dir + "/db.json" + d := sealedFile(t, path, sealedTo(t, key, "hunter2")) + _, state, err := Apply(context.Background(), archHost(t), d, store.State{}, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("meddled"), 0o600); err != nil { + t.Fatal(err) + } + again, _, err := Apply(context.Background(), archHost(t), d, state, + store.OriginCarried, noServices, nil, opener(key)) + if err != nil { + t.Fatal(err) + } + if !again.Changed() { + t.Fatal("a hand-edited credential was left as it was found") + } + on, _ := os.ReadFile(path) + if string(on) != "hunter2" { + t.Fatalf("it was not put back: %q", on) + } +} + +func TestContentAndSealedTogetherIsRefused(t *testing.T) { + // Otherwise nobody can tell by looking whether what landed on the machine was the secret or + // the placeholder. + _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[ + {"id":"f","type":"file","path":"/etc/x","content":"a","sealed":"b"}]}`)) + if err == nil { + t.Fatal("a file that is both literal and sealed was accepted") + } + if !strings.Contains(err.Error(), "not both") { + t.Fatalf("unhelpful refusal: %v", err) + } +} diff --git a/internal/declaration/declaration.go b/internal/declaration/declaration.go index 2c79fd2..41aad56 100644 --- a/internal/declaration/declaration.go +++ b/internal/declaration/declaration.go @@ -83,8 +83,25 @@ type File struct { Path string `json:"path"` Content string `json:"content"` Mode string `json:"mode,omitempty"` + + // Sealed is content encrypted to this node's sealing key, for a file the mesh must deliver + // without being able to read. + // + // The one thing here the host cannot simply write. Everything else in a declaration is + // visible to whatever carried it — the broker relays the message, and the message is signed + // so it cannot be forged, but signing does not make it unreadable. A password travelling in + // `content` would be a password the broker sees, which is the transitive trust the design + // refuses everywhere else (novox/hq ADR 0004). + // + // Exclusive with Content: a file is one or the other, so that "was this secret" is answerable + // by looking rather than by knowing which field won. + Sealed string `json:"sealed,omitempty"` } +// Secret reports whether this file arrived sealed, which is what decides both that it must be +// opened before writing and that its contents must never appear in a report. +func (f *File) Secret() bool { return f.Sealed != "" } + func (f *File) Identity() string { return f.ID } func (f *File) Kind() Type { return TypeFile } func (f *File) Target() string { return f.Path } @@ -94,6 +111,11 @@ func (f *File) validate(where string, _ bool) []string { if f.Path == "" { problems = append(problems, where+": a file needs a path") } + if f.Content != "" && f.Sealed != "" { + problems = append(problems, where+ + ": a file has content or is sealed, not both — otherwise nobody can tell by looking "+ + "whether what landed on the machine was the secret or the placeholder") + } return append(problems, checkMode(where, f.Mode)...) } diff --git a/internal/identity/identity_test.go b/internal/identity/identity_test.go index 9f4c86a..762faca 100644 --- a/internal/identity/identity_test.go +++ b/internal/identity/identity_test.go @@ -311,3 +311,69 @@ func TestAnIdentityThatCannotBeReadIsNotReportedAsAbsent(t *testing.T) { t.Errorf("the error does not say why this is different from having none: %v", err) } } + +func TestASealingKeyOpensOnlyWhatWasSealedToIt(t *testing.T) { + mine, err := GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + theirs, err := GenerateSealingKey() + if err != nil { + t.Fatal(err) + } + sealed, err := Seal(mine.Public, []byte("hunter2")) + if err != nil { + t.Fatal(err) + } + got, err := mine.Unseal(sealed) + if err != nil { + t.Fatal(err) + } + if string(got) != "hunter2" { + t.Fatalf("got %q", got) + } + if _, err := theirs.Unseal(sealed); err == nil { + t.Fatal("another node opened it") + } +} + +func TestSealingTheSameValueTwiceLooksDifferent(t *testing.T) { + // Sealed boxes are randomised, so an observer cannot tell that two nodes were given the same + // password, nor that a rotation changed nothing. Worth asserting because the alternative is + // a subtle leak nobody would look for. + key, _ := GenerateSealingKey() + first, _ := Seal(key.Public, []byte("same")) + second, _ := Seal(key.Public, []byte("same")) + if first == second { + t.Fatal("sealing is deterministic, so equal secrets are visible as equal blobs") + } +} + +func TestANodeWithNoSealingKeySaysWhatToDo(t *testing.T) { + // Rather than making one. A key the mesh was never told about is a key nothing can be sealed + // to, so a node that quietly created one would look fine and receive nothing for ever. + _, err := LoadSealingKey(t.TempDir() + "/absent.key") + if err == nil { + t.Fatal("a sealing key appeared out of nowhere") + } + if !strings.Contains(err.Error(), "join again") { + t.Fatalf("the failure does not say what to do: %v", err) + } +} + +func TestASealingKeyOnDiskSurvivesATrailingNewline(t *testing.T) { + // It is written with one, the way every other key file here is, and reading it back has to + // cope — otherwise the key works until the first restart. + key, _ := GenerateSealingKey() + path := t.TempDir() + "/sealing.key" + if err := os.WriteFile(path, []byte(key.Private+"\n"), 0o600); err != nil { + t.Fatal(err) + } + back, err := LoadSealingKey(path) + if err != nil { + t.Fatal(err) + } + if back.Public != key.Public { + t.Fatalf("a round trip through the disk changed the key") + } +} diff --git a/internal/identity/sealing.go b/internal/identity/sealing.go new file mode 100644 index 0000000..218f8bc --- /dev/null +++ b/internal/identity/sealing.go @@ -0,0 +1,143 @@ +package identity + +import ( + "crypto/ecdh" + "crypto/rand" + "encoding/base64" + "fmt" + "os" + "strings" + + "golang.org/x/crypto/nacl/box" +) + +// The key a secret is sealed to, so the mesh can carry one without ever holding a usable copy. +// +// **The fault this exists to avoid is documented, in another mesh, in its own tooling.** There, +// credentials live in the control plane's database, encrypted at rest — which protects against +// somebody reading the database file and nothing else. The same secret is also in each node's +// environment file in plain text, and, worse, inside every connection string composed from it, so +// the tool for finding copies has to search *by value* rather than by name. Its own documentation +// says the copies inside composed URLs "are often the only copies actually in use". Encryption at +// rest also cost the ability to audit: a query against the encrypted column returns zero rows and +// proves nothing. +// +// So the arrangement here is the other one. **The node generates this key and the mesh only ever +// sees the public half**, exactly as with the identity and overlay keys +// (novox/hq ADR 0004). A secret is sealed to that public half before it is stored, so: +// +// - the control plane's database holds nothing usable, and a copy of it grants nothing +// - the broker relays a blob it cannot read, which is the point of not trusting it +// - *compromise of a node is compromise of that node* becomes true of secrets too, rather +// than being true of identity and quietly false of everything that matters +// +// A third key rather than reusing one of the two that exist. The identity key signs and is +// Ed25519; the overlay key is WireGuard's and is tied to being on the private network, which a +// machine may not be. A key used for two purposes is one rotation away from breaking the other. + +// SealingKey is an X25519 keypair used only for receiving secrets. +type SealingKey struct { + // Public is what the mesh records. + Public string `json:"public"` + // Private never leaves this machine. + Private string `json:"private"` +} + +// GenerateSealingKey makes this node's key for receiving secrets. +func GenerateSealingKey() (SealingKey, error) { + private, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return SealingKey{}, fmt.Errorf("cannot generate this node's sealing key: %w", err) + } + return SealingKey{ + Public: base64.StdEncoding.EncodeToString(private.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(private.Bytes()), + }, nil +} + +// SealingKeyPath is where the private half lives. +func SealingKeyPath(statePath string) string { + return dirOf(statePath) + "/sealing.key" +} + +// LoadSealingKey reads this node's sealing key. +// +// It does not make one. A key the mesh has never been told about is a key nothing can be sealed +// to, so creating one here would produce a node that silently cannot receive any secret and looks +// fine — the key is generated at enrolment, where its public half is reported in the same breath. +func LoadSealingKey(path string) (SealingKey, error) { + raw, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return SealingKey{}, fmt.Errorf( + "this node has no sealing key at %s, so nothing can be sealed to it — it is made "+ + "at enrolment, and a node that joined before secrets existed must join again", + path) + } + return SealingKey{}, err + } + { + private, decodeErr := base64.StdEncoding.DecodeString(strings.TrimSpace(string(raw))) + if decodeErr != nil || len(private) != 32 { + return SealingKey{}, fmt.Errorf( + "%s is not a sealing key; move it aside to have a new one made", path) + } + key, keyErr := ecdh.X25519().NewPrivateKey(private) + if keyErr != nil { + return SealingKey{}, fmt.Errorf("%s is not a usable sealing key: %w", path, keyErr) + } + return SealingKey{ + Public: base64.StdEncoding.EncodeToString(key.PublicKey().Bytes()), + Private: base64.StdEncoding.EncodeToString(key.Bytes()), + }, nil + } +} + +// Unseal opens something the mesh sealed to this node. +// +// Anonymous sealed boxes: the sender is not authenticated here, and does not need to be. What a +// node applies is bounded by the declaration's signature, which is checked before any of this — +// so a blob that arrives in a verified declaration came from the mesh, and this only has to +// answer whether it was meant for this machine. +func (s SealingKey) Unseal(sealed string) ([]byte, error) { + blob, err := base64.StdEncoding.DecodeString(sealed) + if err != nil { + return nil, fmt.Errorf("this is not a sealed value: %w", err) + } + private, err := base64.StdEncoding.DecodeString(s.Private) + if err != nil || len(private) != 32 { + return nil, fmt.Errorf("this node's sealing key is unusable") + } + public, err := base64.StdEncoding.DecodeString(s.Public) + if err != nil || len(public) != 32 { + return nil, fmt.Errorf("this node's sealing key is unusable") + } + + var pub, priv [32]byte + copy(pub[:], public) + copy(priv[:], private) + out, ok := box.OpenAnonymous(nil, blob, &pub, &priv) + if !ok { + // Sealed to a different node, or to a key this one no longer has. Said as one thing + // because from here they are indistinguishable, and both mean the same: this machine + // cannot read it and applying it would write a file of rubbish. + return nil, fmt.Errorf("this was not sealed to this node's current key") + } + return out, nil +} + +// Seal closes a value to a node's public sealing key. Here so that a test can produce what the +// mesh produces, rather than asserting against a blob nobody can regenerate. +func Seal(publicKey string, value []byte) (string, error) { + public, err := base64.StdEncoding.DecodeString(publicKey) + if err != nil || len(public) != 32 { + return "", fmt.Errorf("%q is not a sealing key", publicKey) + } + var pub [32]byte + copy(pub[:], public) + sealed, err := box.SealAnonymous(nil, value, &pub, rand.Reader) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(sealed), nil +} diff --git a/internal/link/enrol.go b/internal/link/enrol.go index cb64a60..35a1d21 100644 --- a/internal/link/enrol.go +++ b/internal/link/enrol.go @@ -35,6 +35,11 @@ type EnrolRequest struct { // and unreachable for a round trip, which is the state everything else here works to avoid. OverlayKey string `json:"overlay_key,omitempty"` + // SealingKey is the public half of the key secrets are sealed to. A third key, and the + // reasoning is the same one twice over: the mesh must be able to send this node something + // nothing else can read, and it must never be able to read it either. + SealingKey string `json:"sealing_key,omitempty"` + Profile map[string]any `json:"profile,omitempty"` } @@ -65,7 +70,7 @@ var ErrRefused = errors.New("the mesh refused this enrolment") // says once it is in, and the secret travels again because the control plane must not have to ask // the broker who connected. func Enrol(ctx context.Context, address, pin, node, secret string, public []byte, - overlayKey string, profile map[string]any, timeout time.Duration) (EnrolReply, error) { + overlayKey, sealingKey string, profile map[string]any, timeout time.Duration) (EnrolReply, error) { config, err := PinnedConfig(pin) if err != nil { @@ -111,7 +116,7 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte } request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, - OverlayKey: overlayKey, Profile: profile} + OverlayKey: overlayKey, SealingKey: sealingKey, Profile: profile} body, err := json.Marshal(request) if err != nil { return EnrolReply{}, err