Feature 57 (config and provisioning): release mesh-host, the reviewed release branch on top of main
mesh/merge-gate head bff1ac14a198 merged into trunk 49c9002c04d3 beside mesh-controller#355@8c381172 — pass: builds mesh-host → ace, g14, novox, shanks; no…
mesh/repo-check head bff1ac14a198 merged into trunk 49c9002c04d3 beside mesh-controller#355@8c381172 — pass: its merge-check.sh passed
mesh/delivery delivered
mesh/delivery-group group release/57-config-and-provisioning delivered: every member is delivered

This commit is contained in:
2026-10-16 08:24:13 +00:00
parent 49c9002c04
commit bff1ac14a1
37 changed files with 3836 additions and 60 deletions
+6
View File
@@ -0,0 +1,6 @@
# Release 57: config and provisioning
This branch carries the work of hq design 57 (ADR 0348, ADR 0349) in this repository. Each package
merges here through its own reviewed pull request; the branch reaches main as one pull request, in one
delivery group with the branch of the same name in mesh-controller, mesh-host, mesh-sdk, mesh-catalog
and hq, after a rehearsal in the lab. This file is deleted by the last commit before that pull request.
+74
View File
@@ -0,0 +1,74 @@
package main
import (
"context"
"crypto/ed25519"
"errors"
"path/filepath"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/filling"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// serveFilling writes filled files on the provisioner client's request for as long as this engine serves (novox/hq
// ADR 0349 §4). A socket that cannot be served is said once, and this engine goes on applying.
func serveFilling(ctx context.Context, opts options, signer ed25519.PublicKey, say func(string)) {
if err := filling.Serve(ctx, filling.Socket, fillingEngine(opts, signer, say), say); err != nil {
say("filled files are not written on request on this machine: " + err.Error())
}
}
// fillingEngine is the write of a filled file as this engine does it: the declaration it keeps, verified, read under
// the apply's lock — the same two locks every apply takes, the process's and the one beside the state — and the file
// written through the apply (apply.WriteFilled), its outcome recorded in the node's state as an apply's is.
func fillingEngine(opts options, signer ed25519.PublicKey, say func(string)) *filling.Engine {
return &filling.Engine{
Declared: func() (*declaration.Declaration, error) {
raw, err := store.LoadDeclared(store.DeclaredPath(opts.state), signer)
if err != nil {
return nil, err
}
return declaration.Parse(raw)
},
UIDOf: filling.UIDOf,
Lock: func() (func(), error) {
applying.Lock()
unlock, err := store.Lock(opts.state, nil)
if err != nil {
applying.Unlock()
return nil, err
}
return func() { unlock(); applying.Unlock() }, nil
},
Place: func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (string, string, error) {
built, err := system.For(builtFor)
if err != nil {
return "", "", err
}
known, err := store.Load(opts.state)
if err != nil {
return "", "", err
}
report, updated, writeErr := apply.WriteFilled(ctx, built, d, known, path, content, apply.ExecRunner,
announceOr(say), apply.KeepIn(filepath.Dir(opts.state)), apply.WindowsIn(filepath.Dir(opts.state)))
// Saved whichever way it went, as an apply's: what was written is on the machine either way.
if err := store.Save(opts.state, updated); err != nil {
return "", "", errors.Join(writeErr, errors.New("written, and the node's state could not be saved: "+
err.Error()))
}
action, digest := "", ""
for _, o := range report.Outcomes {
if o.Target == path && o.Type == string(declaration.TypeFile) {
action = o.Action
}
}
if rec, ok := updated.At(string(declaration.TypeFile), path); ok {
digest = rec.Wrote
}
return action, digest, writeErr
},
}
}
+105
View File
@@ -0,0 +1,105 @@
package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"os"
"os/user"
"path/filepath"
"testing"
"time"
"github.com/novox/mesh-host/internal/filling"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0349 §4 (issue 668): a write of a filled file and an apply are one at a time — the write
// takes the lock the apply holds, and writes only once the apply let it go — and the write is recorded in the
// node's state as an apply's outcome is.
func TestAWriteAndAnApplyOfTheSameFileSerialise(t *testing.T) {
was := builtFor
builtFor = "arch"
t.Cleanup(func() { builtFor = was })
me, err := user.Current()
if err != nil {
t.Fatal(err)
}
if me.Uid == "0" {
t.Skip("run as root, whose account is never the provisioner's")
}
dir := t.TempDir()
opts := options{state: filepath.Join(dir, "state.json")}
public, private, err := ed25519.GenerateKey(nil)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(dir, "m.conf")
raw, err := json.Marshal(map[string]any{"declaration": 1, "provisioner_account": me.Username,
"filled_files": []string{path}, "resources": []any{map[string]any{"id": "m.conf", "type": "file",
"path": path, "content": "from the declaration\n", "owner-at-runtime": "owned", "owner": me.Username}}})
if err != nil {
t.Fatal(err)
}
if err := store.SaveDeclared(store.DeclaredPath(opts.state),
store.Declared{Declaration: raw, Signature: ed25519.Sign(private, raw)}); err != nil {
t.Fatal(err)
}
e := fillingEngine(opts, public, func(string) {})
uid := uint32(os.Getuid())
if err := e.Accepts(uid); err != nil {
t.Fatal(err)
}
// An apply holds the lock.
applying.Lock()
done := make(chan filling.Answer)
go func() {
done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("filled\n")})
}()
select {
case a := <-done:
applying.Unlock()
t.Fatalf("written while an apply held the lock: %+v", a)
case <-time.After(200 * time.Millisecond):
}
if _, err := os.Stat(path); err == nil {
applying.Unlock()
t.Fatal("the file was written while an apply held the lock")
}
applying.Unlock()
a := <-done
if a.Refused != "" || a.Failed != "" || a.Action != "created" {
t.Fatalf("the write after the apply: %+v", a)
}
got, err := os.ReadFile(path)
if err != nil || string(got) != "filled\n" {
t.Fatalf("the file holds %q (%v)", got, err)
}
known, err := store.Load(opts.state)
if err != nil {
t.Fatal(err)
}
if rec, ok := known.Find("m.conf"); !ok || rec.Wrote != a.Digest || rec.Target != path {
t.Errorf("the write is not recorded in the node's state: %+v, answered %q", rec, a.Digest)
}
// And the cross-process lock beside the state too: a write waits for a `reconcile` run by hand.
unlock, err := store.Lock(opts.state, nil)
if err != nil {
t.Fatal(err)
}
go func() {
done <- e.Write(context.Background(), uid, filling.Request{Path: path, Content: []byte("again\n")})
}()
select {
case a := <-done:
unlock()
t.Fatalf("written while another process held the apply's lock: %+v", a)
case <-time.After(200 * time.Millisecond):
}
unlock()
if a := <-done; a.Action != "updated" {
t.Fatalf("the second write: %+v", a)
}
}
+22 -4
View File
@@ -1242,6 +1242,16 @@ func runLink(ctx context.Context, opts options) error {
Say: say,
Timeout: opts.timeout,
}
// And the move of a value sealed to this machine into the vault, asked by the controller (design 57's V3,
// issue 676): the vault is asked over this queue's link, so the queue is named once it exists.
queue.MoveSecret = moveSecretFrom(opts.state, queue)
// And this machine's sealing key stated to the vault in its own name, at start and whenever it changes (ADR 0372,
// issue 786), whether or not any value was ever moved in.
go keepNodeKeyStated(ctx, func() (identity.SealingKey, error) {
return identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
}, func(verb string, args any) (json.RawMessage, error) {
return queue.AskVault(ctx, verb, args)
}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo)
worker := make(chan struct{})
go func() {
defer close(worker)
@@ -1280,6 +1290,9 @@ func runLink(ctx context.Context, opts options) error {
// And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller
// (novox/hq ADR 0272).
go serveMeshCLI(aside, queue, say)
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
// 0349 §4).
go serveFilling(aside, opts, mine.Membership.Signer, say)
held := link.HoldRoused(aside, membership, queue, say, opts.timeout, rousedBySignal(ctx))
// The act in hand finishes and is kept before this process exits: an apply that stood aside with
@@ -1647,7 +1660,7 @@ func declaredResolvConf(d *declaration.Declaration) (string, string, bool) {
}
for _, r := range d.Resources {
f, ok := r.(*declaration.File)
if !ok || f.Path != network.ResolvConf || f.CreateOnce || f.Into != "" {
if !ok || f.Path != network.ResolvConf || f.Seeded() || f.Into != "" {
continue
}
module, ok := liveness.ModuleOf(f.ID)
@@ -1769,8 +1782,13 @@ func judgeWhatRuns(ctx context.Context, j *liveness.Judge, queue *link.Queue, sa
func healthAsReported(st liveness.Statement, ns *network.Statement) *link.Health {
// LibraryContract: this engine reads a resource's declared `health` and judges it (ADR 0240 Phase B), judges a
// user's declared `root` (novox/hq ADR 0266), and reads a declaration's identities, groups and libraries
// (novox/hq ADR 0328), which is what tells the controller it may be sent any of those fields.
h := &link.Health{Contract: link.BusyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
// (novox/hq ADR 0328), a container's busy (ADR 0335) and a file's owner-at-runtime and the filled files (ADR
// 0349 §5) and the provisioner client's account (ADR 0349 §4), which is what tells the controller it may be sent
// any of those fields; and it writes a filled file only on the client's word (HandedContract, design 57 P7),
// which is what tells it it may hand a moved module's files to the provisioner here; and it opens the vault's box
// for a core credential (VaultBoxContract, ADR 0349 §16), which is what tells it it may send that box here; and it states its sealing key to the vault (NodeKeyContract, ADR
// 0372, issue 786).
h := &link.Health{Contract: link.NodeKeyContract, At: st.At.UTC(), Resources: []link.ResourceHealth{}}
for _, r := range st.Resources {
h.Resources = append(h.Resources, link.ResourceHealth{Module: r.Module, Resource: r.ID, Kind: r.Kind,
Target: r.Target, State: r.State, Reason: r.Reason, Since: r.Since.UTC(), Streak: r.Streak,
@@ -2349,7 +2367,7 @@ func sealOpener(statePath string) apply.Unseal {
if err != nil {
return nil, err
}
return key.Unseal(sealed)
return key.Open(sealed)
}
}
+131
View File
@@ -0,0 +1,131 @@
package main
import (
"context"
"crypto/rand"
"crypto/subtle"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"os"
"time"
"golang.org/x/crypto/nacl/box"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
)
// The move of a value sealed to this machine into the vault (design 57's V3, issue 676; internal/link
// movesecret.go). The controller cannot open what it holds; this machine can open what was sealed to it, and so
// hands it to the vault, sealed to a key the vault made for the call. The value is in this process for that one
// call and written nowhere.
// vaultAsk asks one of the vault's verbs in this engine's own name and answers its result.
type vaultAsk func(verb string, args any) (json.RawMessage, error)
// moveSecretFrom is the engine's move: this machine's sealing key, read for each ask, and the vault asked over
// the link open now.
func moveSecretFrom(statePath string, queue *link.Queue) link.MoveSecretFunc {
return func(m link.MoveSecret) (link.MovedSecret, error) {
key, err := identity.LoadSealingKey(identity.SealingKeyPath(statePath))
if err != nil {
return link.MovedSecret{}, err
}
ask := func(verb string, args any) (json.RawMessage, error) {
return queue.AskVault(context.Background(), verb, args)
}
moved, err := moveIntoTheVault(ask, key, m)
if err != nil {
fmt.Fprintf(os.Stderr, "mesh-host: a value was not moved into the vault: %v\n", err)
}
return moved, err
}
}
// moveIntoTheVault opens the row's value with this machine's sealing key, hands it to the vault's move-in in its two
// steps, and answers what the vault kept once its read-back — sealed to a key made here for this one call — opens
// to the same value.
func moveIntoTheVault(ask vaultAsk, key identity.SealingKey, m link.MoveSecret) (link.MovedSecret, error) {
if m.SealedTo != "" && m.SealedTo != key.Public {
return link.MovedSecret{}, errors.New("the controller's store says this value was sealed to a sealing key this " +
"machine no longer has, so it cannot be opened here")
}
value, err := key.Unseal(m.Sealed)
if err != nil {
return link.MovedSecret{}, fmt.Errorf("the value does not open with this machine's sealing key: %w", err)
}
defer wipe(value)
// This machine's sealing key, stated in its own name: the key the vault seals the core's credentials to for the
// controller to carry here (novox/hq ADR 0349 §16, issue 763), never a key the controller names.
first := map[string]any{"origin": m.Origin, "generation": m.Generation, "sealing_key": key.Public}
if !m.MadeAt.IsZero() {
first["made_at"] = m.MadeAt.UTC().Format(time.RFC3339Nano)
}
if m.Own() {
first["node"], first["module"], first["name"] = m.Node, m.Module, m.Name
} else {
first["provider"], first["provision"] = m.Provider, m.Provision
first["consumer"], first["consumer_module"] = m.Consumer, m.ConsumerModule
if m.Local != "" {
first["local"] = m.Local
}
}
raw, err := ask("move-in", first)
if err != nil {
return link.MovedSecret{}, fmt.Errorf("the vault did not open the move: %w", err)
}
var opened struct {
Call string `json:"call"`
SealTo string `json:"seal_to"`
Secret string `json:"secret"`
}
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
return link.MovedSecret{}, errors.New("the vault answered no call and no key to seal to")
}
sealed, err := identity.Seal(opened.SealTo, value)
if err != nil {
return link.MovedSecret{}, fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
}
backPub, backPriv, err := box.GenerateKey(rand.Reader)
if err != nil {
return link.MovedSecret{}, err
}
defer wipe(backPriv[:])
raw, err = ask("move-in", map[string]any{"call": opened.Call, "sealed": sealed,
"read_back_to": base64.StdEncoding.EncodeToString(backPub[:])})
if err != nil {
return link.MovedSecret{}, fmt.Errorf("the vault did not take %s: %w", opened.Secret, err)
}
var kept struct {
Name string `json:"name"`
Generation int `json:"generation"`
Unchanged bool `json:"unchanged"`
ReadBack string `json:"read_back"`
}
if err := json.Unmarshal(raw, &kept); err != nil || kept.Name == "" || kept.ReadBack == "" {
return link.MovedSecret{}, fmt.Errorf("the vault answered %s with no read-back", opened.Secret)
}
blob, err := base64.StdEncoding.DecodeString(kept.ReadBack)
if err != nil {
return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s is not sealed", kept.Name)
}
back, ok := box.OpenAnonymous(nil, blob, backPub, backPriv)
if !ok {
return link.MovedSecret{}, fmt.Errorf("the vault's read-back of %s does not open with the key made for it", kept.Name)
}
defer wipe(back)
if subtle.ConstantTimeCompare(back, value) != 1 {
return link.MovedSecret{}, fmt.Errorf("the vault keeps %s, but it read back another value than the one handed "+
"in: the move of it did not hold", kept.Name)
}
return link.MovedSecret{Secret: kept.Name, Generation: kept.Generation, Unchanged: kept.Unchanged}, nil
}
func wipe(b []byte) {
for i := range b {
b[i] = 0
}
}
+134
View File
@@ -0,0 +1,134 @@
package main
import (
"crypto/rand"
"encoding/base64"
"encoding/json"
"errors"
"strings"
"testing"
"golang.org/x/crypto/nacl/box"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
)
// standInVault answers move-in's two steps as the vault does: a key made for the call, the value opened with it,
// and the value kept read back sealed to the key the second step sent. lie makes its read-back another value.
type standInVault struct {
t *testing.T
pub *[32]byte
priv *[32]byte
opened map[string]any
kept string
lie string
refuse string
carried []string
}
func (v *standInVault) ask(verb string, args any) (json.RawMessage, error) {
raw, _ := json.Marshal(args)
v.carried = append(v.carried, string(raw))
if verb != "move-in" {
v.t.Fatalf("asked %s", verb)
}
if v.refuse != "" {
return nil, errors.New(v.refuse)
}
var a map[string]any
_ = json.Unmarshal(raw, &a)
if a["call"] == nil {
v.opened = a
pub, priv, _ := box.GenerateKey(rand.Reader)
v.pub, v.priv = pub, priv
return json.Marshal(map[string]string{"call": "c1", "seal_to": base64.StdEncoding.EncodeToString(pub[:]),
"secret": "own/anchor/postgres/superuser-password"})
}
blob, _ := base64.StdEncoding.DecodeString(a["sealed"].(string))
value, ok := box.OpenAnonymous(nil, blob, v.pub, v.priv)
if !ok {
v.t.Fatal("what the engine sealed does not open with the call's key")
}
v.kept = string(value)
back := v.kept
if v.lie != "" {
back = v.lie
}
key, _ := base64.StdEncoding.DecodeString(a["read_back_to"].(string))
var to [32]byte
copy(to[:], key)
boxed, _ := box.SealAnonymous(nil, []byte(back), &to, rand.Reader)
return json.Marshal(map[string]any{"name": "own/anchor/postgres/superuser-password", "generation": 1,
"read_back": base64.StdEncoding.EncodeToString(boxed)})
}
const theValue = "the-superusers-password"
func aMoveOf(t *testing.T, key identity.SealingKey) link.MoveSecret {
t.Helper()
sealed, err := identity.Seal(key.Public, []byte(theValue))
if err != nil {
t.Fatal(err)
}
return link.MoveSecret{Node: "anchor", By: "the move", MoveSecretAsk: link.MoveSecretAsk{Module: "postgres",
Name: "superuser-password", Origin: "made", Generation: "g-1", Sealed: sealed, SealedTo: key.Public}}
}
// The engine opens what was sealed to its machine, hands it to the vault sealed to the call's key — never in the
// clear — and answers what the vault kept only once the read-back opens to the same value.
func TestTheEngineMovesAValueSealedToItAndChecksTheReadBack(t *testing.T) {
key, err := identity.GenerateSealingKey()
if err != nil {
t.Fatal(err)
}
vault := &standInVault{t: t}
moved, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key))
if err != nil {
t.Fatal(err)
}
if moved.Secret != "own/anchor/postgres/superuser-password" || moved.Generation != 1 || vault.kept != theValue {
t.Fatalf("moved %+v, the vault kept %q", moved, vault.kept)
}
if vault.opened["module"] != "postgres" || vault.opened["name"] != "superuser-password" || vault.opened["node"] != "anchor" ||
vault.opened["origin"] != "made" || vault.opened["generation"] != "g-1" ||
vault.opened["sealing_key"] != key.Public {
t.Errorf("the first step named %v", vault.opened)
}
if strings.Contains(strings.Join(vault.carried, " "), theValue) {
t.Fatal("the value crossed the bus in the clear")
}
}
// A read-back that is not the value handed in is a failed move, said; so is a value this machine cannot open, and
// the vault's refusal.
func TestTheEngineSaysAMoveThatDidNotHold(t *testing.T) {
key, _ := identity.GenerateSealingKey()
vault := &standInVault{t: t, lie: "something-else"}
if _, err := moveIntoTheVault(vault.ask, key, aMoveOf(t, key)); err == nil ||
!strings.Contains(err.Error(), "read back") {
t.Fatalf("a read-back of another value: %v", err)
}
other, _ := identity.GenerateSealingKey()
if _, err := moveIntoTheVault((&standInVault{t: t}).ask, key, aMoveOf(t, other)); err == nil ||
!strings.Contains(err.Error(), "sealing key") {
t.Fatalf("a value sealed to another key: %v", err)
}
refusing := &standInVault{t: t, refuse: "the move never replaces a value the vault holds"}
if _, err := moveIntoTheVault(refusing.ask, key, aMoveOf(t, key)); err == nil ||
!strings.Contains(err.Error(), "never replaces") {
t.Fatalf("the vault's refusal: %v", err)
}
// A pair credential names both its ends.
pair := aMoveOf(t, key)
pair.Module, pair.Name = "", ""
pair.Provider, pair.Provision, pair.Consumer, pair.ConsumerModule = "anchor", "database", "shanks", "gitea"
ends := &standInVault{t: t}
if _, err := moveIntoTheVault(ends.ask, key, pair); err != nil {
t.Fatal(err)
}
if ends.opened["provider"] != "anchor" || ends.opened["consumer"] != "shanks" || ends.opened["consumer_module"] != "gitea" ||
ends.opened["node"] != nil {
t.Errorf("the pair's first step named %v", ends.opened)
}
}
+62
View File
@@ -0,0 +1,62 @@
package main
import (
"context"
"fmt"
"time"
"github.com/novox/mesh-host/internal/identity"
)
// This machine's sealing key, stated to the vault in the engine's own name (novox/hq ADR 0372, issue 786). The vault
// records it from the machine's own account alone and, when it is a first or a changed key, says the machine's core
// secrets changed so the controller asks for them again sealed to it. Without this a machine that never moved a
// secret in (adopted after feature 57) has no key at the vault. Nothing but the public half crosses.
const (
// nodeKeyRetryFrom and nodeKeyRetryUpTo bound the wait between tries while the vault does not answer: it starts
// after the bus and may not serve the verb yet.
nodeKeyRetryFrom = 5 * time.Second
nodeKeyRetryUpTo = 5 * time.Minute
// nodeKeyLook is how often the key file is read to see whether it changed.
nodeKeyLook = 30 * time.Second
)
// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends.
func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
retryFrom, retryUpTo time.Duration) {
stated, lastFailure := "", ""
wait := retryFrom
fail := func(why string) {
if why != lastFailure {
say("mesh-host: the sealing key was not stated to the vault, trying again: " + why)
lastFailure = why
}
wait = min(wait*2, retryUpTo)
}
look := min(nodeKeyLook, retryFrom)
for {
delay := look
key, err := load()
switch {
case err != nil:
fail(err.Error())
delay = wait
case key.Public != stated:
if _, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
fail(err.Error())
delay = wait
} else {
stated, lastFailure, wait = key.Public, "", retryFrom
say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
}
}
select {
case <-ctx.Done():
return
case <-time.After(delay):
}
}
}
+112
View File
@@ -0,0 +1,112 @@
package main
import (
"context"
"encoding/json"
"errors"
"sync"
"testing"
"time"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
)
// Issue 786 (novox/hq ADR 0372): the engine states its own sealing key to the vault's node-key at start, retries
// while the vault does not answer, and states it again when the key changes; the same key is not stated twice.
type keyVault struct {
mu sync.Mutex
stated []string
fail int
verbs []string
}
func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
v.mu.Lock()
defer v.mu.Unlock()
v.verbs = append(v.verbs, verb)
if v.fail > 0 {
v.fail--
return nil, errors.New("the vault does not answer")
}
raw, _ := json.Marshal(args)
var a struct {
Key string `json:"key"`
}
_ = json.Unmarshal(raw, &a)
v.stated = append(v.stated, a.Key)
return json.RawMessage(`{"recorded":true}`), nil
}
func (v *keyVault) said() []string {
v.mu.Lock()
defer v.mu.Unlock()
return append([]string(nil), v.stated...)
}
func waitFor(t *testing.T, what string, ok func() bool) {
t.Helper()
for i := 0; i < 400; i++ {
if ok() {
return
}
time.Sleep(5 * time.Millisecond)
}
t.Fatalf("timed out: %s", what)
}
func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(t *testing.T) {
first, _ := identity.GenerateSealingKey()
second, _ := identity.GenerateSealingKey()
var mu sync.Mutex
current := first
load := func() (identity.SealingKey, error) {
mu.Lock()
defer mu.Unlock()
return current, nil
}
vault := &keyVault{fail: 2}
var lines []string
var lmu sync.Mutex
say := func(l string) { lmu.Lock(); lines = append(lines, l); lmu.Unlock() }
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond)
waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
if vault.said()[0] != first.Public {
t.Errorf("stated %q, want the engine's own key %q", vault.said()[0], first.Public)
}
vault.mu.Lock()
verbs := append([]string(nil), vault.verbs...)
vault.mu.Unlock()
for _, v := range verbs {
if v != "node-key" {
t.Errorf("asked %s", v)
}
}
// The same key again is not stated again.
time.Sleep(60 * time.Millisecond)
if n := len(vault.said()); n != 1 {
t.Errorf("the same key was stated %d times", n)
}
mu.Lock()
current = second
mu.Unlock()
waitFor(t, "the changed key stated", func() bool { return len(vault.said()) == 2 })
if vault.said()[1] != second.Public {
t.Errorf("stated %q after the change, want %q", vault.said()[1], second.Public)
}
lmu.Lock()
defer lmu.Unlock()
if len(lines) == 0 {
t.Error("the engine said nothing")
}
}
// The statement a host makes names this contract (issue 786), one above the vault's box.
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
if link.NodeKeyContract != 13 {
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
}
}
+6 -3
View File
@@ -87,9 +87,12 @@ func TestTheStatementSaysAnAccountJudgedForRoot(t *testing.T) {
Groups: []string{"openrazer"}}, State: accounts.Healthy, Since: at},
}}
h := withAccounts(healthAsReported(liveness.Statement{At: at}, nil), as)
// At RootContract or later: an engine that also reads a container's busy says BusyContract (novox/hq ADR
// 0335), one above ADR 0334's OwnGroupContract.
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.BusyContract ||
// At RootContract or later: an engine that states its sealing key to the vault says NodeKeyContract (ADR 0372, issue 786), one above
// VaultBoxContract (it opens the vault's box for the core's credentials, ADR 0349 §16, issue 755), one above HandedContract (design 57 P7), one above ProvisionerContract (novox/hq ADR
// 0349 §4), one above ADR 0349 §5's OwnershipContract.
if h.Contract < link.RootContract || link.RootContract != 3 || h.Contract != link.NodeKeyContract ||
link.NodeKeyContract != link.VaultBoxContract+1 || link.VaultBoxContract != link.HandedContract+1 ||
link.HandedContract != link.ProvisionerContract+1 || link.ProvisionerContract != link.OwnershipContract+1 || link.OwnershipContract != link.BusyContract+1 ||
link.BusyContract != link.OwnGroupContract+1 {
t.Fatalf("contract %d", h.Contract)
}
+154
View File
@@ -0,0 +1,154 @@
package main
// Issue 755's proof on a real bus (novox/hq ADR 0349 §16): a core credential the vault sealed to this machine crosses
// the bus inside the declaration, which the controller carries and cannot open, and this engine opens it with its
// sealing key and writes the value. A listener on every subject keeps every byte that crossed; the value is never
// among them. Run against the throwaway bus merge-check.sh starts (MESH_TEST_NATS).
import (
"context"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/seal"
"github.com/nats-io/nats.go"
"github.com/novox/mesh-host/internal/apply"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
const coreCredential = "the-provisioner-clients-bus-password-2025"
func TestNatsTheEngineFillsACoreCredentialFromTheVaultsBoxWithNoValueInTheClear(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS")
if url == "" {
t.Skip("MESH_TEST_NATS unset")
}
// This machine's sealing key, where the engine reads it.
state := filepath.Join(t.TempDir(), "state.json")
key, err := identity.GenerateSealingKey()
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
t.Fatal(err)
}
// Every byte that crosses.
tap, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer tap.Close()
var mu sync.Mutex
var crossed []string
if _, err := tap.Subscribe(">", func(m *nats.Msg) {
mu.Lock()
crossed = append(crossed, m.Subject+" "+string(m.Data))
mu.Unlock()
}); err != nil {
t.Fatal(err)
}
_ = tap.Flush()
// The engine hears its declaration on its own subject.
const node = "vaultbox"
engine, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer engine.Close()
heard, err := engine.SubscribeSync(link.DeclareSubject(node))
if err != nil {
t.Fatal(err)
}
_ = engine.Flush()
// The vault's answer to the controller: the value in the ADR 0206 box, sealed to this machine's key. The
// controller carries it in the declaration as it is.
pub, err := key.PublicPEM()
if err != nil {
t.Fatal(err)
}
box, err := seal.Seal(coreCredential, pub)
if err != nil {
t.Fatal(err)
}
path := filepath.Join(t.TempDir(), "broker")
body, _ := json.Marshal(map[string]any{"declaration": 1, "resources": []map[string]any{
{"id": "need-provisioner-broker", "type": "file", "path": path, "vault-sealed": box}}})
controller, err := nats.Connect(url)
if err != nil {
t.Fatal(err)
}
defer controller.Close()
if err := controller.Publish(link.DeclareSubject(node), body); err != nil {
t.Fatal(err)
}
_ = controller.Flush()
msg, err := heard.NextMsg(5 * time.Second)
if err != nil {
t.Fatalf("the declaration did not reach the engine: %v", err)
}
d, err := declaration.Parse(msg.Data)
if err != nil {
t.Fatal(err)
}
sys, err := system.For("arch")
if err != nil {
t.Fatal(err)
}
if _, _, err := apply.Apply(context.Background(), sys, d, store.State{}, store.OriginCarried,
func(context.Context, string, ...string) (string, error) { return "", nil }, nil, sealOpener(state)); err != nil {
t.Fatal(err)
}
on, err := os.ReadFile(path)
if err != nil || string(on) != coreCredential {
t.Fatalf("the engine wrote %q, %v", on, err)
}
_ = tap.Flush()
time.Sleep(100 * time.Millisecond)
mu.Lock()
all := strings.Join(crossed, "\n")
mu.Unlock()
if !strings.Contains(all, link.DeclareSubject(node)) || !strings.Contains(all, "vault-sealed") {
t.Fatalf("the listener heard nothing of the declaration:\n%s", all)
}
for _, form := range []string{coreCredential, base64.StdEncoding.EncodeToString([]byte(coreCredential))} {
if strings.Contains(all, form) {
t.Fatal("the core credential crossed the bus in the clear")
}
}
}
// The engine opens a value the controller sealed (the NaCl box) as before, and the vault's box, with one key.
func TestTheEnginesOpenerOpensBothBoxes(t *testing.T) {
state := filepath.Join(t.TempDir(), "state.json")
key, _ := identity.GenerateSealingKey()
if err := os.WriteFile(identity.SealingKeyPath(state), []byte(key.Private), 0o600); err != nil {
t.Fatal(err)
}
open := sealOpener(state)
nacl, _ := identity.Seal(key.Public, []byte("old"))
if got, err := open(nacl); err != nil || string(got) != "old" {
t.Errorf("the controller's box opened to %q, %v", got, err)
}
pub, _ := key.PublicPEM()
box, _ := seal.Seal("new", pub)
raw, _ := json.Marshal(box)
if got, err := open(string(raw)); err != nil || string(got) != "new" {
t.Errorf("the vault's box opened to %q, %v", got, err)
}
}
+1 -1
View File
@@ -221,7 +221,7 @@
"path": "/var/lib/nats-module/conf/accounts.conf",
"mode": "0600",
"create-once": true,
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_lease.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
"content": "// The first user list, carried by the installer because at genesis there is no mesh to\n// compose one. A bootstrap credential, rotated with the store's and replaced by the\n// controller's own composition from its first start onward.\naccounts {\n MESH {\n jetstream: enabled\n users = [\n { user: \"controller\", password: \"$2a$10$AHqJgOifIVbU41KmATiMhuXFs8xa7Wl2HuN4UVBCXdN2jIQzjqApy\", permissions: {\n publish: { allow: [\"$JS.ACK.CONTROL.controller.>\", \"$JS.ACK.EVENTS.controller.>\", \"$JS.ACK.DEAD_LETTER_NOTICES.controller.>\", \"$JS.API.>\", \"$KV.mesh-controller_asked.>\", \"$KV.mesh-controller_assignments.>\", \"$KV.mesh-controller_calls.>\", \"$KV.mesh-controller_hand-acts.>\", \"$KV.mesh-controller_conditions.>\", \"$KV.mesh-controller_condition-history.>\", \"$KV.mesh-controller_declared-config.>\", \"$KV.mesh-controller_lease.>\", \"$KV.mesh-controller_predefined.>\", \"$KV.SEAT_MESH_BUILD_MACHINE_cancelled.>\", \"$KV.SEAT_NODE_BUILD_AGENT_cancelled.>\", \"_INBOX.enrol.>\", \"mesh.assignment.>\", \"mesh.mod.*.tool.>\", \"mesh.node.>\", \"mesh.seat.mesh-build-machine.accept.>\", \"mesh.seat.node-build-agent.accept.>\", \"mesh.seat.mesh-build-machine.tool.>\", \"mesh.seat.node-build-agent.tool.>\", \"mesh.seat.mesh-controller.event.applied\", \"mesh.seat.mesh-controller.event.built-before\", \"mesh.seat.mesh-controller.event.refused\", \"mesh.seat.mesh-controller.event.rolled-back\", \"mesh.seat.mesh-controller.event.checked\", \"mesh.seat.mesh-controller.event.plan-moved\", \"mesh.seat.mesh-controller.event.assignment-changed\", \"mesh.seat.mesh-controller.event.declared-config-changed\", \"mesh.seat.mesh-controller.event.predefined-changed\", \"mesh.seat.mesh-controller.event.condition-raised\", \"mesh.seat.mesh-controller.event.condition-changed\", \"mesh.seat.mesh-controller.event.condition-cleared\", \"mesh.seat.mesh-controller.event.doctor-heartbeat\", \"mesh.seat.mesh-controller.event.secret-replaced\", \"mesh.seat.mesh-controller.event.healer-acted\", \"$SRV.INFO\", \"mesh.seat.node-intrusion-prevention.tool.banned.*\", \"mesh.seat.node-backup.tool.backed-up.*\", \"mesh.seat.node-backup.tool.now.*\", \"mesh.seat.mesh-delivery.tool.stalled\", \"mesh.seat.mesh-delivery.tool.close\", \"mesh.seat.node-launcher.tool.secret.*\", \"mesh.seat.mesh-delivery.tool.release\", \"mesh.seat.mesh-delivery.tool.stop\", \"mesh.seat.mesh-delivery.tool.times\", \"mesh.seat.mesh-delivery.tool.groups\", \"mesh.seat.node-service-manager.tool.restart.*\", \"mesh.seat.operator-channel.tool.tell\", \"mesh.seat.mesh-controller.tool.plans\", \"mesh.events.dead.>\", \"mesh.again.>\"] }\n subscribe: { allow: [\"$JS.API.>\", \"$JS.EVENT.ADVISORY.CONSUMER.MAX_DELIVERIES.>\", \"$JS.EVENT.ADVISORY.CONSUMER.DELETED.>\", \"_DELIVER.controller\", \"_DELIVER.controller.>\", \"_INBOX.controller.>\", \"mesh.control.>\", \"mesh.mod.*.event.provisioner.failing\", \"mesh.mod.*.event.provisioner.recovered\", \"mesh.mod.*.event.provisioner.retirement\", \"mesh.mod.gitea.event.pull.merged\", \"mesh.mod.gitea.event.pull.updated\", \"mesh.mod.mesh-catalog.event.catching-up\", \"mesh.mod.mesh-catalog.event.upgraded\", \"mesh.seat.mesh-build-machine.event.built\", \"mesh.seat.node-build-agent.event.built\", \"mesh.seat.operator-channel.event.decided.mesh-controller\", \"mesh.seat.mesh-controller.tool.>\", \"$SRV.PING\", \"$SRV.INFO\", \"$SRV.PING.mesh-controller\", \"$SRV.PING.mesh-controller.>\", \"$SRV.INFO.mesh-controller\", \"$SRV.INFO.mesh-controller.>\", \"$SRV.STATS\", \"$SRV.STATS.mesh-controller\", \"$SRV.STATS.mesh-controller.>\"] }\n allow_responses: { max: 1, ttl: \"1m\" }\n } }\n ]\n }\n}\n"
},
// The upstream server, until the broker step adopts it: the `nats` module's own image, built by the
// mesh, takes its place there once, on these same directories (ADR 0333 §3), as the bus's planned
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
golang.org/x/sys v0.48.0
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970 h1:9tFDQsgmI+4X7/BpZGXIr+HemPKE7YddYGqWV0lINAI=
git.novox.be/novox/mesh-sdk/go v0.1.11-0.20261009143344-f047d0a4a970/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d h1:BR8uwv4BqyLaBwITuLylrd9DRUnu0D5DjJw0RuQSXbk=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010190644-622b6b4f556d/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c h1:l5onJwoIeH8yE/PjVlEeoPyXBsHp2NQiWLllz2fwX7s=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261010191001-33917f91ac3c/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19 h1:mAMt3KFmrHzDiBv4MLL94opWcG9y73ISMrBUyisapQE=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261015210729-d3e182503c19/go.mod h1:GFuZUElBZ9A++mxgIKo97aXXo+kV0uJ/UkbhQPPIbrY=
github.com/klauspost/compress v1.20.0 h1:a3C1ke2ohxFymNlb2HWAHjDeKCI90scRskErZkR0ezA=
github.com/klauspost/compress v1.20.0/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/nats-io/nats.go v1.54.0 h1:vsXoOxjHp/GmPUN+EcI7uOf/uB+iAP+kEsAFNQN0yzA=
+1 -1
View File
@@ -119,7 +119,7 @@ func secretsOf(d *declaration.Declaration, account string) []string {
var out []string
for _, r := range d.Resources {
f, ok := r.(*declaration.File)
if !ok || f.Path == "" || (f.Sealed == "" && len(f.Secrets) == 0) || f.Owner == account {
if !ok || f.Path == "" || (!f.Secret() && len(f.Secrets) == 0) || f.Owner == account {
continue
}
out = append(out, f.Path)
+68 -32
View File
@@ -15,6 +15,7 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
@@ -781,6 +782,16 @@ func ApplyMindingWindows(
}
}
// **A filled file is the provisioner client's to write** (novox/hq ADR 0349 §5, design 57 P7): one writer per
// file. The declaration says where it goes and how it is written, and only the write verb (WriteFilled)
// writes it, on the client's word. This apply — the controller's send and the five-minute re-apply alike —
// neither writes it nor records it, and nothing that reads it is moved for it; so only what the verb placed
// is recorded, and only that is removed as an orphan when the file leaves the declaration.
if f, isFile := resource.(*declaration.File); isFile && d.Fills(f.Path) {
report.Outcomes = append(report.Outcomes, leftToTheClient(f, &known, handedFiles, &report, log))
continue
}
// The private network takes over the tunnel it found, ahead of the service that replaces
// it (novox/hq ADR 0105): its configuration kept, its unit stopped and disabled, never
// flushed. A failure here fails the service too — the mesh's interface is not started on a
@@ -999,34 +1010,7 @@ func ApplyMindingWindows(
kept, keptMode, keptOwner = from.Kept, from.KeptMode, from.KeptOwner
}
// Only now. The record follows the fact, never leads it.
known.Record(store.Applied{
Origin: origin,
ID: resource.Identity(), Type: string(resource.Kind()),
Target: outcome.Target, AppliedAt: time.Now().UTC(),
Wrote: outcome.wrote,
Into: outcome.into,
Kept: kept,
KeptMode: keptMode,
KeptOwner: keptOwner,
Reads: outcome.reads,
BusySince: outcome.busySince,
Stateless: outcome.stateless,
Scope: outcome.scope,
User: outcome.user,
Found: outcome.found,
DeclaredUnit: outcome.declared,
Shell: outcome.shell,
Linger: outcome.linger,
Groups: outcome.groups,
Unpacked: outcome.unpacked,
AsFound: outcome.asFound,
Declared: outcome.asFoundDeclared,
FoundAt: outcome.asFoundAt,
Holds: holds(resource),
})
if outcome.found != nil {
known.DropFound(resource.Identity())
}
recordApplied(&known, origin, resource, outcome, kept, keptMode, keptOwner)
// Its module has been taken, and what was held for it is now the mesh's. A file written
// into, or a service whose lifecycle is the machine's, replaced nothing that was found, so
// its outcome says what the apply did, not that a cutover happened; a hold from when it was
@@ -1075,7 +1059,7 @@ func ApplyMindingWindows(
if outcome.Action == heldStill || outcome.Action == waitsBusy {
// Not changed: nothing about it moved, so nothing that restarts on it restarts.
log(fmt.Sprintf(" %s %s (%s): %s", outcome.Action, outcome.ID, outcome.Target, outcome.Detail))
} else if outcome.Action != "unchanged" {
} else if moved(outcome) {
changed[resource.Identity()] = true
// With the detail, when there is one: "updated app" says a container was replaced;
// which file made that happen is what somebody reading the log at the time needs
@@ -1168,6 +1152,47 @@ func ApplyMindingWindows(
return report, known, nil
}
// recordApplied records what applying a resource did, once it is done: the record follows the fact, never leads it.
// kept, keptMode and keptOwner are where the original of what it replaced was kept, and how it was found. What an
// earlier apply found its unit as goes with it.
func recordApplied(known *store.State, origin string, resource declaration.Resource, outcome Outcome,
kept, keptMode, keptOwner string) {
known.Record(store.Applied{
Origin: origin,
ID: resource.Identity(), Type: string(resource.Kind()),
Target: outcome.Target, AppliedAt: time.Now().UTC(),
Wrote: outcome.wrote,
Into: outcome.into,
Kept: kept,
KeptMode: keptMode,
KeptOwner: keptOwner,
Reads: outcome.reads,
BusySince: outcome.busySince,
Stateless: outcome.stateless,
Scope: outcome.scope,
User: outcome.user,
Found: outcome.found,
DeclaredUnit: outcome.declared,
Shell: outcome.shell,
Linger: outcome.linger,
Groups: outcome.groups,
Unpacked: outcome.unpacked,
AsFound: outcome.asFound,
Declared: outcome.asFoundDeclared,
FoundAt: outcome.asFoundAt,
Holds: holds(resource),
})
if outcome.found != nil {
known.DropFound(resource.Identity())
}
}
// moved reports whether applying a resource moved it, so what restarts or reloads on it does: anything but
// unchanged, and not a container held still or waiting for its busy one to say idle.
func moved(outcome Outcome) bool {
return outcome.Action != "unchanged" && outcome.Action != heldStill && outcome.Action != waitsBusy
}
// afterWhatTheyRead is the resources in declared order, except that **a service comes after every
// resource it names under restart-on or reload-on** (novox/hq issue 260).
//
@@ -1236,7 +1261,9 @@ func afterWhatTheyRead(resources []declaration.Resource) []declaration.Resource
const guardPrefix = declaration.AdoptionPrefix + "guard"
// Unseal opens a value the mesh sealed to this node. Nil when the node has no sealing key, which
// makes every sealed file an error rather than a silently skipped one.
// makes every sealed file an error rather than a silently skipped one. A value the vault sealed (a core
// credential, novox/hq ADR 0349 §16) is given as its box's JSON, which begins with `{`; one the controller sealed
// is base64, which never does (identity.SealingKey.Open).
type Unseal func(sealed string) ([]byte, error)
func applyOne(ctx context.Context, sys system.System, r declaration.Resource, run Runner,
@@ -1576,7 +1603,16 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepF
return out, fmt.Errorf(
"%s is sealed to this node and this node has no sealing key", r.Path)
}
opened, err := unseal(r.Sealed)
sealed := r.Sealed
if r.VaultSealed != nil {
// The vault's box, handed to the opener as its JSON (Unseal).
raw, err := json.Marshal(r.VaultSealed)
if err != nil {
return out, err
}
sealed = string(raw)
}
opened, err := unseal(sealed)
if err != nil {
return out, fmt.Errorf("cannot open %s: %w", r.Path, err)
}
@@ -1617,7 +1653,7 @@ func applyFile(r *declaration.File, previous store.Applied, unseal Unseal, keepF
// A seed that is already there is left exactly as it is — whatever has grown in it since is
// not the mesh's to put back (novox/hq issue 035). What this host records is what it once
// wrote, so a later declaration that changes the seed is not mistaken for drift either.
if r.CreateOnce && existed {
if r.Seeded() && existed {
out.wrote = previous.Wrote
if out.wrote == "" {
out.wrote = digestOf(string(existing))
+230
View File
@@ -0,0 +1,230 @@
package apply
import (
"context"
"fmt"
"slices"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
"github.com/novox/mesh-host/internal/system"
)
// WriteFilled writes what the provisioner's client filled into one file the declaration names as filled (novox/hq
// ADR 0349 §4), and makes what reads the file take the change, exactly as an apply that changed it would.
//
// **The declared file says everything but the content.** Its path, mode, owner and who owns it at runtime are the
// declaration's; the client's request carries the content alone. So the file is applied as the declared one with
// that content in place of the declaration's: an owned file whole through applyFile, a seed only when absent, a
// managed file's keys through applyInto or its block through applyBlock — the same guards on where nothing is
// placed, the same Keep of an original found there, the same record. What the declaration seals or fills with
// `${secret:}` the client has already filled, so the content is written as it came; a file that carried a secret
// keeps the mode a secret is written with when the declaration gives none.
//
// **Then what reads it, in the apply's order** (afterWhatTheyRead): a service naming it under restart-on is
// restarted and under reload-on reloaded, a process or container naming it under restart-on is restarted, and a
// running container that reads it at creation — an env-file, a file mounted into it — is recreated, because what
// it read is recorded by digest and this write changed the digest. Each of those is applied and recorded as an
// apply does, and what moves goes on moving what names it. Nothing else of the declaration is touched.
//
// The caller holds the apply's lock: a write and an apply never write one file at once. log is said the path, the
// action and the digest, never the content.
func WriteFilled(
ctx context.Context,
sys system.System,
d *declaration.Declaration,
known store.State,
path string,
content []byte,
run Runner,
log func(string),
keep Keep,
windows *Windows,
) (report Report, _ store.State, _ error) {
if log == nil {
log = func(string) {}
}
if !d.Fills(path) {
return report, known, &Error{Resource: path, Err: fmt.Errorf("this node's declaration does not name %s "+
"as a file the provisioner fills, so nothing was written", path)}
}
var declared []*declaration.File
for _, r := range d.Resources {
if f, ok := r.(*declaration.File); ok && f.Path == path {
declared = append(declared, f)
}
}
if len(declared) != 1 {
// The parser refuses a filled path no file declares; two at one path would need the request to say whose
// keys or block it carries, and it carries the content only.
return report, known, &Error{Resource: path, Err: fmt.Errorf("%d resources declare %s, and a filled file "+
"is written for exactly one, so nothing was written", len(declared), path)}
}
file := filledAs(declared[0], content)
if held, isHeld := known.HeldAt(file.ID); isHeld {
return report, known, &Error{Resource: file.ID, Err: fmt.Errorf("%s is held as found for a module not "+
"yet taken (%s), so nothing was written", path, held.Why)}
}
in := inputs{declares: map[string]string{}, known: &known, windows: windows, away: newStoreAway(windows, log),
groups: groupsWanted(d.Resources), handed: Handed{}}
refreshRuntimeRoots(ctx)
in.refused = refusedPlaces(d.Resources)
in.agentHomes = rootNeverHomes(d)
in.dirsBefore = map[string]bool{}
// A container that reads the file and mounts a library whose disk or mount is not there is held, as a full apply
// holds it: not created over the missing disk (novox/hq ADR 0328 §9).
in.held = heldLibraries(ctx, d.Libraries)
for _, r := range d.Resources {
in.declares[r.Identity()] = declaredDigest(r)
}
changed := map[string]bool{}
// The file, as the apply loop applies it.
was, _ := known.Find(file.ID)
var keepFound Keep
if (was.ID == "" || was.Target != file.Path) && !known.Recorded(string(declaration.TypeFile), file.Path) {
keepFound = keep
}
outcome, err := applyOne(ctx, sys, file, run, changed, in, was, nil, keepFound)
if err != nil {
log(fmt.Sprintf(" failed %s (%s): %v", file.ID, file.Path, err))
return report, known, &Error{Resource: file.ID, Err: err, Done: report}
}
recordKeeping(&known, file, outcome, was)
report.Outcomes = append(report.Outcomes, outcome)
if !moved(outcome) {
return report, known, nil
}
changed[file.ID] = true
line := fmt.Sprintf(" %s %s (%s), filled: %s", outcome.Action, file.ID, file.Path, outcome.wrote)
if outcome.Detail != "" {
line += ": " + outcome.Detail
}
log(line)
// What reads it, in the apply's order, each moving what names it in turn.
var failures []*Error
for _, r := range afterWhatTheyRead(d.Resources) {
if r.Identity() == file.ID || !readsChange(r, path, changed) {
continue
}
if _, isHeld := known.HeldAt(r.Identity()); isHeld {
log(fmt.Sprintf(" held %s (%s): held as found for a module not yet taken, so it was not restarted "+
"for %s", r.Identity(), r.Target(), file.ID))
continue
}
was, _ := known.Find(r.Identity())
o, err := applyOne(ctx, sys, r, run, changed, in, was, nil, nil)
if err != nil {
if o.found != nil {
known.KeepFound(r.Identity(), store.OriginDeclared, *o.found)
}
failures = append(failures, &Error{Resource: r.Identity(), Err: err})
log(fmt.Sprintf(" failed %s (%s): %v", r.Identity(), o.Target, err))
continue
}
if o.Action == "waiting" {
if o.found != nil {
known.KeepFound(r.Identity(), store.OriginDeclared, *o.found)
}
report.Outcomes = append(report.Outcomes, o)
log(fmt.Sprintf(" waiting %s (%s): %s", o.ID, o.Target, o.Detail))
continue
}
recordKeeping(&known, r, o, was)
report.Outcomes = append(report.Outcomes, o)
if o.busy != nil {
report.Busy = append(report.Busy, *o.busy)
}
if moved(o) {
changed[r.Identity()] = true
line := fmt.Sprintf(" %s %s (%s)", o.Action, o.ID, o.Target)
if o.Detail != "" {
line += ": " + o.Detail
}
log(line)
} else if o.Action != "unchanged" {
log(fmt.Sprintf(" %s %s (%s): %s", o.Action, o.ID, o.Target, o.Detail))
}
}
if len(failures) > 0 {
first := failures[0]
first.Done = report
first.Others = len(failures) - 1
return report, known, first
}
return report, known, nil
}
// filledAs is the declared file with the filled content in place of the declaration's.
func filledAs(declared *declaration.File, content []byte) *declaration.File {
f := *declared
f.Content, f.Sealed, f.VaultSealed, f.Bytes, f.Secrets = string(content), "", nil, "", nil
if f.Mode == "" && (declared.Secret() || len(declared.Secrets) > 0) {
// Written as the declaration would have written it: a file carrying a credential is not world-readable.
f.Mode = "0600"
}
return &f
}
// readsChange reports whether applying r is what makes it take a change of what changed: a service's restart-on or
// reload-on, a process's or container's restart-on, or a running container that reads path at creation.
func readsChange(r declaration.Resource, path string, changed map[string]bool) bool {
switch res := r.(type) {
case *declaration.Service:
return len(restartedBy(res.RestartOn, changed)) > 0 || len(restartedBy(res.ReloadOn, changed)) > 0
case *declaration.Process:
return len(restartedBy(res.RestartOn, changed)) > 0
case *declaration.Container:
if len(restartedBy(res.RestartOn, changed)) > 0 {
return true
}
if res.RunOnce || res.Schedule != "" {
return false
}
if slices.Contains(res.EnvFile, path) {
return true
}
for _, v := range res.Volumes {
if mountSource(v) == path {
return true
}
}
}
return false
}
// recordKeeping records r as applied, carrying where the original it replaced was kept from its record when this
// write kept none — for as long as the resource is recorded at the same target, as the apply loop carries it.
func recordKeeping(known *store.State, r declaration.Resource, o Outcome, was store.Applied) {
kept, keptMode, keptOwner := o.kept, o.keptMode, o.keptOwner
if kept == "" && was.Target == o.Target {
kept, keptMode, keptOwner = was.Kept, was.KeptMode, was.KeptOwner
}
recordApplied(known, store.OriginDeclared, r, o, kept, keptMode, keptOwner)
}
// leftToTheClient is a filled file as an apply leaves it (novox/hq ADR 0349 §5, design 57 P7): untouched, and said
// so. A file the mesh wrote at the same path under another id, going in this apply, is handed to it as an apply hands
// a whole file (novox/hq ADR 0223): its record — what the mesh wrote there, and the original it kept — is the filled
// file's now, so the verb judges what is there against it, and the file is neither removed nor written.
func leftToTheClient(f *declaration.File, known *store.State, handedFiles map[string]store.Applied, report *Report,
log func(string)) Outcome {
if from, handed := handedFiles[f.ID]; handed {
if _, recorded := known.Find(f.ID); !recorded {
rec := from
rec.ID = f.ID
known.Record(rec)
}
delete(handedFiles, f.ID)
known.Forget(from.ID)
detail := "no longer declared; " + f.ID + " declares the file now, filled by the provisioner's client, so " +
"it was handed to it, not removed"
report.Outcomes = append(report.Outcomes, Outcome{ID: from.ID, Type: from.Type, Target: from.Target,
Action: "forgotten", Detail: detail})
log(fmt.Sprintf(" forgotten %s (%s): %s", from.ID, from.Target, detail))
}
return Outcome{ID: f.ID, Type: string(declaration.TypeFile), Target: f.Path, Action: "unchanged",
Detail: "filled by the provisioner's client, which alone has it written"}
}
+218
View File
@@ -0,0 +1,218 @@
package apply
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0349 §5, one writer per file (design 57 P7, issue 673): a file the declaration names as
// filled is written only through the write verb (WriteFilled), on the provisioner client's word. An apply — the
// controller's send and the five-minute re-apply alike — never writes it, never records it, and removes it as an
// orphan only when the verb placed it.
// An apply leaves a filled file alone, whatever who owns it at runtime says: absent stays absent and is not
// recorded, and what the verb wrote stays as it wrote it through every re-apply.
func TestAnApplyNeverWritesAFilledFile(t *testing.T) {
for _, c := range []struct{ name, fields, content string }{
{"owned", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n"},
{"seed", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n"},
{"managed block", `"into":"block","content":"","owner-at-runtime":"managed"`, "10.42.0.1 ace\n"},
} {
t.Run(c.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "the-file")
d := filledDecl(t, path, c.fields, "")
report, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared,
noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); err == nil {
t.Fatalf("an apply wrote a filled file the client has not filled: %q", readText(t, path))
}
if _, ok := state.Find("m.conf"); ok {
t.Errorf("an apply recorded a filled file it did not write")
}
if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" || !strings.Contains(o.Detail, "provisioner") {
t.Errorf("an apply did not say it left the file to the provisioner's client: %+v", o)
}
_, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte(c.content),
noServices, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
placed, _ := state.Find("m.conf")
want, wantMode := readText(t, path), modeAt(t, path)
// The re-apply, twice over: nothing written, the verb's record kept as it wrote it.
for i := 0; i < 2; i++ {
var said []string
report, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
noServices, func(line string) { said = append(said, line) }, nil)
if err != nil {
t.Fatal(err)
}
if got := readText(t, path); got != want || modeAt(t, path) != wantMode {
t.Fatalf("re-apply %d wrote the filled file again: %q, want %q", i+1, got, want)
}
if o := outcomeOf(report, "m.conf"); o.Action != "unchanged" {
t.Errorf("re-apply %d: %+v", i+1, o)
}
if rec, _ := state.Find("m.conf"); rec.Wrote != placed.Wrote || rec.Target != path {
t.Errorf("re-apply %d changed the verb's record: %+v, was %+v", i+1, rec, placed)
}
if len(said) != 0 {
t.Errorf("re-apply %d said something of an untouched file: %q", i+1, said)
}
}
})
}
}
// What reads a filled file is moved by the verb's write, once, never again by an apply: a service under
// restart-on is not restarted and a container reading it is not recreated by the re-apply.
func TestAnApplyMovesNothingForAFilledFile(t *testing.T) {
env := filepath.Join(t.TempDir(), "forge.env")
d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`,
`{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`)
m := &machine{containers: map[string]*fakeContainer{}}
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"),
m.run, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
m.asked = nil
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, m.run, nil,
nil); err != nil {
t.Fatal(err)
}
if m.removed("forge") || m.did("docker run") {
t.Errorf("the re-apply recreated the container reading the filled file: %v", m.asked)
}
if got := readText(t, env); got != "PORT=5432\n" {
t.Errorf("the re-apply wrote the filled file: %q", got)
}
conf := filepath.Join(t.TempDir(), "daemon.conf")
d = filledDecl(t, conf, `"content":"","owner-at-runtime":"owned"`,
`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`)
var commands []string
_, state, err = WriteFilled(context.Background(), archHost(t), d, store.State{}, conf, []byte("level = 1\n"),
recordingServices(&commands), nil, nil, nil)
if err != nil {
t.Fatal(err)
}
commands = nil
if _, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared,
recordingServices(&commands), nil, nil); err != nil {
t.Fatal(err)
}
if n := count(commands, "systemctl stop m.service"); n != 0 {
t.Errorf("the re-apply restarted what reads the filled file: %v", commands)
}
}
// A filled file the verb never placed is not the mesh's to remove: one found there, or put there by hand, stays
// when the file leaves the declaration.
func TestAFilledFileTheVerbNeverPlacedIsNotRemoved(t *testing.T) {
path := filepath.Join(t.TempDir(), "the-file")
if err := os.WriteFile(path, []byte("the machine's own\n"), 0o644); err != nil {
t.Fatal(err)
}
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices,
nil, nil)
if err != nil {
t.Fatal(err)
}
gone := elsewhere(t)
report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices,
nil, nil)
if err != nil {
t.Fatal(err)
}
if got := readText(t, path); got != "the machine's own\n" {
t.Errorf("a filled file the verb never placed was touched when it left: %q", got)
}
if o := outcomeOf(report, "m.conf"); o.Action == "removed" {
t.Errorf("a filled file the verb never placed was removed: %+v", o)
}
if _, ok := state.Find("m.conf"); ok {
t.Errorf("a record of the filled file was left")
}
}
// What the verb placed is the mesh's: when the file leaves the declaration it is removed as an orphan, as a file
// an apply wrote is.
func TestAFilledFileTheVerbPlacedIsRemovedWhenItLeaves(t *testing.T) {
path := filepath.Join(t.TempDir(), "the-file")
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices,
nil, nil)
if err != nil {
t.Fatal(err)
}
_, state, err = WriteFilled(context.Background(), archHost(t), d, state, path, []byte("port = 8080\n"),
noServices, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
gone := elsewhere(t)
report, state, err := Apply(context.Background(), archHost(t), gone, state, store.OriginDeclared, noServices,
nil, nil)
if err != nil {
t.Fatal(err)
}
if _, err := os.Stat(path); err == nil {
t.Errorf("a filled file the verb placed stayed when it left the declaration: %q", readText(t, path))
}
if o := outcomeOf(report, "m.conf"); o.Action != "removed" {
t.Errorf("not removed: %+v", o)
}
if _, ok := state.Find("m.conf"); ok {
t.Errorf("its record stayed")
}
}
// A file the mesh wrote the old way and that is now filled keeps its record for the verb to judge against, and is
// not written over by the apply that hands it to the client; a file at the same path recorded under another id
// is handed to the filled one, not removed.
func TestAFileThatMovesToTheClientIsHandedNotRewritten(t *testing.T) {
path := filepath.Join(t.TempDir(), "the-file")
old := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.old","type":"file","path":%q,`+
`"content":"port = 1\n"}]}`, path))
_, state, err := Apply(context.Background(), archHost(t), old, store.State{}, store.OriginDeclared, noServices,
nil, nil)
if err != nil {
t.Fatal(err)
}
was, _ := state.Find("m.old")
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`, "")
_, state, err = Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := readText(t, path); got != "port = 1\n" {
t.Errorf("the file handed to the client was rewritten or removed: %q", got)
}
if _, ok := state.Find("m.old"); ok {
t.Errorf("the record going was kept beside the filled file's")
}
if rec, ok := state.Find("m.conf"); !ok || rec.Wrote != was.Wrote || rec.Target != path {
t.Errorf("the filled file did not take the record of what the mesh wrote there: %+v", rec)
}
}
// elsewhere is a declaration with nothing at the filled path: one unrelated file.
func elsewhere(t *testing.T) *declaration.Declaration {
t.Helper()
return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"n.other","type":"file","path":%q,`+
`"content":"x\n"}]}`, filepath.Join(t.TempDir(), "other")))
}
+57
View File
@@ -0,0 +1,57 @@
//go:build linux
package apply
import (
"context"
"fmt"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// A container that reads a filled file and mounts a library whose disk is not there is held by the write verb as a
// full apply holds it (novox/hq ADR 0328 §9, issue 668): never created or recreated over the missing disk.
func TestAWriteDoesNotRecreateAContainerHeldForItsLibrary(t *testing.T) {
// No folder is given the group: the library's root is absent, and nothing is made.
gid := 61000
libraries(t, nil, nil)
base := t.TempDir()
root := filepath.Join(base, "gone", "media")
env := filepath.Join(base, "radarr.env")
raw := fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":[%q],
"groups":[{"name":"mesh-media","gid":%d}],"libraries":[%s],"resources":[
{"id":"radarr.env","type":"file","path":%q,"content":"","mode":"0600","owner-at-runtime":"owned"},
{"id":"radarr.app","type":"container","name":"radarr","image":%q,"env-file":[%q],"volumes":[%q]}]}`,
env, gid, libJSON("media", root, gid, true, false, "movies"), env, pinned, env,
filepath.Join(root, "movies")+":/movies")
d, err := declaration.Parse([]byte(raw))
if err != nil {
t.Fatal(err)
}
var ran []string
run := func(_ context.Context, name string, args ...string) (string, error) {
ran = append(ran, name+" "+strings.Join(args, " "))
return "", nil
}
var state store.State
for _, content := range []string{"PORT=1\n", "PORT=2\n"} {
var report Report
report, state, err = WriteFilled(context.Background(), archHost(t), d, state, env, []byte(content), run,
nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if o := outcomeOf(report, "radarr.app"); o.Action != "waiting" {
t.Errorf("after writing %q the container is %+v, not held for its library", content, o)
}
}
for _, line := range ran {
if strings.HasPrefix(line, "docker run") || strings.HasPrefix(line, "docker rm") {
t.Errorf("the container was created or removed over a missing disk: %v", ran)
}
}
}
+271
View File
@@ -0,0 +1,271 @@
package apply
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"testing"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0349 §4 (issue 668): the provisioner's client has the node-engine write a file it filled,
// and the node-engine writes it as an apply of the declared file with that content would — through applyFile,
// applyInto or applyBlock by who owns it at runtime — and makes what reads it take the change as an apply does.
// filledDecl is a declaration naming path as filled, with the file's fields after id, type and path, and more
// resources after it.
func filledDecl(t *testing.T, path, fields, more string) *declaration.Declaration {
t.Helper()
if more != "" {
more = "," + more
}
return parseTrusted(t, fmt.Sprintf(`{"declaration":1,"provisioner_account":"mesh-node-provisioner",
"filled_files":[%q],"resources":[{"id":"m.conf","type":"file","path":%q,%s}%s]}`, path, path, fields, more))
}
func TestAWriteOfAPathNotFilledIsRefused(t *testing.T) {
dir := t.TempDir()
filled := filepath.Join(dir, "filled.conf")
other := filepath.Join(dir, "other.conf")
d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[
{"id":"m.conf","type":"file","path":%q,"content":"x\n","owner-at-runtime":"owned"},
{"id":"m.other","type":"file","path":%q,"content":"y\n"}
]}`, filled, filled, other))
for _, path := range []string{other, filepath.Join(dir, "nothing.conf")} {
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("evil\n"),
noServices, nil, nil, nil)
if err == nil || !strings.Contains(err.Error(), "does not name") {
t.Errorf("%s: a write of a path the declaration does not fill was not refused: %v", path, err)
}
if _, err := os.Stat(path); err == nil && path != other {
t.Errorf("%s was written", path)
}
if len(state.Resources) != 0 {
t.Errorf("a refused write recorded something: %+v", state.Resources)
}
}
if _, err := os.Stat(other); err == nil {
t.Errorf("%s was written", other)
}
}
// Each ownership is written by the write verb exactly as an apply of the declared file with the same content: the
// same action, the same bytes on disk, the same mode.
func TestAFilledFileIsWrittenAsItsOwnershipSays(t *testing.T) {
for _, c := range []struct {
name, fields, content, before, action string
}{
{"owned, absent", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n", "", "created"},
{"owned, over what is there", `"content":"","mode":"0640","owner-at-runtime":"owned"`, "port = 8080\n",
"port = 1\n", "updated"},
{"seed, absent", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n", "", "created"},
{"seed, present", `"content":"","mode":"0600","owner-at-runtime":"seed"`, "user default on\n",
"user default on\nuser app-one on\n", "kept"},
{"managed json", `"into":"json","content":"{}","owner-at-runtime":"managed"`,
`{"insecure-registries":["10.42.0.1:5000"]}`, machinesOwn, "updated"},
{"managed block", `"into":"block","content":"x\n","owner-at-runtime":"managed"`, "10.42.0.1 ace\n",
workstationHosts, "updated"},
} {
t.Run(c.name, func(t *testing.T) {
at := func() string {
path := filepath.Join(t.TempDir(), "the-file")
if c.before != "" {
if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil {
t.Fatal(err)
}
}
return path
}
written := at()
report, state, err := WriteFilled(context.Background(), archHost(t), filledDecl(t, written, c.fields, ""),
store.State{}, written, []byte(c.content), noServices, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
applied := at()
withContent := strings.Replace(c.fields, fmt.Sprintf(`"content":%q`, contentOf(t, c.fields)),
fmt.Sprintf(`"content":%q`, c.content), 1)
// The same file declared the old way: an apply writes a filled file only on the client's word (P7).
plain := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"m.conf","type":"file",`+
`"path":%q,%s}]}`, applied, withContent))
want, _, err := Apply(context.Background(), archHost(t), plain, store.State{}, store.OriginDeclared,
noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != c.action || got != want.Outcomes[0].Action {
t.Errorf("written %s; an apply of the same content %s; want %s", got, want.Outcomes[0].Action, c.action)
}
if got, apply := readText(t, written), readText(t, applied); got != apply {
t.Errorf("written, the file holds\n%q\nan apply of the same content leaves\n%q", got, apply)
}
if a, b := modeAt(t, written), modeAt(t, applied); a != b {
t.Errorf("written mode %o, an apply's %o", a, b)
}
if rec, ok := state.Find("m.conf"); !ok || rec.Target != written || rec.Origin != store.OriginDeclared {
t.Errorf("the write is not recorded as the file's: %+v", rec)
}
})
}
}
func contentOf(t *testing.T, fields string) string {
t.Helper()
var holder struct {
Content string `json:"content"`
}
if err := json.Unmarshal([]byte("{"+fields+"}"), &holder); err != nil {
t.Fatal(err)
}
return holder.Content
}
func modeAt(t *testing.T, path string) os.FileMode {
t.Helper()
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
return info.Mode().Perm()
}
// A file that carried a secret keeps the mode a secret is written with: the client filled the value, and the file
// is no more readable for it.
func TestAFilledFileThatCarriedASecretIsNotWorldReadable(t *testing.T) {
path := filepath.Join(t.TempDir(), "token.env")
d := filledDecl(t, path, `"content":"TOKEN=${secret:token}\n","secrets":{"token":"c2VhbGVk"},"owner-at-runtime":"owned"`, "")
if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("TOKEN=abc\n"),
noServices, nil, nil, nil); err != nil {
t.Fatal(err)
}
if m := modeAt(t, path); m != 0o600 {
t.Errorf("a filled file that carried a secret is mode %o", m)
}
}
func TestAWriteUnderReloadOnReloadsItsProgramOnce(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.conf")
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`,
`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","reload-on":["m.conf"]},
{"id":"m.other","type":"service","unit":"other.service","state":"running"}`)
var commands []string
var said []string
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"),
recordingServices(&commands), func(line string) { said = append(said, line) }, nil, nil)
if err != nil {
t.Fatal(err)
}
if n := count(commands, "systemctl reload m.service"); n != 1 {
t.Errorf("the program was reloaded %d times; commands were %v", n, commands)
}
joined := strings.Join(commands, "\n")
if strings.Contains(joined, "restart m.service") || strings.Contains(joined, "stop m.service") {
t.Errorf("the program was restarted, not reloaded: %v", commands)
}
if strings.Contains(joined, "other.service") {
t.Errorf("a service that does not read the file was touched: %v", commands)
}
if _, ok := state.Find("m.daemon"); !ok {
t.Errorf("the reloaded service was not recorded")
}
// The log says the path and the digest, never the content.
for _, line := range said {
if strings.Contains(line, "level = 1") {
t.Errorf("the content was said: %q", line)
}
}
if !strings.Contains(strings.Join(said, "\n"), digestOf("level = 1\n")) {
t.Errorf("the digest was not said: %q", said)
}
// The same content again moves nothing, and reloads nothing.
commands = nil
report, _, err := WriteFilled(context.Background(), archHost(t), d, state, path, []byte("level = 1\n"),
recordingServices(&commands), nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if report.Outcomes[0].Action != "unchanged" || count(commands, "reload") != 0 {
t.Errorf("an unchanged write reloaded: %v %+v", commands, report.Outcomes)
}
}
func TestAWriteUnderRestartOnRestartsItsProgram(t *testing.T) {
path := filepath.Join(t.TempDir(), "daemon.conf")
d := filledDecl(t, path, `"content":"","owner-at-runtime":"owned"`,
`{"id":"m.daemon","type":"service","unit":"m.service","state":"running","restart-on":["m.conf"]}`)
var commands []string
if _, _, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("level = 1\n"),
recordingServices(&commands), nil, nil, nil); err != nil {
t.Fatal(err)
}
// A restart is a stop and a start, once.
stops, starts := count(commands, "systemctl stop m.service"), count(commands, "systemctl start m.service")
if stops != 1 || starts != 1 {
t.Errorf("the program was stopped %d and started %d times; commands were %v", stops, starts, commands)
}
}
// A running container that reads the file at creation is recreated, as an apply recreates it for a changed file.
func TestAWriteOfAFileAContainerReadsRecreatesIt(t *testing.T) {
env := filepath.Join(t.TempDir(), "forge.env")
d := filledDecl(t, env, `"content":"","mode":"0600","owner-at-runtime":"owned"`,
`{"id":"forge.server","type":"container","name":"forge","image":"`+pinned+`","env-file":["`+env+`"]}`)
m := &machine{containers: map[string]*fakeContainer{}}
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, env, []byte("PORT=5432\n"),
m.run, nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if !m.did("docker run") {
t.Fatalf("the container was not created: %v", m.asked)
}
m.asked = nil
report, _, err := WriteFilled(context.Background(), archHost(t), d, state, env, []byte("PORT=5433\n"), m.run,
nil, nil, nil)
if err != nil {
t.Fatal(err)
}
if !m.removed("forge") || !m.did("docker run") {
t.Fatalf("the container kept what it read before the write: %v", m.asked)
}
if o := outcomeOf(report, "forge.server"); o.Action != "updated" || !strings.Contains(o.Detail, env) {
t.Errorf("the recreate did not say which file changed: %+v", o)
}
}
func count(lines []string, prefix string) int {
n := 0
for _, l := range lines {
if strings.Contains(l, prefix) {
n++
}
}
return n
}
// Two resources declaring one filled path — two modules' blocks in one file — are refused: the request carries the
// content only, so it cannot say whose block it is, and nothing is written.
func TestAFilledPathTwoResourcesDeclareIsRefused(t *testing.T) {
path := filepath.Join(t.TempDir(), "hosts")
if err := os.WriteFile(path, []byte(workstationHosts), 0o644); err != nil {
t.Fatal(err)
}
d := parseTrusted(t, fmt.Sprintf(`{"declaration":1,"filled_files":[%q],"resources":[
{"id":"a.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"},
{"id":"b.hosts","type":"file","path":%q,"into":"block","content":"10.42.0.2 g14\n","owner-at-runtime":"managed"}
]}`, path, path, path))
_, state, err := WriteFilled(context.Background(), archHost(t), d, store.State{}, path, []byte("6.6.6.6 evil\n"),
noServices, nil, nil, nil)
if err == nil || !strings.Contains(err.Error(), "2 resources declare") {
t.Fatalf("a path two resources declare was not refused: %v", err)
}
if got := readText(t, path); got != workstationHosts || len(state.Resources) != 0 {
t.Errorf("something was written or recorded: %q %+v", got, state.Resources)
}
}
+12 -6
View File
@@ -318,6 +318,17 @@ func heldRoots(verdicts map[string]libraryVerdict, libs []declaration.Library) m
return out
}
// heldLibraries is the root of every library whose disk or mount is not there, with its name, judged now and read
// only: what a full apply holds its containers for (ADR 0328 §9), for one that judges without applying libraries.
func heldLibraries(ctx context.Context, libs []declaration.Library) map[string]string {
verdicts := map[string]libraryVerdict{}
for _, l := range libs {
rec, _ := libraryRecord(l.Library)
verdicts[l.Library] = assessLibrary(ctx, l, rec)
}
return heldRoots(verdicts, libs)
}
// mountsHeld is the library a container waits for: one whose root it mounts, or mounts below, is held.
func mountsHeld(c *declaration.Container, held map[string]string) string {
for _, v := range c.Volumes {
@@ -341,12 +352,7 @@ func WaitingOnLibrary(d *declaration.Declaration) map[string]bool {
if d == nil || len(d.Libraries) == 0 {
return out
}
verdicts := map[string]libraryVerdict{}
for _, l := range d.Libraries {
rec, _ := libraryRecord(l.Library)
verdicts[l.Library] = assessLibrary(context.Background(), l, rec)
}
held := heldRoots(verdicts, d.Libraries)
held := heldLibraries(context.Background(), d.Libraries)
for _, r := range d.Resources {
if c, ok := r.(*declaration.Container); ok && mountsHeld(c, held) != "" {
out[c.ID] = true
+129
View File
@@ -0,0 +1,129 @@
package apply
import (
"context"
"fmt"
"os"
"path/filepath"
"testing"
"github.com/novox/mesh-host/internal/store"
)
// Defends novox/hq ADR 0349 §5 (issue 667): a file that says who owns it at runtime is written as the
// form it names always was — owned as a whole file whose drift is written back, seed as create-once,
// managed as only its keys or its marked block. Each is applied beside its old form, through the same
// drift, and the two must end the same: the same actions, the same bytes on disk.
// ownershipCase is one ownership word and the old form it maps onto.
type ownershipCase struct {
name string
// said and old are the resource's fields after id, type and path: with the word, and in the old form.
said, old string
// before is what is at the path before the first apply; "" leaves the path absent.
before string
// drift is what the machine writes there between the two applies.
drift string
// first and second are the actions the old form is known to report.
first, second string
}
func TestEachOwnershipBehavesAsItsOldForm(t *testing.T) {
for _, c := range []ownershipCase{
{
name: "owned is a whole file whose drift is written back",
said: `"content":"correct\n","mode":"0644","owner-at-runtime":"owned"`,
old: `"content":"correct\n","mode":"0644"`,
drift: "someone edited this\n",
first: "created", second: "corrected",
},
{
name: "seed is written once and then the program's",
said: `"content":"user default on\n","mode":"0600","owner-at-runtime":"seed"`,
old: `"content":"user default on\n","mode":"0600","create-once":true`,
drift: "user default on\nuser app-one on\n",
first: "created", second: "kept",
},
{
name: "managed json sets only its keys",
said: `"into":"json","content":"{\"insecure-registries\":[\"10.42.0.1:5000\"]}","owner-at-runtime":"managed"`,
old: `"into":"json","content":"{\"insecure-registries\":[\"10.42.0.1:5000\"]}"`,
before: machinesOwn,
drift: `{"data-root":"/srv/other","log-opts":{"max-size":"10m"},"insecure-registries":["192.0.2.7:5000"]}`,
first: "updated", second: "corrected",
},
{
name: "managed block holds only its region",
said: `"into":"block","content":"10.42.0.1 ace\n","owner-at-runtime":"managed"`,
old: `"into":"block","content":"10.42.0.1 ace\n"`,
before: workstationHosts,
drift: workstationHosts + "\n" + marked("f", "6.6.6.6 evil\n") + "192.0.2.9 the-machines-own\n",
first: "updated", second: "corrected",
},
} {
t.Run(c.name, func(t *testing.T) {
saidActions, saidBytes := applyThroughDrift(t, c, c.said)
oldActions, oldBytes := applyThroughDrift(t, c, c.old)
if saidActions != oldActions {
t.Errorf("with the word the actions were %s; the old form's were %s", saidActions, oldActions)
}
if want := c.first + " " + c.second; oldActions != want {
t.Errorf("the old form's actions were %s; known to be %s", oldActions, want)
}
if saidBytes != oldBytes {
t.Errorf("with the word the file ends\n%q\nthe old form's ends\n%q", saidBytes, oldBytes)
}
})
}
}
// applyThroughDrift applies one file, lets the machine write the case's drift into it, applies again,
// and says the two actions and what the file holds at the end.
func applyThroughDrift(t *testing.T, c ownershipCase, fields string) (string, string) {
t.Helper()
path := filepath.Join(t.TempDir(), "the-file")
if c.before != "" {
if err := os.WriteFile(path, []byte(c.before), 0o644); err != nil {
t.Fatal(err)
}
}
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[{"id":"f","type":"file","path":%q,%s}]}`, path, fields))
first, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte(c.drift), 0o644); err != nil {
t.Fatal(err)
}
second, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
return first.Outcomes[0].Action + " " + second.Outcomes[0].Action, readText(t, path)
}
// A seed said by the word alone, with no create-once beside it, is still kept once present: the word is
// enough, and the file the program grew is not written back.
func TestASeedByTheWordAloneKeepsWhatGrew(t *testing.T) {
path := filepath.Join(t.TempDir(), "acl.conf")
d := parse(t, fmt.Sprintf(`{"declaration":1,"resources":[
{"id":"acl","type":"file","path":%q,"content":"user default on\n","owner-at-runtime":"seed"}
]}`, path))
_, state, err := Apply(context.Background(), archHost(t), d, store.State{}, store.OriginDeclared, noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, []byte("grown\n"), 0o600); err != nil {
t.Fatal(err)
}
report, _, err := Apply(context.Background(), archHost(t), d, state, store.OriginDeclared, noServices, nil, nil)
if err != nil {
t.Fatal(err)
}
if got := report.Outcomes[0].Action; got != "kept" {
t.Errorf("a seed was %q", got)
}
if got := readText(t, path); got != "grown\n" {
t.Errorf("what grew in the seed was written over: %q", got)
}
}
+112
View File
@@ -0,0 +1,112 @@
package apply
// A core credential the vault sealed to this machine (novox/hq ADR 0349 §16, issue 755): the controller carries the
// vault's box (ADR 0206) in the declaration and cannot open it; this machine opens it with its sealing key and writes
// the value, as it writes a value the controller sealed.
import (
"context"
"encoding/json"
"os"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/seal"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/store"
)
// vaultSealedTo is what the vault answers the controller: the value in the ADR 0206 box, sealed to the machine's
// sealing key as SPKI PEM.
func vaultSealedTo(t *testing.T, key identity.SealingKey, value string) seal.Box {
t.Helper()
pub, err := key.PublicPEM()
if err != nil {
t.Fatal(err)
}
box, err := seal.Seal(value, pub)
if err != nil {
t.Fatal(err)
}
return box
}
func vaultSealedFile(t *testing.T, path string, box seal.Box) *declaration.Declaration {
t.Helper()
raw := map[string]any{"declaration": 1, "resources": []map[string]any{
{"id": "broker", "type": "file", "path": path, "vault-sealed": box},
}}
body, _ := json.Marshal(raw)
d, err := declaration.Parse(body)
if err != nil {
t.Fatal(err)
}
return d
}
func boxOpener(key identity.SealingKey) Unseal {
return func(sealed string) ([]byte, error) { return key.Open(sealed) }
}
func TestAFileTheVaultSealedToThisMachineIsOpenedAndWrittenUnreadableToOthers(t *testing.T) {
key, err := identity.GenerateSealingKey()
if err != nil {
t.Fatal(err)
}
path := t.TempDir() + "/broker"
d := vaultSealedFile(t, path, vaultSealedTo(t, key, "the-provisioners-bus-password"))
report, state, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginCarried, noServices, nil, boxOpener(key))
if err != nil {
t.Fatal(err)
}
on, err := os.ReadFile(path)
if err != nil || string(on) != "the-provisioners-bus-password" {
t.Fatalf("the file holds %q, %v", on, err)
}
if info, _ := os.Stat(path); info.Mode().Perm() != 0o600 {
t.Errorf("a credential landed mode %o", info.Mode().Perm())
}
said, _ := json.Marshal(report)
kept, _ := json.Marshal(state)
for what, blob := range map[string][]byte{"the report": said, "the node's state": kept} {
if strings.Contains(string(blob), "the-provisioners-bus-password") {
t.Fatalf("%s carries the value in the clear", what)
}
}
}
func TestAVaultBoxSealedToAnotherMachineIsRefusedAndNothingWritten(t *testing.T) {
mine, _ := identity.GenerateSealingKey()
theirs, _ := identity.GenerateSealingKey()
path := t.TempDir() + "/broker"
d := vaultSealedFile(t, path, vaultSealedTo(t, theirs, "not for you"))
if _, _, err := Apply(context.Background(), archHost(t), d, store.State{},
store.OriginCarried, noServices, nil, boxOpener(mine)); err == nil {
t.Fatal("a box sealed to another machine was applied")
}
if _, err := os.Stat(path); err == nil {
t.Fatal("something was written before the failure")
}
}
func TestAVaultSealedFileSaysWhatIsInItOnce(t *testing.T) {
key, _ := identity.GenerateSealingKey()
box, _ := json.Marshal(vaultSealedTo(t, key, "x"))
for what, extra := range map[string]string{
"content": `"content":"a"`, "sealed": `"sealed":"b"`, "bytes": `"bytes":"Yw=="`,
"secrets": `"content":"${secret:x}","secrets":{"x":"c"}`, "into": `"into":"json","content":"{}"`,
} {
_, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x",` +
`"vault-sealed":` + string(box) + `,` + extra + `}]}`))
if err == nil {
t.Errorf("a vault-sealed file that also says %s was accepted", what)
}
}
if _, err := declaration.Parse([]byte(`{"declaration":1,"resources":[{"id":"f","type":"file","path":"/etc/x",` +
`"vault-sealed":{"v":2,"eph":"a","iv":"b","tag":"c","ct":"d"}}]}`)); err == nil {
t.Error("a box of a version the SDK does not open was accepted")
}
}
+169 -3
View File
@@ -19,6 +19,8 @@ import (
"sort"
"strconv"
"strings"
"git.novox.be/novox/mesh-sdk/go/seal"
)
// Version is the vocabulary this host speaks. A declaration naming any other version is
@@ -205,6 +207,12 @@ type File struct {
// by looking rather than by knowing which field won.
Sealed string `json:"sealed,omitempty"`
// VaultSealed is content the vault sealed to this node's sealing key, in the mesh's sealed box (novox/hq ADR
// 0206): a core module's own credential, which the controller carries in the declaration and cannot open
// (ADR 0349 §16, issue 755). The same promise as Sealed, from the vault rather than the controller; sent only to
// an engine that states VaultBoxContract. Exclusive with Content, Sealed and Bytes.
VaultSealed *seal.Box `json:"vault-sealed,omitempty"`
// Trusted is the catalogue's word on whether the settings this file's content took are trusted (novox/hq
// issue 339). It is the controller's to act on, and a controller takes it out before it sends a declaration.
// Accepted here, and nothing is done with it, so that a controller that passes it on — an older one, or one
@@ -237,11 +245,48 @@ type File struct {
// Owner is the user this belongs to, by name. Absent means root.
Owner string `json:"owner,omitempty"`
// OwnerAtRuntime says who owns the file once it is placed (novox/hq ADR 0349 §5): "owned", the mesh's
// whole, written back whenever it differs; "seed", written once and then the program's; or "managed",
// only its named keys or its marked block the mesh's, the program keeping the rest.
//
// **Three words for what the forms above already do.** Owned is an ordinary file, seed is create-once
// and managed is a file written into, so the word changes no write: a file that says it is applied
// exactly as its old form is, and one that does not say is owned by its old form (Ownership). The word
// is what the provisioner's files say of themselves, which the old forms only implied. Seed is enough
// alone; managed still needs `into`, which says the format. A word that contradicts the form is refused.
OwnerAtRuntime string `json:"owner-at-runtime,omitempty"`
}
// The words a file says who owns it at runtime with (novox/hq ADR 0349 §5).
const (
RuntimeOwned = "owned"
RuntimeSeed = "seed"
RuntimeManaged = "managed"
)
// Ownership is who owns the file at runtime: the word it says, or the one its old form implies — a file
// written into is managed, one created once is a seed, and any other is owned.
func (f *File) Ownership() string {
switch {
case f.OwnerAtRuntime != "":
return f.OwnerAtRuntime
case f.Into != "":
return RuntimeManaged
case f.CreateOnce:
return RuntimeSeed
default:
return RuntimeOwned
}
}
// Seeded reports whether the file is written only when absent: created once, by create-once or by the
// word seed.
func (f *File) Seeded() bool { return f.CreateOnce || f.OwnerAtRuntime == RuntimeSeed }
// Secret reports whether this file arrived sealed, which is what decides both that it must be
// opened before writing and that its contents must never appear in a report.
func (f *File) Secret() bool { return f.Sealed != "" }
func (f *File) Secret() bool { return f.Sealed != "" || f.VaultSealed != nil }
func (f *File) Identity() string { return f.ID }
func (f *File) Kind() Type { return TypeFile }
@@ -314,12 +359,13 @@ func (f *File) validate(where string, _ bool) []string {
where, f.At))
}
if f.Into == IntoJSON || f.Into == IntoBlock {
if f.Sealed != "" || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
if f.Secret() || f.Bytes != "" || len(f.Secrets) > 0 || f.CreateOnce {
problems = append(problems, where+
": a file written into says only its part, in content — not sealed, bytes, "+
"secrets or create-once")
}
}
problems = append(problems, f.checkOwnership(where)...)
var said []string
for name, value := range map[string]string{
"content": f.Content, "sealed": f.Sealed, "bytes": f.Bytes,
@@ -328,6 +374,16 @@ func (f *File) validate(where string, _ bool) []string {
said = append(said, name)
}
}
if f.VaultSealed != nil {
said = append(said, "vault-sealed")
if f.VaultSealed.V != seal.Version {
problems = append(problems, fmt.Sprintf("%s: vault-sealed is a sealed box of version %d, and this "+
"node-engine opens version %d", where, f.VaultSealed.V, seal.Version))
}
if len(f.Secrets) > 0 {
problems = append(problems, where+": a file the vault sealed whole has no holes for secrets")
}
}
if len(said) > 1 {
sort.Strings(said)
problems = append(problems, where+
@@ -361,6 +417,42 @@ func (f *File) validate(where string, _ bool) []string {
return append(problems, checkMode(where, f.Mode)...)
}
// checkOwnership refuses a word that contradicts the file's form (novox/hq ADR 0349 §5): owned is a whole
// file written back, so not created once nor written into; a seed is a whole file created once, so not
// written into; managed is a file written into, which says its format. Managed and created once together
// is already refused by the rule that a file written into is not created once.
func (f *File) checkOwnership(where string) []string {
switch f.OwnerAtRuntime {
case "":
case RuntimeOwned:
var also []string
if f.CreateOnce {
also = append(also, "create-once")
}
if f.Into != "" {
also = append(also, "into")
}
if len(also) > 0 {
return []string{fmt.Sprintf("%s: owned at runtime is the mesh's whole file, written back whenever it "+
"differs, and this also says %s", where, strings.Join(also, " and "))}
}
case RuntimeSeed:
if f.Into != "" {
return []string{where + ": a seed is a whole file written once, and this says into; a file the " +
"program shares with the mesh is managed"}
}
case RuntimeManaged:
if f.Into == "" {
return []string{where + ": a managed file names the format its keys or block are written into, " +
"with into \"json\" or \"block\""}
}
default:
return []string{fmt.Sprintf("%s: owner-at-runtime %q; a placed file is %q, %q or %q, or omits it to be "+
"owned by its form", where, f.OwnerAtRuntime, RuntimeOwned, RuntimeSeed, RuntimeManaged)}
}
return nil
}
// User is a login on the machine.
//
// The thing that makes a shell, a chat client or a desktop expressible at all: each is a package
@@ -1565,6 +1657,71 @@ type Declaration struct {
// this is its home, whether the operator's warrant to take it was carried out, the sub-paths its modules ask for,
// and the tool runner's account to put in its group (§11). Absent is a controller older than ADR 0328.
Libraries []Library
// Filled is the paths of the files the provisioner fills on this machine (novox/hq ADR 0349 §4, §5): each a
// file this declaration declares, which says who owns it at runtime. The provisioner's client has the
// node-engine write only these. Absent is a controller that hands no file to the provisioner, and every
// file is written from the declaration as before.
Filled []string
// Provisioner is the account the provisioner's client runs as on this machine, by name (novox/hq ADR 0349 §4):
// the only account whose write of a filled file the node-engine's local write socket accepts. Absent is a
// controller that names none, and every such write is refused.
Provisioner string
}
// checkProvisioner refuses a provisioner client's account that is no account name, or is root: the account whose
// writes the node-engine accepts must be one that could not write the file without it.
func checkProvisioner(name string) []string {
switch {
case name == "":
return nil
case !accountName.MatchString(name):
return []string{fmt.Sprintf("provisioner_account %q is not an account name", name)}
case name == "root":
return []string{"provisioner_account is root; the provisioner's client runs as an account of its own"}
}
return nil
}
// Fills reports whether the provisioner fills the file at path on this machine.
func (d *Declaration) Fills(path string) bool { return slices.Contains(d.Filled, path) }
// checkFilled refuses a filled path that is not a declared file saying who owns it at runtime, or that is named
// twice or empty. Every file declared at the path must say it: the provisioner writes the path, and a file there
// the mesh writes some other way would be a second writer of it (ADR 0349 §5).
func checkFilled(filled []string, resources []Resource) []string {
var problems []string
files := map[string][]*File{}
for _, r := range resources {
if f, ok := r.(*File); ok {
files[f.Path] = append(files[f.Path], f)
}
}
seen := map[string]bool{}
for _, path := range filled {
switch {
case path == "":
problems = append(problems, "filled_files names a file with no path")
continue
case seen[path]:
problems = append(problems, fmt.Sprintf("filled_files names %q twice", path))
continue
}
seen[path] = true
if len(files[path]) == 0 {
problems = append(problems, fmt.Sprintf("filled_files names %q, and this declaration declares no "+
"file there", path))
continue
}
for _, f := range files[path] {
if f.OwnerAtRuntime == "" {
problems = append(problems, fmt.Sprintf("resource %q: the provisioner fills %q, and a file it "+
"fills says who owns it at runtime with owner-at-runtime", f.ID, path))
}
}
}
return problems
}
// Identity is one module's identity, as the controller allocated it (ADR 0328 §4): its user `mesh-<module>` with
@@ -1904,6 +2061,12 @@ type envelope struct {
// sends them only to a host whose health says link.LibraryContract.
Groups []Group `json:"groups,omitempty"`
Libraries []Library `json:"libraries,omitempty"`
// Filled is optional too (novox/hq ADR 0349 §5), as is a file's `owner-at-runtime`. **An older host refuses
// these keys**; a controller sends them only to a host whose health says link.OwnershipContract.
Filled []string `json:"filled_files,omitempty"`
// Provisioner is optional too (novox/hq ADR 0349 §4). **An older host refuses this key**; a controller sends it
// only to a host whose health says link.ProvisionerContract.
Provisioner string `json:"provisioner_account,omitempty"`
}
func parse(raw []byte, allowActions bool) (*Declaration, error) {
@@ -1922,8 +2085,10 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
d := &Declaration{Version: env.Version, For: env.For, Adoption: env.Adoption, Sequence: env.Sequence,
Epoch: env.Epoch, LeftOut: env.LeftOut, Generation: env.Generation, PutBack: env.PutBack,
Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries}
Identities: env.Identities, Groups: env.Groups, Libraries: env.Libraries, Filled: env.Filled,
Provisioner: env.Provisioner}
problems := checkIdentities(env.Identities, env.Groups)
problems = append(problems, checkProvisioner(env.Provisioner)...)
problems = append(problems, checkLibraries(env.Groups, env.Libraries)...)
if env.Generation < 0 {
// Below zero would read as "none claimed" and pass every refusal of an older generation.
@@ -2015,6 +2180,7 @@ func parse(raw []byte, allowActions bool) (*Declaration, error) {
problems = append(problems, checkWhileStopped(d.Resources)...)
problems = append(problems, checkAdoption(env.Adoption, d.Resources, allowActions)...)
problems = append(problems, checkReplaces(d.Resources)...)
problems = append(problems, checkFilled(env.Filled, d.Resources)...)
if env.Adoption == nil {
for _, r := range d.Resources {
if r.Kind() == TypeOpening {
+130
View File
@@ -0,0 +1,130 @@
package declaration
import (
"strings"
"testing"
)
// Defends novox/hq ADR 0349 §5 (issue 667): a file the provisioner places says who owns it at runtime
// — owned, seed or managed — and the declaration names the files the provisioner fills. The words are
// read and judged here before any controller sends them.
func TestAFileSaysWhoOwnsItAtRuntime(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,"resources":[
{"id":"o","type":"file","path":"/etc/o","content":"a","owner-at-runtime":"owned"},
{"id":"s","type":"file","path":"/etc/s","content":"a","owner-at-runtime":"seed"},
{"id":"s2","type":"file","path":"/etc/s2","content":"a","owner-at-runtime":"seed","create-once":true},
{"id":"mj","type":"file","path":"/etc/mj","into":"json","content":"{\"k\":1}","owner-at-runtime":"managed"},
{"id":"mb","type":"file","path":"/etc/mb","into":"block","content":"a\n","owner-at-runtime":"managed"}
]}`))
if err != nil {
t.Fatal(err)
}
want := map[string]string{"o": RuntimeOwned, "s": RuntimeSeed, "s2": RuntimeSeed, "mj": RuntimeManaged,
"mb": RuntimeManaged}
for _, r := range d.Resources {
f := r.(*File)
if f.OwnerAtRuntime != want[f.ID] || f.Ownership() != want[f.ID] {
t.Errorf("%s: read as %q, ownership %q; want %q", f.ID, f.OwnerAtRuntime, f.Ownership(), want[f.ID])
}
}
}
// A file that does not say is owned by its old form, so the word means exactly what that form does.
func TestAFileThatDoesNotSayIsOwnedByItsOldForm(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,"resources":[
{"id":"o","type":"file","path":"/etc/o","content":"a"},
{"id":"s","type":"file","path":"/etc/s","content":"a","create-once":true},
{"id":"mj","type":"file","path":"/etc/mj","into":"json","content":"{}"},
{"id":"mb","type":"file","path":"/etc/mb","into":"block","content":"a\n"}
]}`))
if err != nil {
t.Fatal(err)
}
want := map[string]string{"o": RuntimeOwned, "s": RuntimeSeed, "mj": RuntimeManaged, "mb": RuntimeManaged}
for _, r := range d.Resources {
f := r.(*File)
if f.OwnerAtRuntime != "" || f.Ownership() != want[f.ID] {
t.Errorf("%s: said %q, ownership %q; want unsaid and %q", f.ID, f.OwnerAtRuntime, f.Ownership(), want[f.ID])
}
if f.Seeded() != (want[f.ID] == RuntimeSeed) {
t.Errorf("%s: seeded %v", f.ID, f.Seeded())
}
}
}
func TestAnOwnershipThatContradictsTheFileIsRefused(t *testing.T) {
for name, c := range map[string]struct{ resource, refusal string }{
"another word": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"mine"}`,
`owner-at-runtime "mine"`},
"owned and create-once": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"owned","create-once":true}`,
"create-once"},
"owned and into": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","owner-at-runtime":"owned"}`,
"into"},
"seed and into": {`{"id":"f","type":"file","path":"/etc/x","into":"block","content":"a","owner-at-runtime":"seed"}`,
"into"},
"managed and whole": {`{"id":"f","type":"file","path":"/etc/x","content":"a","owner-at-runtime":"managed"}`,
"into"},
"managed and create-once": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","owner-at-runtime":"managed","create-once":true}`,
"create-once"},
// The rule that was there before stays: a file written into is never sealed or created once.
"managed and sealed": {`{"id":"f","type":"file","path":"/etc/x","into":"json","content":"{}","sealed":"x","owner-at-runtime":"managed"}`,
"sealed"},
} {
_, err := Parse([]byte(`{"declaration":1,"resources":[` + c.resource + `]}`))
if err == nil || !strings.Contains(err.Error(), c.refusal) {
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
}
}
}
func TestTheDeclarationNamesTheFilesTheProvisionerFills(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,"filled_files":["/etc/a","/etc/b"],"resources":[
{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"},
{"id":"b","type":"file","path":"/etc/b","into":"block","content":"b\n","owner-at-runtime":"managed"},
{"id":"c","type":"file","path":"/etc/c","content":"c"}
]}`))
if err != nil {
t.Fatal(err)
}
if len(d.Filled) != 2 || d.Filled[0] != "/etc/a" || d.Filled[1] != "/etc/b" {
t.Fatalf("filled files read as %v", d.Filled)
}
if !d.Fills("/etc/a") || !d.Fills("/etc/b") || d.Fills("/etc/c") {
t.Errorf("Fills: a %v, b %v, c %v", d.Fills("/etc/a"), d.Fills("/etc/b"), d.Fills("/etc/c"))
}
// A declaration that names none fills none, as every declaration before this did.
d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"c","type":"file","path":"/etc/c","content":"c"}]}`))
if err != nil {
t.Fatal(err)
}
if d.Filled != nil || d.Fills("/etc/c") {
t.Errorf("a declaration naming no filled files fills %v", d.Filled)
}
}
func TestAFilledFileMustBeADeclaredFileThatSaysItsOwner(t *testing.T) {
for name, c := range map[string]struct{ filled, resources, refusal string }{
"not declared": {`["/etc/none"]`,
`{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`,
`"/etc/none"`},
"a directory, not a file": {`["/etc/d"]`,
`{"id":"d","type":"directory","path":"/etc/d"}`,
`"/etc/d"`},
"says no owner": {`["/etc/a"]`,
`{"id":"a","type":"file","path":"/etc/a","content":"a"}`,
"owner-at-runtime"},
"named twice": {`["/etc/a","/etc/a"]`,
`{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`,
"twice"},
"no path": {`[""]`,
`{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}`,
"no path"},
} {
_, err := Parse([]byte(`{"declaration":1,"filled_files":` + c.filled + `,"resources":[` + c.resources + `]}`))
if err == nil || !strings.Contains(err.Error(), c.refusal) {
t.Errorf("%s: want a refusal naming %q, got %v", name, c.refusal, err)
}
}
}
+37
View File
@@ -0,0 +1,37 @@
package declaration
import (
"strings"
"testing"
)
// Defends novox/hq ADR 0349 §4 (issue 668): the declaration names the account the provisioner's client runs as,
// the only one whose write of a filled file the node-engine accepts. Read here before any controller sends it.
func TestTheDeclarationNamesTheProvisionersAccount(t *testing.T) {
d, err := Parse([]byte(`{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/a"],
"resources":[{"id":"a","type":"file","path":"/etc/a","content":"a","owner-at-runtime":"owned"}]}`))
if err != nil {
t.Fatal(err)
}
if d.Provisioner != "mesh-node-provisioner" {
t.Errorf("the provisioner's account read as %q", d.Provisioner)
}
d, err = Parse([]byte(`{"declaration":1,"resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`))
if err != nil {
t.Fatal(err)
}
if d.Provisioner != "" {
t.Errorf("a declaration naming no account names %q", d.Provisioner)
}
}
func TestAProvisionersAccountThatIsNoAccountIsRefused(t *testing.T) {
for name, account := range map[string]string{"root": "root", "not a name": "a b", "a path": "../x"} {
_, err := Parse([]byte(`{"declaration":1,"provisioner_account":"` + account + `",
"resources":[{"id":"a","type":"file","path":"/etc/a","content":"a"}]}`))
if err == nil || !strings.Contains(err.Error(), "provisioner_account") {
t.Errorf("%s: want a refusal naming provisioner_account, got %v", name, err)
}
}
}
+512
View File
@@ -0,0 +1,512 @@
// Package filling is the node-engine's narrow local write verb (novox/hq ADR 0349 §4, design 57 P2): a socket on
// this machine through which the provisioner's client has a file it filled written, and nothing else.
//
// **The client fills; the node-engine writes.** The client renders a module's template with the values each domain
// owns and hands the node-engine the content; the node-engine writes it as the declared file, with the declared
// file's path, owner, mode and ownership at runtime, and makes what reads it take the change. So the one process that
// writes the machine's files is still the node-engine, and the client needs no right to write any of them.
//
// **Narrow by three checks, each from something the client cannot say for itself.** The account asking, from the
// kernel (SO_PEERCRED), in this machine's own namespaces, must be the account the node's current declaration names
// as the provisioner's client's; the path, clean and absolute, must be one the declaration names as filled; and the
// write takes the apply's lock, so a write and an apply never write one file at once. While the declaration names no
// account — every declaration before the provisioner's client has one — every write is refused.
//
// **No listening control surface** (novox/hq ADR 0004): the socket has no network address. And the content is never
// said: the journal has the path, the account and the content's digest.
package filling
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net"
"os"
"os/user"
"path/filepath"
"sort"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/novox/mesh-host/internal/declaration"
"github.com/novox/mesh-host/internal/meshcli"
)
// Socket is where the node-engine accepts a write of a filled file, beside mesh-cli's socket.
const Socket = "/run/nox-mesh-host/fill.sock"
// MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive.
const MaxRequest = 8 << 20
// Request is one write, sent as one JSON object on one line: the path of a filled file and what the client filled
// into it (base64 in the JSON). The content only: the declaration says everything else about the file.
type Request struct {
Path string `json:"path"`
Content []byte `json:"content"`
}
// Answer is what came of one request. Refused is a write that was not attempted, with why; Failed one that was
// attempted and did not complete, with why — the file, or something that reads it. Otherwise Action is what the
// write did to the file (created, updated, corrected, unchanged, kept) and Digest what the file holds now, as the
// node-engine records it.
type Answer struct {
Path string `json:"path,omitempty"`
Action string `json:"action,omitempty"`
Digest string `json:"digest,omitempty"`
Refused string `json:"refused,omitempty"`
Failed string `json:"failed,omitempty"`
}
// Refusal is an answer that nothing was written, and why.
func Refusal(path, why string) Answer { return Answer{Path: path, Refused: why} }
// Writer is the node-engine's side of the socket.
type Writer interface {
// Accepts judges the asking account before its request is read: nil when the current declaration names it as
// the provisioner's client's.
Accepts(uid uint32) error
// Write judges the request and writes it, holding the apply's lock.
Write(ctx context.Context, uid uint32, req Request) Answer
}
// Engine is the Writer the node-engine serves: the declaration it applies now, read again under the apply's lock.
type Engine struct {
// Declared reads the declaration this node applies now, verified.
Declared func() (*declaration.Declaration, error)
// UIDOf is the user id of an account, by name, from this machine's user database.
UIDOf func(account string) (uint32, error)
// Lock takes the apply's lock and returns what releases it: the same the apply takes, so a write and an apply
// are one at a time.
Lock func() (func(), error)
// Place writes the content into the filled file at path, as the declared file, and makes what reads it take the
// change. Called holding the lock, with the declaration read under it.
Place func(ctx context.Context, d *declaration.Declaration, path string, content []byte) (action, digest string,
err error)
}
// Accepts reads the declaration and judges the account.
func (e *Engine) Accepts(uid uint32) error {
d, err := e.Declared()
if err != nil {
return fmt.Errorf("this node's declaration cannot be read, so no write is accepted: %w", err)
}
return JudgeAccount(d, uid, e.UIDOf)
}
// Write takes the apply's lock, reads the declaration under it, judges the account and the path again — the
// declaration may have changed since the request was accepted — and writes.
func (e *Engine) Write(ctx context.Context, uid uint32, req Request) Answer {
if err := JudgeShape(req.Path); err != nil {
return Refusal(req.Path, err.Error())
}
unlock, err := e.Lock()
if err != nil {
return Refusal(req.Path, "the apply's lock could not be taken, so nothing was written: "+err.Error())
}
defer unlock()
d, err := e.Declared()
if err != nil {
return Refusal(req.Path, "this node's declaration cannot be read, so nothing was written: "+err.Error())
}
if err := JudgeAccount(d, uid, e.UIDOf); err != nil {
return Refusal(req.Path, err.Error())
}
if err := JudgePath(d, req.Path); err != nil {
return Refusal(req.Path, err.Error())
}
action, digest, err := e.Place(ctx, d, req.Path, req.Content)
if err != nil {
return Answer{Path: req.Path, Action: action, Digest: digest, Failed: err.Error()}
}
return Answer{Path: req.Path, Action: action, Digest: digest}
}
// JudgeAccount refuses every account but the one the declaration names as the provisioner's client's, by its user
// id here — and every account while the declaration names none.
func JudgeAccount(d *declaration.Declaration, uid uint32, uidOf func(string) (uint32, error)) error {
if d.Provisioner == "" {
return errors.New("this node's declaration names no provisioner client's account, so no write is accepted")
}
want, err := uidOf(d.Provisioner)
if err != nil {
return fmt.Errorf("the provisioner client's account %s is not on this machine, so no write is accepted: %w",
d.Provisioner, err)
}
if want == 0 {
return fmt.Errorf("the provisioner client's account %s is root here, so no write is accepted", d.Provisioner)
}
if uid != want {
return fmt.Errorf("user id %d is not the provisioner client's account %s (user id %d), so nothing was written",
uid, d.Provisioner, want)
}
return nil
}
// JudgeShape refuses a path that is not clean and absolute: a relative path, `..`, a doubled or trailing slash.
// Compared as written with the declaration's, a path that is not clean could name a file the declaration does not.
func JudgeShape(path string) error {
switch {
case path == "":
return errors.New("the request names no path, so nothing was written")
case !filepath.IsAbs(path):
return fmt.Errorf("%q is not an absolute path, so nothing was written", path)
case filepath.Clean(path) != path:
return fmt.Errorf("%q is not a clean path (it is %q cleaned), so nothing was written", path,
filepath.Clean(path))
}
return nil
}
// JudgePath refuses a path that is not clean and absolute, or that the declaration does not name as filled. Where it
// leads — a link on the way, a place nothing is written — is judged by the write, by the apply's own rules.
func JudgePath(d *declaration.Declaration, path string) error {
if err := JudgeShape(path); err != nil {
return err
}
if !d.Fills(path) {
return fmt.Errorf("%s is not a file this node's declaration names as filled, so nothing was written", path)
}
return nil
}
// UIDOf is the user id of an account by name, from this machine's user database.
func UIDOf(account string) (uint32, error) {
u, err := user.Lookup(account)
if err != nil {
return 0, err
}
id, err := strconv.ParseUint(u.Uid, 10, 32)
if err != nil {
return 0, fmt.Errorf("user id %q: %w", u.Uid, err)
}
return uint32(id), nil
}
// peerOf and sameNamespaces are the kernel's word on the asking process, as mesh-cli's socket reads it. Variables so
// a test can say.
var (
peerOf = meshcli.PeerCredentials
sameNamespaces = meshcli.SameNamespaces
)
// within bounds how long an accepted connection may take to say its request, and refusedWithin how long a busy one
// is read and dropped before its refusal is written (so the client is not answered with a broken pipe). Variables so
// a test need not wait.
var (
within = 30 * time.Second
refusedWithin = 1 * time.Second
)
// atOnce bounds the accepted connections served at the same time: one client, writing one file at a time. One over
// is refused busy. busyWaitingAtOnce bounds how many busy refusals wait for their request at once, per Serve; over it
// a busy refusal is written at once, as mesh-cli's socket does. judgingAtOnce bounds the callers judged at once, and
// judgeWaiting the connections waiting to be judged — each an open file, so far below the engine's open-file limit;
// one over is closed at once, unread. Variables so a test can say.
var (
atOnce = 4
busyWaitingAtOnce = 8
judgingAtOnce = 16
judgeWaiting = 256
)
// refusalsSaidEvery is how often the refusals counted and not yet said are said: each kind of refusal of each account
// is said at once the first time, and what follows it within this time is counted and said at its end, or when the
// socket stops. A variable so a test need not wait.
var refusalsSaidEvery = time.Minute
// acceptBackoff is how long the socket waits after an accept that failed before it accepts again, at first and at
// most: a socket that stopped at the first such failure (too many open files, say) would stop the client's writes
// until the engine restarted. Variables so a test need not wait.
var (
acceptBackoff = 50 * time.Millisecond
acceptBackoffMax = 5 * time.Second
)
// acceptFrom accepts one connection. A variable so a test can make it fail.
var acceptFrom = func(l *net.UnixListener) (*net.UnixConn, error) { return l.AcceptUnix() }
// Serve listens on socket until ctx ends, and answers each connection: one request, one answer.
//
// The directory is made 0755 and the socket 0666, as mesh-cli's: who may write is judged per connection from the
// kernel's word, against the current declaration, so the socket needs no account that may not exist yet. **The
// caller is judged before it holds anything**: one the kernel does not say is the provisioner client's account is
// answered and closed at once, and only the client's connections are served or wait as busy, so no other account can
// hold the socket from it. A file at the socket's path that is not a socket is never removed: the engine says so and
// serves nothing.
func Serve(ctx context.Context, socket string, w Writer, say func(string)) error {
if say == nil {
say = func(string) {}
}
dir := filepath.Dir(socket)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("the filled files' socket directory: %w", err)
}
if problem := meshcli.NotOwnDirectory(dir); problem != "" {
return fmt.Errorf("filled files are not written on request on this machine: %s", problem)
}
if err := os.Chmod(dir, 0o755); err != nil {
return fmt.Errorf("the filled files' socket directory: %w", err)
}
if info, err := os.Lstat(socket); err == nil {
if info.Mode()&os.ModeSocket == 0 {
return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; filled "+
"files are not written on request until it is moved", socket)
}
if err := os.Remove(socket); err != nil {
return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err)
}
}
l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"})
if err != nil {
return fmt.Errorf("listening for filled files at %s: %w", socket, err)
}
if err := os.Chmod(socket, 0o666); err != nil {
l.Close()
return fmt.Errorf("opening the filled files' socket: %w", err)
}
go func() {
<-ctx.Done()
l.Close()
}()
slots := make(chan struct{}, atOnce)
// Judging reads the declaration: at most judging callers are judged at once, and the rest wait their turn — a
// few milliseconds each — rather than being refused, so a flood delays the client's write and never stops it.
judging := make(chan struct{}, judgingAtOnce)
var waiting atomic.Int64
busyWaiting := make(chan struct{}, busyWaitingAtOnce)
refusals := &refusalWords{say: say}
// Said once everything has stopped: what was counted is never lost with the socket.
defer refusals.flush()
var wg sync.WaitGroup
defer wg.Wait()
wg.Add(1)
go func() {
defer wg.Done()
tick := time.NewTicker(refusalsSaidEvery)
defer tick.Stop()
for {
select {
case <-ctx.Done():
return
case <-tick.C:
refusals.flush()
}
}
}()
backoff := time.Duration(0)
for {
conn, err := acceptFrom(l)
if err != nil {
if ctx.Err() != nil {
return nil
}
if errors.Is(err, net.ErrClosed) {
return fmt.Errorf("the filled files' socket was closed: %w", err)
}
// Said, waited for, and accepted again: the failure is the machine's for a moment, not the socket's.
backoff = min(max(2*backoff, acceptBackoff), acceptBackoffMax)
say(fmt.Sprintf("the filled files' socket could not accept a connection (%v); accepting again in %s",
err, backoff))
select {
case <-ctx.Done():
return nil
case <-time.After(backoff):
}
continue
}
backoff = 0
if waiting.Add(1) > int64(judgeWaiting) {
// Too many waiting to be judged already: this one is closed unread, and counted.
waiting.Add(-1)
refusals.said(^uint32(0), "overflow", "too many connections are waiting to be judged; closed unread")
conn.Close()
continue
}
wg.Add(1)
go func() {
defer wg.Done()
// Judged before it holds a slot or is waited for.
judging <- struct{}{}
uid, ok := judgePeer(conn, w, refusals)
<-judging
waiting.Add(-1)
if !ok {
return
}
readBy := time.Now().Add(within)
_ = conn.SetDeadline(readBy)
select {
case slots <- struct{}{}:
defer func() { <-slots }()
answer(ctx, conn, uid, w, say)
default:
busy(conn, readBy, busyWaiting, say)
}
}()
}
}
// judgePeer reads the asking account from the kernel and judges it. A caller that is not the provisioner client's
// account is answered with why and closed at once, its request never read: false.
func judgePeer(conn *net.UnixConn, w Writer, refusals *refusalWords) (uint32, bool) {
closeWith := func(uid uint32, kind, why string) (uint32, bool) {
refusals.said(uid, kind, why)
_ = conn.SetWriteDeadline(time.Now().Add(refusedWithin))
_ = json.NewEncoder(conn).Encode(Refusal("", why))
conn.Close()
return uid, false
}
uid, pid, err := peerOf(conn)
if err != nil {
return closeWith(^uint32(0), "unread", "the node-engine could not read which account is asking, so nothing was "+
"written: "+err.Error())
}
if err := sameNamespaces(pid); err != nil {
return closeWith(uid, "namespace", "the asking process does not run in this machine's own namespaces, so nothing was "+
"written: "+err.Error())
}
if err := w.Accepts(uid); err != nil {
return closeWith(uid, "account", err.Error())
}
return uid, true
}
// refusalWords says refused callers in the journal without letting a flood flood it: the first refusal of each kind
// for each account at once, and those that follow it counted and said by flush — on a timer and when the socket stops
// — so an account that stops still has its count said, and a refusal of another kind never hides behind one said.
type refusalWords struct {
mu sync.Mutex
say func(string)
pending map[refusalKey]*refusalCount
}
type refusalKey struct {
uid uint32
kind string
}
type refusalCount struct {
n int
why string
}
func (r *refusalWords) said(uid uint32, kind, why string) {
r.mu.Lock()
defer r.mu.Unlock()
if r.pending == nil {
r.pending = map[refusalKey]*refusalCount{}
}
k := refusalKey{uid, kind}
if c, ok := r.pending[k]; ok {
c.n++
c.why = why
return
}
r.pending[k] = &refusalCount{}
r.say(fmt.Sprintf("a filled file's write from %s was refused (%s): %s", who(k), kind, why))
}
// flush says what was counted since each was last said, and forgets every key, so the next refusal of each is said at
// once again.
func (r *refusalWords) flush() {
r.mu.Lock()
defer r.mu.Unlock()
keys := make([]refusalKey, 0, len(r.pending))
for k := range r.pending {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
if keys[i].uid != keys[j].uid {
return keys[i].uid < keys[j].uid
}
return keys[i].kind < keys[j].kind
})
for _, k := range keys {
if c := r.pending[k]; c.n > 0 {
r.say(fmt.Sprintf("%d more writes of a filled file from %s were refused (%s) since it was last said; the last: %s",
c.n, who(k), k.kind, c.why))
}
}
r.pending = nil
}
// who names a caller by its user id, or says why it has none.
func who(k refusalKey) string {
switch {
case k.kind == "overflow":
return "a caller not yet judged"
case k.uid == ^uint32(0):
return "an account that could not be read"
}
return fmt.Sprintf("user id %d", k.uid)
}
// answer serves one connection from the provisioner client's account.
func answer(ctx context.Context, conn *net.UnixConn, uid uint32, w Writer, say func(string)) {
var req Request
dec := json.NewDecoder(io.LimitReader(conn, MaxRequest))
dec.DisallowUnknownFields()
if err := dec.Decode(&req); err != nil {
// Answered even when the deadline to say it passed: the deadline covered the write too.
_ = conn.SetWriteDeadline(time.Now().Add(refusedWithin))
reply(conn, Refusal("", "the request could not be read, so nothing was written: "+err.Error()), say)
return
}
// The write waits for the apply's lock as long as an apply takes, and the client waits with it.
_ = conn.SetDeadline(time.Time{})
a := w.Write(ctx, uid, req)
switch {
case a.Refused != "":
say(fmt.Sprintf("a write of %s from user id %d was refused: %s", req.Path, uid, a.Refused))
case a.Failed != "":
say(fmt.Sprintf("a write of %s from user id %d failed: %s", req.Path, uid, a.Failed))
default:
say(fmt.Sprintf("wrote %s for the provisioner's client (user id %d): %s, %s", req.Path, uid, a.Action,
a.Digest))
}
_ = conn.SetWriteDeadline(time.Now().Add(within))
reply(conn, a, say)
}
// busy answers the client's connection over atOnce: its request read and dropped first, briefly, so the client reads
// the answer rather than a broken pipe — while no more than busyWaiting such refusals wait at once; over that, at once.
func busy(conn *net.UnixConn, readBy time.Time, busyWaiting chan struct{}, say func(string)) {
select {
case busyWaiting <- struct{}{}:
by := time.Now().Add(refusedWithin)
if readBy.Before(by) {
by = readBy
}
_ = conn.SetReadDeadline(by)
drain(conn)
<-busyWaiting
default:
}
_ = conn.SetWriteDeadline(time.Now().Add(refusedWithin))
reply(conn, Refusal("", "busy: other writes are being served; ask again"), say)
}
// drain reads and drops a request up to its end — the newline after it, the end of the stream or the deadline —
// and at most MaxRequest bytes, keeping none of it.
func drain(r io.Reader) {
var buf [4096]byte
for read := 0; read < MaxRequest; {
n, err := r.Read(buf[:])
read += n
if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil {
return
}
}
}
func reply(conn *net.UnixConn, a Answer, say func(string)) {
defer conn.Close()
if err := json.NewEncoder(conn).Encode(a); err != nil {
say("a filled file's answer could not be written back: " + err.Error())
}
}
+561
View File
@@ -0,0 +1,561 @@
package filling
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"github.com/novox/mesh-host/internal/declaration"
)
// Defends novox/hq ADR 0349 §4 (issue 668): the node-engine writes a filled file on request only for the provisioner
// client's account, only for a clean path the current declaration names as filled, and one at a time with the apply.
const provisionerUID = 4242
func declared(t *testing.T, raw string) *declaration.Declaration {
t.Helper()
d, err := declaration.Parse([]byte(raw))
if err != nil {
t.Fatal(err)
}
return d
}
const withAccount = `{"declaration":1,"provisioner_account":"mesh-node-provisioner","filled_files":["/etc/m/m.conf"],
"resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"},
{"id":"m.other","type":"file","path":"/etc/m/other.conf","content":"y"}]}`
const withoutAccount = `{"declaration":1,"filled_files":["/etc/m/m.conf"],
"resources":[{"id":"m.conf","type":"file","path":"/etc/m/m.conf","content":"x","owner-at-runtime":"owned"}]}`
// engine is an Engine over a fixed declaration, recording what it was asked to place.
type engine struct {
Engine
mu sync.Mutex
placed []string
}
func newEngine(t *testing.T, raw string, lock sync.Locker) *engine {
d := declared(t, raw)
e := &engine{}
e.Engine = Engine{
Declared: func() (*declaration.Declaration, error) { return d, nil },
UIDOf: func(name string) (uint32, error) {
if name == "mesh-node-provisioner" {
return provisionerUID, nil
}
return 0, errors.New("no such account")
},
Lock: func() (func(), error) { lock.Lock(); return lock.Unlock, nil },
Place: func(_ context.Context, _ *declaration.Declaration, path string, content []byte) (string, string, error) {
e.mu.Lock()
defer e.mu.Unlock()
e.placed = append(e.placed, path)
return "updated", "digest", nil
},
}
return e
}
func TestAWriteFromTheProvisionersAccountOfAFilledPathIsPlaced(t *testing.T) {
e := newEngine(t, withAccount, &sync.Mutex{})
if err := e.Accepts(provisionerUID); err != nil {
t.Fatal(err)
}
a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
if a.Refused != "" || a.Failed != "" || a.Action != "updated" || len(e.placed) != 1 {
t.Fatalf("answer %+v, placed %v", a, e.placed)
}
}
func TestAnUndeclaredPathIsRefused(t *testing.T) {
e := newEngine(t, withAccount, &sync.Mutex{})
for _, path := range []string{"/etc/m/other.conf", "/etc/m/none.conf", "/etc/passwd"} {
a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("evil")})
if a.Refused == "" || !strings.Contains(a.Refused, "not a file this node's declaration names as filled") {
t.Errorf("%s: %+v", path, a)
}
}
if len(e.placed) != 0 {
t.Errorf("placed %v", e.placed)
}
}
func TestAnotherAccountIsRefused(t *testing.T) {
e := newEngine(t, withAccount, &sync.Mutex{})
for _, uid := range []uint32{0, 1000, provisionerUID + 1} {
if err := e.Accepts(uid); err == nil {
t.Errorf("user id %d was accepted", uid)
}
a := e.Write(context.Background(), uid, Request{Path: "/etc/m/m.conf", Content: []byte("evil")})
if a.Refused == "" || !strings.Contains(a.Refused, "is not the provisioner client's account") {
t.Errorf("user id %d: %+v", uid, a)
}
}
if len(e.placed) != 0 {
t.Errorf("placed %v", e.placed)
}
}
func TestEveryWriteIsRefusedWhileTheDeclarationNamesNoAccount(t *testing.T) {
e := newEngine(t, withoutAccount, &sync.Mutex{})
if err := e.Accepts(provisionerUID); err == nil || !strings.Contains(err.Error(), "names no provisioner") {
t.Errorf("accepted with no account named: %v", err)
}
a := e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
if a.Refused == "" || len(e.placed) != 0 {
t.Errorf("written with no account named: %+v", a)
}
}
func TestAnAccountThatIsRootHereIsRefused(t *testing.T) {
e := newEngine(t, withAccount, &sync.Mutex{})
e.UIDOf = func(string) (uint32, error) { return 0, nil }
if err := e.Accepts(0); err == nil || !strings.Contains(err.Error(), "root") {
t.Errorf("an account that is root here was accepted: %v", err)
}
}
func TestAnUncleanPathIsRefused(t *testing.T) {
e := newEngine(t, withAccount, &sync.Mutex{})
for _, path := range []string{"", "etc/m/m.conf", "m.conf", "/etc//m/m.conf", "/etc/m/../m/m.conf",
"/etc/m/./m.conf", "/etc/m/m.conf/"} {
a := e.Write(context.Background(), provisionerUID, Request{Path: path, Content: []byte("x")})
if a.Refused == "" {
t.Errorf("%q was not refused: %+v", path, a)
}
}
if len(e.placed) != 0 {
t.Errorf("placed %v", e.placed)
}
}
// A write waits for an apply holding the apply's lock, and writes only once it is let go.
func TestAWriteWaitsForTheApply(t *testing.T) {
var applying sync.Mutex
e := newEngine(t, withAccount, &applying)
applying.Lock()
done := make(chan Answer)
go func() {
done <- e.Write(context.Background(), provisionerUID, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
}()
select {
case a := <-done:
t.Fatalf("written while the apply held the lock: %+v", a)
case <-time.After(100 * time.Millisecond):
}
e.mu.Lock()
n := len(e.placed)
e.mu.Unlock()
if n != 0 {
t.Fatalf("placed while the apply held the lock")
}
applying.Unlock()
if a := <-done; a.Action != "updated" {
t.Fatalf("after the apply: %+v", a)
}
}
// fixed is a Writer that accepts one user id and answers every write.
type fixed struct {
uid uint32
mu sync.Mutex
writes []Request
}
func (f *fixed) Accepts(uid uint32) error {
if uid != f.uid {
return errors.New("not the provisioner client's account")
}
return nil
}
func (f *fixed) Write(_ context.Context, _ uint32, req Request) Answer {
f.mu.Lock()
defer f.mu.Unlock()
f.writes = append(f.writes, req)
return Answer{Path: req.Path, Action: "updated", Digest: "d"}
}
// serve serves w on a socket in a fresh directory and returns its path, and what was said.
func serve(t *testing.T, w Writer) (string, func() []string) {
t.Helper()
dir, err := os.MkdirTemp("", "fill")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
socket := filepath.Join(dir, "fill.sock")
ctx, cancel := context.WithCancel(context.Background())
var mu sync.Mutex
var said []string
stopped := make(chan error)
go func() {
stopped <- Serve(ctx, socket, w, func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() })
}()
t.Cleanup(func() {
cancel()
if err := <-stopped; err != nil {
t.Error(err)
}
})
for i := 0; i < 100; i++ {
if _, err := os.Stat(socket); err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
return socket, func() []string { mu.Lock(); defer mu.Unlock(); return append([]string(nil), said...) }
}
func ask(t *testing.T, socket string, req Request) Answer {
t.Helper()
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Bounded: a socket that never answers fails the test instead of hanging it.
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
if err := json.NewEncoder(conn).Encode(req); err != nil {
t.Fatal(err)
}
var a Answer
if err := json.NewDecoder(conn).Decode(&a); err != nil {
t.Fatal(err)
}
return a
}
// Over a real socket, the kernel says who is asking: this test's own account is the provisioner's here.
func TestTheSocketWritesForTheAccountTheKernelSays(t *testing.T) {
w := &fixed{uid: uint32(os.Getuid())}
socket, said := serve(t, w)
a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("secret value")})
if a.Refused != "" || a.Action != "updated" || len(w.writes) != 1 || string(w.writes[0].Content) != "secret value" {
t.Fatalf("answer %+v, writes %+v", a, w.writes)
}
for _, line := range said() {
if strings.Contains(line, "secret value") {
t.Errorf("the content was said: %q", line)
}
}
}
func TestTheSocketRefusesAnotherAccountBeforeReadingItsRequest(t *testing.T) {
w := &fixed{uid: uint32(os.Getuid()) + 1}
socket, said := serve(t, w)
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Closed at once, its request never read: the write may meet a closed socket, and the refusal is there to read.
_ = json.NewEncoder(conn).Encode(Request{Path: "/etc/m/m.conf", Content: []byte("secret value")})
var a Answer
if err := json.NewDecoder(conn).Decode(&a); err != nil {
t.Fatal(err)
}
if a.Refused == "" || len(w.writes) != 0 {
t.Fatalf("another account was written for: %+v", a)
}
for _, line := range said() {
if strings.Contains(line, "secret value") {
t.Errorf("the content was said: %q", line)
}
}
}
// floodersAreAnother has the kernel's word say another account for every connection a test binds to a name holding
// "flood": the test runs as one account, and the socket must tell the client from everyone else.
func floodersAreAnother(t *testing.T) {
t.Helper()
was := peerOf
peerOf = func(conn *net.UnixConn) (uint32, int32, error) {
uid, pid, err := was(conn)
if a, ok := conn.RemoteAddr().(*net.UnixAddr); ok && a != nil && strings.Contains(a.Name, "flood") {
uid++
}
return uid, pid, err
}
t.Cleanup(func() { peerOf = was })
}
// A flood of connections from another account, each saying nothing and held open, does not stop the client's write:
// each is refused and closed before it holds anything.
func TestAFloodFromAnotherAccountDoesNotStopTheClient(t *testing.T) {
floodersAreAnother(t)
w := &fixed{uid: uint32(os.Getuid())}
socket, _ := serve(t, w)
from := t.TempDir()
var flood []*net.UnixConn
t.Cleanup(func() {
for _, c := range flood {
c.Close()
}
})
for i := 0; i < 3*atOnce+busyWaitingCap(); i++ {
c, err := net.DialUnix("unix", &net.UnixAddr{Name: filepath.Join(from, fmt.Sprintf("flood-%d", i)), Net: "unix"},
&net.UnixAddr{Name: socket, Net: "unix"})
if err != nil {
t.Fatal(err)
}
flood = append(flood, c)
}
start := time.Now()
a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")})
if a.Refused != "" || a.Action != "updated" {
t.Fatalf("the client's write during a flood: %+v", a)
}
if took := time.Since(start); took > refusedWithin/2 {
t.Errorf("the client's write waited %s behind the flood", took)
}
// And every flooder was closed: it reads its refusal and then the end.
for _, c := range flood {
_ = c.SetReadDeadline(time.Now().Add(2 * time.Second))
var got Answer
if err := json.NewDecoder(c).Decode(&got); err != nil || got.Refused == "" {
t.Fatalf("a flooder was not refused at once: %+v %v", got, err)
}
}
}
func busyWaitingCap() int { return busyWaitingAtOnce }
// A client that connects and says nothing holds one slot only until within, and another write goes on meanwhile.
func TestASilentClientDoesNotHoldTheSocket(t *testing.T) {
was := within
within = 300 * time.Millisecond
t.Cleanup(func() { within = was })
w := &fixed{uid: uint32(os.Getuid())}
socket, _ := serve(t, w)
silent, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer silent.Close()
if a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}); a.Action != "updated" {
t.Fatalf("a write beside a silent client: %+v", a)
}
_ = silent.SetReadDeadline(time.Now().Add(3 * time.Second))
var got Answer
if err := json.NewDecoder(silent).Decode(&got); err != nil || got.Refused == "" {
t.Errorf("the silent client was not let go after within: %+v %v", got, err)
}
}
// A request carrying more than the path and the content is refused: it says what the declaration says.
func TestARequestWithAnUnknownFieldIsRefused(t *testing.T) {
w := &fixed{uid: uint32(os.Getuid())}
socket, _ := serve(t, w)
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if _, err := conn.Write([]byte(`{"path":"/etc/m/m.conf","content":"eA==","mode":"0777"}` + "\n")); err != nil {
t.Fatal(err)
}
var a Answer
if err := json.NewDecoder(conn).Decode(&a); err != nil {
t.Fatal(err)
}
if a.Refused == "" || !strings.Contains(a.Refused, "mode") || len(w.writes) != 0 {
t.Errorf("a request with an unknown field: %+v, writes %v", a, w.writes)
}
}
// An accept that fails is said, waited for and tried again: the socket goes on serving.
func TestTheSocketGoesOnAfterAnAcceptFails(t *testing.T) {
wasAccept, wasBackoff := acceptFrom, acceptBackoff
failures := 3
var mu sync.Mutex
acceptFrom = func(l *net.UnixListener) (*net.UnixConn, error) {
mu.Lock()
defer mu.Unlock()
if failures > 0 {
failures--
return nil, errors.New("accept: too many open files")
}
return l.AcceptUnix()
}
acceptBackoff = time.Millisecond
t.Cleanup(func() { acceptFrom, acceptBackoff = wasAccept, wasBackoff })
w := &fixed{uid: uint32(os.Getuid())}
socket, said := serve(t, w)
if a := ask(t, socket, Request{Path: "/etc/m/m.conf", Content: []byte("x")}); a.Action != "updated" {
t.Fatalf("after failed accepts: %+v", a)
}
if !strings.Contains(strings.Join(said(), "\n"), "too many open files") {
t.Errorf("the failed accepts were not said: %v", said())
}
}
// The socket's directory is given the mode the socket needs, as mesh-cli's is.
func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) {
dir, err := os.MkdirTemp("", "fill")
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { os.RemoveAll(dir) })
if err := os.Chmod(dir, 0o700); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
stopped := make(chan error)
go func() { stopped <- Serve(ctx, filepath.Join(dir, "fill.sock"), &fixed{}, nil) }()
for i := 0; i < 100; i++ {
if _, err := os.Stat(filepath.Join(dir, "fill.sock")); err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
cancel()
if err := <-stopped; err != nil {
t.Fatal(err)
}
info, err := os.Stat(dir)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0o755 {
t.Errorf("the socket's directory is %o", info.Mode().Perm())
}
}
// The first refusal of each kind for each account is said at once; what follows within the time is counted, and the
// count is said when the time is up, even when the account has stopped.
func TestRefusalsAreSaidOnceAndTheirCountIsNeverLost(t *testing.T) {
var mu sync.Mutex
var said []string
r := &refusalWords{say: func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }}
for i := 0; i < 3; i++ {
r.said(1000, "account", "not the provisioner client's account")
}
// Another kind from the same account is said at once, not hidden behind the first.
r.said(1000, "namespace", "another namespace")
if len(said) != 2 || !strings.Contains(said[0], "(account)") || !strings.Contains(said[1], "(namespace)") {
t.Fatalf("said at once: %q", said)
}
// The account stops; the time is up; the two counted are said.
r.flush()
if len(said) != 3 || !strings.Contains(said[2], "2 more") || !strings.Contains(said[2], "user id 1000") {
t.Fatalf("after the time: %q", said)
}
// And the next refusal is said at once again.
r.said(1000, "account", "not the provisioner client's account")
if len(said) != 4 {
t.Fatalf("after the count was said: %q", said)
}
}
// refusingSocket serves a socket that refuses this test's account, with counted refusals said every every. It returns
// how to be refused n times, everything said so far, and how to stop it.
func refusingSocket(t *testing.T, every time.Duration) (func(int), func() string, func()) {
t.Helper()
was := refusalsSaidEvery
refusalsSaidEvery = every
dir, err := os.MkdirTemp("", "fill")
if err != nil {
t.Fatal(err)
}
socket := filepath.Join(dir, "fill.sock")
ctx, cancel := context.WithCancel(context.Background())
var mu sync.Mutex
var said []string
all := func() string { mu.Lock(); defer mu.Unlock(); return strings.Join(said, "\n") }
stopped := make(chan error)
go func() {
stopped <- Serve(ctx, socket, &fixed{uid: uint32(os.Getuid()) + 1},
func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() })
}()
for i := 0; i < 100; i++ {
if _, err := os.Stat(socket); err == nil {
break
}
time.Sleep(10 * time.Millisecond)
}
refuse := func(n int) {
for i := 0; i < n; i++ {
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
var a Answer
_ = json.NewDecoder(conn).Decode(&a)
conn.Close()
}
}
var once sync.Once
stop := func() {
once.Do(func() {
cancel()
if err := <-stopped; err != nil {
t.Error(err)
}
refusalsSaidEvery = was
os.RemoveAll(dir)
})
}
t.Cleanup(stop)
return refuse, all, stop
}
// Refusals counted while the socket serves are said on its timer, though the account has stopped asking.
func TestTheSocketSaysItsCountedRefusalsOnATimer(t *testing.T) {
refuse, all, _ := refusingSocket(t, 200*time.Millisecond)
refuse(3)
// "more writes", not a number: a tick may fall between two refusals and split the count.
deadline := time.Now().Add(3 * time.Second)
for !strings.Contains(all(), "more writes") && time.Now().Before(deadline) {
time.Sleep(20 * time.Millisecond)
}
if !strings.Contains(all(), "more writes") {
t.Fatalf("the counted refusals were not said on the timer: %s", all())
}
}
// Refusals counted when the socket stops are said as it stops.
func TestTheSocketSaysItsCountedRefusalsWhenItStops(t *testing.T) {
refuse, all, stop := refusingSocket(t, time.Hour)
refuse(5)
if strings.Contains(all(), "4 more") {
t.Fatalf("said before the socket stopped: %s", all())
}
stop()
if !strings.Contains(all(), "4 more") {
t.Errorf("the refusals counted when the socket stopped were lost: %s", all())
}
}
// A connection over the bound of those waiting to be judged is closed unread, and said.
func TestAConnectionOverTheWaitingBoundIsClosedUnread(t *testing.T) {
was := judgeWaiting
judgeWaiting = 0
t.Cleanup(func() { judgeWaiting = was })
w := &fixed{uid: uint32(os.Getuid())}
socket, said := serve(t, w)
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
var a Answer
if err := json.NewDecoder(conn).Decode(&a); err == nil {
t.Fatalf("answered over the bound: %+v", a)
}
if len(w.writes) != 0 || !strings.Contains(strings.Join(said(), "\n"), "(overflow)") {
t.Errorf("writes %v, said %q", w.writes, said())
}
}
+64
View File
@@ -3,11 +3,15 @@ package identity
import (
"crypto/ecdh"
"crypto/rand"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"fmt"
"os"
"strings"
"git.novox.be/novox/mesh-sdk/go/seal"
"golang.org/x/crypto/nacl/box"
)
@@ -141,3 +145,63 @@ func Seal(publicKey string, value []byte) (string, error) {
}
return base64.StdEncoding.EncodeToString(sealed), nil
}
// **The vault's box, opened with the same key** (novox/hq ADR 0349 §16, ADR 0206, issue 755). The core's own
// credentials travel in the declaration as values the vault sealed to this machine, which the controller carries and
// cannot open. The vault seals in the mesh's sealed box (mesh-sdk go/seal), whose keys travel as PEM: the public half
// as SPKI, the private as PKCS #8. They are this key's two halves in another wrapping, never another key.
// PublicPEM is this machine's sealing key's public half as SPKI PEM, the form the vault seals to.
func (s SealingKey) PublicPEM() (string, error) {
raw, err := base64.StdEncoding.DecodeString(s.Public)
if err != nil || len(raw) != 32 {
return "", fmt.Errorf("this node's sealing key is unusable")
}
pub, err := ecdh.X25519().NewPublicKey(raw)
if err != nil {
return "", fmt.Errorf("this node's sealing key is unusable: %w", err)
}
der, err := x509.MarshalPKIXPublicKey(pub)
if err != nil {
return "", err
}
return string(pem.EncodeToMemory(&pem.Block{Type: "PUBLIC KEY", Bytes: der})), nil
}
// OpenVaultBox opens a value the vault sealed to this machine. It fails for a box sealed to another key, or changed.
func (s SealingKey) OpenVaultBox(b seal.Box) ([]byte, error) {
raw, err := base64.StdEncoding.DecodeString(s.Private)
if err != nil || len(raw) != 32 {
return nil, fmt.Errorf("this node's sealing key is unusable")
}
priv, err := ecdh.X25519().NewPrivateKey(raw)
if err != nil {
return nil, fmt.Errorf("this node's sealing key is unusable: %w", err)
}
der, err := x509.MarshalPKCS8PrivateKey(priv)
if err != nil {
return nil, err
}
block := &pem.Block{Type: "PRIVATE KEY", Bytes: der}
opened, err := seal.Open(b, string(pem.EncodeToMemory(block)))
for i := range block.Bytes {
block.Bytes[i] = 0
}
if err != nil {
return nil, fmt.Errorf("this was not sealed by the vault to this node's current key")
}
return []byte(opened), nil
}
// Open opens anything the mesh sealed to this machine: the vault's box, given as its JSON (it begins with `{`), or
// the controller's sealed value, base64 (which never does).
func (s SealingKey) Open(sealed string) ([]byte, error) {
if strings.HasPrefix(strings.TrimSpace(sealed), "{") {
var b seal.Box
if err := json.Unmarshal([]byte(sealed), &b); err != nil {
return nil, fmt.Errorf("this is not the vault's sealed box: %w", err)
}
return s.OpenVaultBox(b)
}
return s.Unseal(sealed)
}
+8 -1
View File
@@ -143,7 +143,14 @@ func believed(check HandOverCheck, context string, withPath bool, body []byte) (
// believedBytes is believed with the ask's bytes as signed, for an ask that carries more than a hand-over's
// fields: read out of them only once they are believed.
func believedBytes(check HandOverCheck, context string, withPath bool, body []byte) (HandOverAsk, []byte, string) {
if len(body) > MaxHandOverAsk {
return believedBytesWithin(check, context, withPath, body, MaxHandOverAsk)
}
// believedBytesWithin is believedBytes for an ask bounded at max bytes: a move carries a sealed value, more than
// a hand-over's few hundred bytes (movesecret.go).
func believedBytesWithin(check HandOverCheck, context string, withPath bool, body []byte, max int) (HandOverAsk,
[]byte, string) {
if len(body) > max {
return HandOverAsk{}, nil, fmt.Sprintf("the ask is %d bytes, more than an ask", len(body))
}
var signed SignedHandOver
+23
View File
@@ -68,6 +68,8 @@ type natsLink struct {
judging *nats.Subscription
// acting hears the controller asking for the take act on a library (novox/hq ADR 0328 §7).
acting *nats.Subscription
// moving hears the controller asking to move a value sealed to this machine into the vault (issue 676).
moving *nats.Subscription
node string
// signer is the mesh's key, which a hand-over ask is verified with as a declaration is (issue 356).
signer ed25519.PublicKey
@@ -279,6 +281,24 @@ func (l *natsLink) HearAccessActs(handle AccessActFunc) {
}
}
// HearMoveSecrets listens for the controller asking this node to move one value sealed to it into the vault (design
// 57's V3, issue 676), on this machine's own subject, verified as a hand-over is (AnswerMoveSecret). A bus whose
// user list is older than the grant refuses the subscription; the controller then hears no answer.
func (l *natsLink) HearMoveSecrets(handle MoveSecretFunc) {
if handle == nil {
return
}
check := HandOverCheck{Signer: l.signer, Node: l.node}
moving, err := l.conn.Subscribe(AskMoveSecretSubject(l.node), func(msg *nats.Msg) {
if body, answer := MoveSecretReply(check, handle, msg.Reply, msg.Data); answer {
_ = msg.Respond(body)
}
})
if err == nil {
l.moving = moving
}
}
func (l *natsLink) Declarations() <-chan Declaration { return l.arrived }
func (l *natsLink) AskedToReport() <-chan struct{} { return l.asked }
func (l *natsLink) Lost() <-chan error { return l.lost }
@@ -299,6 +319,9 @@ func (l *natsLink) Close() {
if l.acting != nil {
_ = l.acting.Unsubscribe()
}
if l.moving != nil {
_ = l.moving.Unsubscribe()
}
if l.judging != nil {
_ = l.judging.Unsubscribe()
}
+33
View File
@@ -295,6 +295,39 @@ const OwnGroupContract = 7
// for it.
const BusyContract = 8
// OwnershipContract is the statement of a host that also reads a file's `owner-at-runtime` and a declaration's
// `filled_files` (novox/hq ADR 0349 §5): it writes an owned file back on drift, a seed once and a managed file's
// keys or block only, exactly as their old forms, and refuses a filled path that is not a declared file saying
// its owner. Like the fields before it, its presence is what tells the controller this host may be sent those
// fields; an older host refuses the whole declaration for them.
const OwnershipContract = 9
// ProvisionerContract is the statement of a host that also reads a declaration's `provisioner_account` (novox/hq ADR
// 0349 §4) and serves the local write socket: it accepts a write of a filled file from that account alone. Its
// presence is what tells the controller this host may be sent the field; an older host refuses the whole
// declaration for it.
const ProvisionerContract = 10
// HandedContract is the statement of a host that writes a filled file only on the provisioner client's word (novox/hq
// ADR 0349 §5, design 57 P7): an apply neither writes nor records a file the declaration names as filled, and removes
// one as an orphan only when the write verb placed it. Its presence is what tells the controller it may hand a moved
// module's files to the provisioner here (mesh-controller internal/link/protocol.go's HandedContract): to an older
// host a filled file would have two writers.
const HandedContract = 11
// VaultBoxContract is the statement of a host that also reads a file's `vault-sealed` (novox/hq ADR 0349 §16, issue
// 755): a core module's own credential the vault sealed to this machine in the mesh's sealed box (ADR 0206), which
// the controller carries and cannot open, opened here with this machine's sealing key and written as a sealed file
// is. Its presence is what tells the controller it may send the vault's box here (mesh-controller internal/link
// protocol.go's VaultBoxContract); an older host refuses the whole declaration for the field, and is sent the
// controller's sealed value as before.
const VaultBoxContract = 12
// NodeKeyContract is the statement of a host that states its own sealing key to the vault's node-key, in its own name,
// at its start and whenever the key changes (novox/hq ADR 0372, issue 786). The same number as mesh-controller
// internal/link/protocol.go's NodeKeyContract.
const NodeKeyContract = 13
// Health is one statement of every long-running resource's state on this machine (to-be 48 §4). Said in
// every report, as an event on each change, and again every minute while anything is not healthy — so a
// lost statement is not a lost fault.
+209
View File
@@ -0,0 +1,209 @@
package link
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"time"
)
// The move of a secret into the vault, asked over the bus (design 57's V3, issue 676; the operator's decision of
// 2026-10-14 on that issue).
//
// **Only the machine a value was sealed to can open it**, so the controller, which cannot, asks each machine's
// engine to move what was sealed to it: one row of its store at a time, on AskMoveSecretSubject, signed as a
// hand-over is and held to the same checks (believed), under a signing context of its own, naming the row — a
// module's own secret on this machine, or a pair credential with this machine at one end — its origin, the
// controller's generation, when it was made, and the value as the store holds it, sealed to this machine. The
// engine opens it with its sealing key, hands it to the vault's `move-in` in the vault's two steps, and answers
// what the vault kept only once the vault's read-back opens to the same value. Nothing is rotated; the engine
// writes nothing on this machine. The controller holds the same shapes in its own link code, with a test on each
// side holding the field names.
// AskMoveSecretSubject is where the controller asks this node's engine to move one value sealed to it into the vault.
func AskMoveSecretSubject(node string) string { return "mesh.node." + node + ".ask.move-secret" }
// MoveSecretContext is prefixed to a move ask's bytes before signing: never another ask's, nor a declaration's.
const MoveSecretContext = "novox-mesh move-secret v1\n"
// MoveSecretAsk is a move's own fields, beside a hand-over's (node, by, expires, nonce; no path): the row — module
// and name for a module's own secret on the ask's node; provider, provision, consumer, consumer module and local
// name for a pair credential — where its value came from, the controller's generation, when it was made, the value
// sealed to this machine as the store holds it, and the sealing key the store says it was sealed to.
type MoveSecretAsk struct {
Module string `json:"module,omitempty"`
Name string `json:"name,omitempty"`
Provider string `json:"provider,omitempty"`
Provision string `json:"provision,omitempty"`
Consumer string `json:"consumer,omitempty"`
ConsumerModule string `json:"consumer_module,omitempty"`
Local string `json:"local,omitempty"`
Origin string `json:"origin"`
Generation string `json:"generation,omitempty"`
MadeAt time.Time `json:"made_at"`
Sealed string `json:"sealed"`
SealedTo string `json:"sealed_to,omitempty"`
}
// MoveSecretAnswer is the engine's answer: what the vault kept, or why nothing was moved — never both.
type MoveSecretAnswer struct {
Said string `json:"said,omitempty"`
Refused string `json:"refused,omitempty"`
Secret string `json:"secret,omitempty"`
Generation int `json:"generation,omitempty"`
Unchanged bool `json:"unchanged,omitempty"`
}
// MoveSecret is one move asked and believed, as the handler is given it: the row, this machine, and who asked.
type MoveSecret struct {
MoveSecretAsk
Node, By string
}
// Own says the row is a module's own secret rather than a pair credential.
func (m MoveSecret) Own() bool { return m.Module != "" || m.Name != "" }
// MovedSecret is what the vault kept: its name, its generation, and whether it held the value already.
type MovedSecret struct {
Secret string
Generation int
Unchanged bool
}
// MoveSecretFunc moves one value into the vault: what it kept, or why not.
type MoveSecretFunc func(MoveSecret) (MovedSecret, error)
// MaxMoveSecretAsk bounds a move's request: a row's names, a time, a value sealed and a signature.
const MaxMoveSecretAsk = 64 << 10
// movedOrigins are the origins the controller's store holds.
var movedOrigins = map[string]bool{"made": true, "accepted": true, "issued": true}
func refusedMove(why string) MoveSecretAnswer {
return MoveSecretAnswer{Refused: why + "; nothing was moved"}
}
// oneName is a short name of one line.
func oneName(s string) bool {
return s != "" && len(s) <= MaxActField && !strings.ContainsAny(s, "\r\n\x00/ ")
}
// AnswerMoveSecret is the engine's judgement of one move ask's bytes: believed as a hand-over is, then the row —
// one shape, this machine's, a value to open and an origin — and only then the handler.
func AnswerMoveSecret(check HandOverCheck, handle MoveSecretFunc, body []byte) MoveSecretAnswer {
if handle == nil {
return refusedMove("this engine moves no secret into the vault")
}
ask, signed, refused := believedBytesWithin(check, MoveSecretContext, false, body, MaxMoveSecretAsk)
if refused != "" {
return refusedMove(refused)
}
var move MoveSecretAsk
if err := json.Unmarshal(signed, &move); err != nil {
return refusedMove("the signed ask is not a move's ask: " + err.Error())
}
own := move.Module != "" || move.Name != ""
pair := move.Provider != "" || move.Provision != "" || move.Consumer != "" || move.ConsumerModule != "" ||
move.Local != ""
switch {
case own == pair:
return refusedMove("the ask names neither one module's own secret nor one pair credential")
case own && (!oneName(move.Module) || !oneName(move.Name)):
return refusedMove("the ask's module and name are not short names")
case pair && (!oneName(move.Provider) || !oneName(move.Provision) || !oneName(move.Consumer) ||
!oneName(move.ConsumerModule) || (move.Local != "" && !oneName(move.Local))):
return refusedMove("the ask's provider, provision, consumer, consuming module and local name are not short names")
case pair && move.Provider != ask.Node && move.Consumer != ask.Node:
return refusedMove(fmt.Sprintf("the pair is between %s and %s, and this is %s", move.Provider, move.Consumer,
ask.Node))
case !movedOrigins[move.Origin]:
return refusedMove(fmt.Sprintf("the origin %q is not made, accepted or issued", move.Origin))
case strings.TrimSpace(move.Sealed) == "":
return refusedMove("the ask carries no value sealed to this machine")
case len(move.Generation) > MaxActField || strings.ContainsAny(move.Generation, "\r\n\x00"):
return refusedMove("the ask's generation is not one short line")
}
moved, err := handle(MoveSecret{MoveSecretAsk: move, Node: ask.Node, By: ask.By})
if err != nil {
return refusedMove(err.Error())
}
said := fmt.Sprintf("%s is in the vault at generation %d, read back the same", moved.Secret, moved.Generation)
if moved.Unchanged {
said = fmt.Sprintf("the vault held %s already, the same value (generation %d)", moved.Secret, moved.Generation)
}
return MoveSecretAnswer{Said: said, Secret: moved.Secret, Generation: moved.Generation, Unchanged: moved.Unchanged}
}
// MoveSecretReply is what a move request is answered with, and whether to answer, as HandOverReply.
func MoveSecretReply(check HandOverCheck, handle MoveSecretFunc, reply string, body []byte) ([]byte, bool) {
if reply == "" {
return nil, false
}
answer, err := json.Marshal(AnswerMoveSecret(check, handle, body))
if err != nil {
return nil, false
}
return answer, true
}
// MoveSecrets is a link that hears the controller asking this node to move a value into the vault. Optional.
type MoveSecrets interface {
HearMoveSecrets(MoveSecretFunc)
}
// ---- asking the vault ------------------------------------------------------------------------------------------
// VaultSeat is the vault's seat; its verbs are asked on `mesh.seat.<seat>.call.<verb>.<caller>`, the subject that
// names its caller (novox/hq issue 365), which the bus lets only that user publish.
const VaultSeat = "mesh-vault"
// NodeEngineUser is the bus user of a machine's node-engine, as the controller's broker names it.
func NodeEngineUser(node string) string { return "node." + node }
// CalledVaultSubject is where this node's engine asks one of the vault's verbs in its own name: each dot of its
// user's name written `~`, as the controller's broker writes a caller's token.
func CalledVaultSubject(verb, node string) string {
return "mesh.seat." + VaultSeat + ".call." + verb + "." + strings.ReplaceAll(NodeEngineUser(node), ".", "~")
}
// VaultWithin is how long the engine gives the vault to answer one step.
const VaultWithin = 15 * time.Second
// AskVault asks one of the vault's verbs in this node's engine's own name over the link open now, and answers its
// result, or its refusal as an error.
func (q *Queue) AskVault(ctx context.Context, verb string, args any) (json.RawMessage, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}
asker, ok := bus.(ToolBus)
if !ok {
return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the vault", ErrNotAsked)
}
body, err := json.Marshal(args)
if err != nil {
return nil, err
}
asking, cancel := context.WithTimeout(ctx, VaultWithin)
defer cancel()
reply, err := asker.Ask(asking, CalledVaultSubject(verb, q.Membership.Node), body)
if err != nil {
return nil, fmt.Errorf("the vault's %s could not be asked: %w", verb, err)
}
var env struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
if err := json.Unmarshal(reply, &env); err != nil {
return nil, fmt.Errorf("the vault answered %s with something that is not a tool's answer", verb)
}
if env.Error != "" {
return nil, errors.New(env.Error)
}
return env.Result, nil
}
+141
View File
@@ -0,0 +1,141 @@
package link
import (
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// The move's ask (design 57's V3, issue 676): a hand-over's fields and the row's own in one signed object, under a
// signing context of its own. The controller's side holds the same names and words.
func TestTheMovesAskKeepsItsFieldNamesAndContext(t *testing.T) {
body, _ := json.Marshal(MoveSecretAsk{Module: "m", Name: "n", Provider: "p", Provision: "q", Consumer: "c",
ConsumerModule: "cm", Local: "l", Origin: "made", Generation: "g", MadeAt: time.Unix(1, 0).UTC(),
Sealed: "s", SealedTo: "k"})
if got := keysOf(t, body); got != "consumer consumer_module generation local made_at module name origin provider "+
"provision sealed sealed_to" {
t.Fatalf("the move's fields are %q", got)
}
if AskMoveSecretSubject("ace") != "mesh.node.ace.ask.move-secret" {
t.Fatalf("the subject is %q", AskMoveSecretSubject("ace"))
}
if MoveSecretContext != "novox-mesh move-secret v1\n" {
t.Fatalf("the signing context is %q", MoveSecretContext)
}
answer, _ := json.Marshal(MoveSecretAnswer{Said: "s", Refused: "r", Secret: "own/a/b/c", Generation: 1, Unchanged: true})
if got := keysOf(t, answer); got != "generation refused said secret unchanged" {
t.Fatalf("the answer's fields are %q", got)
}
}
// signedMove is a move's ask as the controller makes it.
func signedMove(t *testing.T, key ed25519.PrivateKey, ask HandOverAsk, move MoveSecretAsk) []byte {
t.Helper()
raw, _ := json.Marshal(struct {
HandOverAsk
MoveSecretAsk
}{ask, move})
body, _ := json.Marshal(SignedHandOver{Ask: raw, Signature: ed25519.Sign(key, append([]byte(MoveSecretContext), raw...))})
return body
}
// A move is believed as every signed ask is, then names one row this machine holds — its own secret, or a pair
// credential with this machine at one end — with a value sealed to it and an origin the store has; only then is the
// handler given it. A hand-over's signature is not a move's.
func TestAMoveIsTakenOnlyForARowThisMachineHolds(t *testing.T) {
m, key := aMember(t)
now := time.Now()
var given []MoveSecret
handle := func(s MoveSecret) (MovedSecret, error) {
given = append(given, s)
return MovedSecret{Secret: "own/" + m.Node + "/postgres/superuser-password", Generation: 1}, nil
}
check := HandOverCheck{Signer: m.Signer, Node: m.Node, Nonces: &Nonces{}}
ask := func(nonce string) HandOverAsk {
return HandOverAsk{Node: m.Node, By: "the move", Expires: now.Add(HandOverGood), Nonce: nonce}
}
own := MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2VhbGVk"}
for why, body := range map[string][]byte{
"a hand-over's": signedAsk(t, key, ask(aNonce+"01")),
"no value": signedMove(t, key, ask(aNonce+"02"), MoveSecretAsk{Module: "postgres", Name: "x", Origin: "made"}),
"no origin": signedMove(t, key, ask(aNonce+"03"), MoveSecretAsk{Module: "postgres", Name: "x", Sealed: "c2Vh"}),
"both shapes": signedMove(t, key, ask(aNonce+"04"), MoveSecretAsk{Module: "postgres", Name: "x", Provider: m.Node,
Provision: "database", Consumer: "shanks", ConsumerModule: "gitea", Origin: "made", Sealed: "c2Vh"}),
"neither end": signedMove(t, key, ask(aNonce+"05"), MoveSecretAsk{Provider: "anchor", Provision: "database",
Consumer: "shanks", ConsumerModule: "gitea", Origin: "made", Sealed: "c2Vh"}),
"a path": signedMove(t, key, HandOverAsk{Node: m.Node, Path: "/etc", By: "x", Expires: now.Add(HandOverGood),
Nonce: aNonce + "06"}, own),
"two lines": signedMove(t, key, ask(aNonce+"07"), MoveSecretAsk{Module: "postgres", Name: "x\ny", Origin: "made",
Sealed: "c2Vh"}),
} {
if a := AnswerMoveSecret(check, handle, body); a.Refused == "" || !strings.HasSuffix(a.Refused, "nothing was moved") {
t.Errorf("%s: not refused, or refused without saying nothing was moved: %+v", why, a)
}
}
if len(given) != 0 {
t.Fatalf("a refused ask reached the move: %+v", given)
}
a := AnswerMoveSecret(check, handle, signedMove(t, key, ask(aNonce+"08"), own))
if a.Refused != "" || a.Secret != "own/"+m.Node+"/postgres/superuser-password" || a.Generation != 1 || a.Said == "" ||
len(given) != 1 || given[0].Name != "superuser-password" || given[0].By != "the move" {
t.Fatalf("a good ask: %+v %+v", a, given)
}
pair := MoveSecretAsk{Provider: "anchor", Provision: "database", Consumer: m.Node, ConsumerModule: "gitea",
Origin: "accepted", Sealed: "c2Vh"}
if a := AnswerMoveSecret(check, handle, signedMove(t, key, ask(aNonce+"09"), pair)); a.Refused != "" {
t.Fatalf("a pair this machine consumes was refused: %+v", a)
}
// What the move refuses is said, and nothing else.
refusing := func(MoveSecret) (MovedSecret, error) { return MovedSecret{}, errors.New("the vault said no") }
if a := AnswerMoveSecret(check, refusing, signedMove(t, key, ask(aNonce+"10"), own)); !strings.Contains(a.Refused,
"the vault said no") || a.Secret != "" {
t.Fatalf("a refused move: %+v", a)
}
}
// **Against a real server**: the move's signed ask on the node's own subject is answered with what the handler
// moved, and the engine asks the vault on the subject that names it, `node~<machine>` last.
func TestNatsAMoveIsAskedOfTheNodeAndTheVaultIsAskedInItsName(t *testing.T) {
conn, js := aBus(t)
m, key := aMember(t)
vault, err := conn.Subscribe("mesh.seat.mesh-vault.call.move-in.node~"+m.Node, func(msg *nats.Msg) {
_ = msg.Respond([]byte(`{"result":{"call":"c1","seal_to":"k"}}`))
})
if err != nil {
t.Fatal(err)
}
defer vault.Unsubscribe()
l := &natsLink{conn: conn, js: js, node: m.Node, signer: m.Signer}
q := &Queue{Membership: m}
q.attach(t.Context(), l)
l.HearMoveSecrets(func(s MoveSecret) (MovedSecret, error) {
raw, err := q.AskVault(t.Context(), "move-in", map[string]string{"module": s.Module})
if err != nil || !strings.Contains(string(raw), `"call":"c1"`) {
return MovedSecret{}, fmt.Errorf("the vault answered %s, %v", raw, err)
}
return MovedSecret{Secret: "own/" + s.Node + "/" + s.Module + "/" + s.Name, Generation: 1}, nil
})
defer l.Close()
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
ask := HandOverAsk{Node: m.Node, By: "the move", Expires: time.Now().Add(HandOverGood), Nonce: strings.Repeat("ab", 16)}
reply, err := conn.Request(AskMoveSecretSubject(m.Node), signedMove(t, key, ask,
MoveSecretAsk{Module: "postgres", Name: "superuser-password", Origin: "made", Sealed: "c2Vh"}), 5*time.Second)
if err != nil {
t.Fatal(err)
}
var said MoveSecretAnswer
if err := json.Unmarshal(reply.Data, &said); err != nil || said.Secret != "own/"+m.Node+"/postgres/superuser-password" {
t.Fatalf("answered %s, %v", reply.Data, err)
}
if _, err := (&Queue{Membership: Membership{Node: "nowhere"}}).AskVault(t.Context(), "move-in", nil); !errors.Is(err, ErrNotAsked) {
t.Fatalf("a queue with no link asked the vault: %v", err)
}
}
+5 -2
View File
@@ -65,8 +65,11 @@ type Queue struct {
// AccessAct judges and runs the take act on an access's folder, asked by the controller on an approved
// warrant (novox/hq ADR 0328 §7). Nil answers every such ask with a refusal.
AccessAct AccessActFunc
Say Announce
Timeout time.Duration
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
MoveSecret MoveSecretFunc
Say Announce
Timeout time.Duration
once sync.Once
wake chan struct{}
+4
View File
@@ -256,6 +256,10 @@ func serve(ctx context.Context, link Link, m Membership, queue *Queue, say Annou
if s, ok := link.(AccessActs); ok {
s.HearAccessActs(queue.AccessAct)
}
// And asking to move a value sealed to this machine into the vault (design 57's V3, issue 676).
if s, ok := link.(MoveSecrets); ok {
s.HearMoveSecrets(queue.MoveSecret)
}
for {
// Asked to stop — by standing aside, say — nothing further is taken in.
+19
View File
@@ -0,0 +1,19 @@
package meshcli
import "net"
// The kernel's word on who is at the other end of a socket, shared with the node-engine's other local socket, the
// write of a filled file (internal/filling, novox/hq ADR 0349 §4): one reading of it, so the two never judge a peer
// differently.
// PeerCredentials is the user id and process id of the process at the other end, as the kernel recorded them when
// it connected (SO_PEERCRED). Nothing the process says is read.
func PeerCredentials(conn *net.UnixConn) (uint32, int32, error) { return peerCredentials(conn) }
// SameNamespaces refuses a process whose mount or user namespace is not the engine's: a user id is this machine's
// account only in this machine's namespaces.
func SameNamespaces(pid int32) error { return namespacesOf(pid) }
// NotOwnDirectory says what is wrong with a socket's directory, or nothing: it must be a directory, not a link,
// owned by the account the engine runs as.
func NotOwnDirectory(dir string) string { return notOwnDirectory(dir) }
+13
View File
@@ -18,7 +18,20 @@ func TestTheValidatorIsTheHosts(t *testing.T) {
if problems := Declaration([]byte(ordered)); problems != nil {
t.Fatalf("a declaration carrying its order was refused: %v", problems)
}
// And one carrying a file's owner-at-runtime and the files the provisioner fills (novox/hq ADR 0349 §5): the
// reader takes them before any controller sends them.
owned := `{"declaration": 1, "filled_files": ["/etc/x"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}`
if problems := Declaration([]byte(owned)); problems != nil {
t.Fatalf("a declaration saying who owns a file at runtime was refused: %v", problems)
}
// And one naming the provisioner client's account (novox/hq ADR 0349 §4).
named := `{"declaration": 1, "provisioner_account": "mesh-node-provisioner", "filled_files": ["/etc/x"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}`
if problems := Declaration([]byte(named)); problems != nil {
t.Fatalf("a declaration naming the provisioner client's account was refused: %v", problems)
}
for _, bad := range []string{
`{"declaration": 1, "provisioner_account": "root", "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a"}]}`,
`{"declaration": 1, "filled_files": ["/etc/none"], "resources": [{"id": "f", "type": "file", "path": "/etc/x", "content": "a", "owner-at-runtime": "owned"}]}`,
`{"declaration": 1, "resources": []}`,
`{"declaration": 99, "resources": [{"id": "d", "type": "directory", "path": "/x"}]}`,
`{"declaration": 1, "resources": [{"id": "d", "type": "nothing-the-host-knows"}]}`,