The engine states its key again until it hears the vault's key, and at a slow interval after, so a vault delivered after it or a changed vault key is heard (review of the adoption handing, ADR 0547 clause 9)
This commit is contained in:
@@ -1153,7 +1153,8 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
return identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
return identity.LoadSealingKey(identity.SealingKeyPath(opts.state))
|
||||||
}, func(verb string, args any) (json.RawMessage, error) {
|
}, func(verb string, args any) (json.RawMessage, error) {
|
||||||
return queue.AskVault(ctx, verb, args)
|
return queue.AskVault(ctx, verb, args)
|
||||||
}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, vaultKey.heard)
|
}, say, nodeKeyRetryFrom, nodeKeyRetryUpTo, &keyHearing{heard: vaultKey.heard,
|
||||||
|
held: func() bool { return vaultKey.get() != "" }, look: nodeKeyLook, every: nodeKeyReask})
|
||||||
// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
|
// And whether what it runs stays up, looked at on its own clock and said when it changes (novox/hq
|
||||||
// ADR 0240) — between applies, which is when a container crash-loops.
|
// ADR 0240) — between applies, which is when a container crash-loops.
|
||||||
if judging != nil {
|
if judging != nil {
|
||||||
|
|||||||
@@ -73,13 +73,29 @@ func (h *heardKey) following(kick func()) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
||||||
|
// vault takes the same key again and changes nothing (ADR 0547 clause 9).
|
||||||
|
const nodeKeyReask = 10 * time.Minute
|
||||||
|
|
||||||
|
// keyHearing is what the keeper does with the vault's own key in node-key's answers (ADR 0547 clause 9): heard takes
|
||||||
|
// it from an answer; held says whether one is held, and while none is the key is stated again at every look, so a vault
|
||||||
|
// delivered after this engine is heard within a look; every is how often it is stated again all the same, so a changed
|
||||||
|
// vault key is heard.
|
||||||
|
type keyHearing struct {
|
||||||
|
heard func(json.RawMessage)
|
||||||
|
held func() bool
|
||||||
|
// look is how often it is stated again while no vault key is held; every how often once one is.
|
||||||
|
look, every time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
|
// keepNodeKeyStated states the sealing key load answers to the vault at start, retrying with a growing wait from
|
||||||
// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
|
// retryFrom to retryUpTo while it does not answer, and states it again whenever the key changes. It says one line
|
||||||
// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. Each
|
// when a key is stated and one when a try fails for another reason than the last, and returns when ctx ends. With
|
||||||
// answer goes to heard, which takes the vault's own key from it.
|
// hearing, each answer goes to it, and the key is stated again while no vault key is held and at its interval.
|
||||||
func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
|
func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, error), ask vaultAsk, say func(string),
|
||||||
retryFrom, retryUpTo time.Duration, heard func(json.RawMessage)) {
|
retryFrom, retryUpTo time.Duration, hearing *keyHearing) {
|
||||||
stated, lastFailure := "", ""
|
stated, lastFailure := "", ""
|
||||||
|
var askedAt time.Time
|
||||||
wait := retryFrom
|
wait := retryFrom
|
||||||
fail := func(why string) {
|
fail := func(why string) {
|
||||||
if why != lastFailure {
|
if why != lastFailure {
|
||||||
@@ -92,20 +108,29 @@ func keepNodeKeyStated(ctx context.Context, load func() (identity.SealingKey, er
|
|||||||
for {
|
for {
|
||||||
delay := look
|
delay := look
|
||||||
key, err := load()
|
key, err := load()
|
||||||
|
again := false
|
||||||
|
if hearing != nil && stated != "" {
|
||||||
|
since := time.Since(askedAt)
|
||||||
|
held := hearing.held == nil || hearing.held()
|
||||||
|
again = (!held && since >= hearing.look) || (held && hearing.every > 0 && since >= hearing.every)
|
||||||
|
}
|
||||||
switch {
|
switch {
|
||||||
case err != nil:
|
case err != nil:
|
||||||
fail(err.Error())
|
fail(err.Error())
|
||||||
delay = wait
|
delay = wait
|
||||||
case key.Public != stated:
|
case key.Public != stated || again:
|
||||||
if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
|
if answer, err := ask("node-key", map[string]string{"key": key.Public}); err != nil {
|
||||||
fail(err.Error())
|
fail(err.Error())
|
||||||
delay = wait
|
delay = wait
|
||||||
} else {
|
} else {
|
||||||
if heard != nil {
|
askedAt = time.Now()
|
||||||
heard(answer)
|
if hearing != nil && hearing.heard != nil {
|
||||||
|
hearing.heard(answer)
|
||||||
|
}
|
||||||
|
if key.Public != stated {
|
||||||
|
say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
|
||||||
}
|
}
|
||||||
stated, lastFailure, wait = key.Public, "", retryFrom
|
stated, lastFailure, wait = key.Public, "", retryFrom
|
||||||
say(fmt.Sprintf("mesh-host: this machine's sealing key %s is stated to the vault", key.Public))
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
|
|||||||
@@ -19,6 +19,10 @@ type keyVault struct {
|
|||||||
stated []string
|
stated []string
|
||||||
fail int
|
fail int
|
||||||
verbs []string
|
verbs []string
|
||||||
|
// keyless is how many answers carry no vault key (a vault before ADR 0547 clause 9's, or not yet started); key the
|
||||||
|
// vault's key the others carry.
|
||||||
|
keyless int
|
||||||
|
key string
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
|
func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
|
||||||
@@ -35,7 +39,12 @@ func (v *keyVault) ask(verb string, args any) (json.RawMessage, error) {
|
|||||||
}
|
}
|
||||||
_ = json.Unmarshal(raw, &a)
|
_ = json.Unmarshal(raw, &a)
|
||||||
v.stated = append(v.stated, a.Key)
|
v.stated = append(v.stated, a.Key)
|
||||||
return json.RawMessage(`{"recorded":true}`), nil
|
if v.key == "" || v.keyless > 0 {
|
||||||
|
v.keyless--
|
||||||
|
return json.RawMessage(`{"recorded":true}`), nil
|
||||||
|
}
|
||||||
|
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": v.key})
|
||||||
|
return answer, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (v *keyVault) said() []string {
|
func (v *keyVault) said() []string {
|
||||||
@@ -74,7 +83,7 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(
|
|||||||
var hmu sync.Mutex
|
var hmu sync.Mutex
|
||||||
var answers []string
|
var answers []string
|
||||||
heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() }
|
heard := func(a json.RawMessage) { hmu.Lock(); answers = append(answers, string(a)); hmu.Unlock() }
|
||||||
go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, heard)
|
go keepNodeKeyStated(ctx, load, vault.ask, say, time.Millisecond, 5*time.Millisecond, &keyHearing{heard: heard})
|
||||||
|
|
||||||
waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
|
waitFor(t, "the first key stated after two refusals", func() bool { return len(vault.said()) == 1 })
|
||||||
// Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing.
|
// Its answer is handed on, so the vault's own key in it is heard (ADR 0547 clause 9); refusals hand nothing.
|
||||||
@@ -107,6 +116,49 @@ func TestTheEngineStatesItsSealingKeyToTheVaultAtStartRetriesAndRestatesAChange(
|
|||||||
if len(lines) == 0 {
|
if len(lines) == 0 {
|
||||||
t.Error("the engine said nothing")
|
t.Error("the engine said nothing")
|
||||||
}
|
}
|
||||||
|
asksAgainUntilItHearsTheVaultsKey(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ADR 0547 clause 9: the vault's key is heard though the engine's own key never changes. While none is held the key
|
||||||
|
// is stated again at each look, so a vault delivered after the engine (or answering no key yet) is heard; once one is
|
||||||
|
// held, again only at the slow interval, so a changed vault key is heard; never more.
|
||||||
|
func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
|
||||||
|
sealing, _ := identity.GenerateSealingKey()
|
||||||
|
load := func() (identity.SealingKey, error) { return sealing, nil }
|
||||||
|
pem := func(c string) string { return "-----BEGIN PUBLIC KEY-----\n" + c + "\n-----END PUBLIC KEY-----\n" }
|
||||||
|
for _, c := range []struct {
|
||||||
|
name string
|
||||||
|
keyless int
|
||||||
|
every time.Duration
|
||||||
|
change bool
|
||||||
|
wantAtLeast int
|
||||||
|
wantAtMost int
|
||||||
|
}{
|
||||||
|
{name: "a vault with no key yet is asked again until it answers one, then not", keyless: 3, wantAtLeast: 4, wantAtMost: 4},
|
||||||
|
{name: "a held key is asked again at the slow interval, and a changed one heard", every: 20 * time.Millisecond,
|
||||||
|
change: true, wantAtLeast: 3, wantAtMost: 1 << 20},
|
||||||
|
} {
|
||||||
|
t.Run(c.name, func(t *testing.T) {
|
||||||
|
vault := &keyVault{keyless: c.keyless, key: pem("AAAA")}
|
||||||
|
h := &heardKey{}
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
defer cancel()
|
||||||
|
go keepNodeKeyStated(ctx, load, vault.ask, func(string) {}, time.Millisecond, 5*time.Millisecond,
|
||||||
|
&keyHearing{heard: h.heard, held: func() bool { return h.get() != "" }, look: time.Millisecond, every: c.every})
|
||||||
|
waitFor(t, "the vault's key heard", func() bool { return h.get() == pem("AAAA") })
|
||||||
|
if c.change {
|
||||||
|
vault.mu.Lock()
|
||||||
|
vault.key = pem("BBBB")
|
||||||
|
vault.mu.Unlock()
|
||||||
|
waitFor(t, "the changed vault key heard", func() bool { return h.get() == pem("BBBB") })
|
||||||
|
}
|
||||||
|
time.Sleep(60 * time.Millisecond)
|
||||||
|
n := len(vault.said())
|
||||||
|
if n < c.wantAtLeast || n > c.wantAtMost {
|
||||||
|
t.Errorf("node-key asked %d times, want %d to %d", n, c.wantAtLeast, c.wantAtMost)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
||||||
|
|||||||
Reference in New Issue
Block a user