diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 73157a0..f4822b0 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -493,8 +493,12 @@ func enrol(ctx context.Context, opts options) error { _ = json.Unmarshal(raw, &reported) } + // Signed with the identity just generated, so the mesh can tell this machine from anyone else + // who knows its public key (novox/hq issue 083). + proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public, + sealing.Public, serving.Public)) reply, err := link.Enrol(ctx, token.Broker, token.Fingerprint, *name, token.Secret, - mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, opts.timeout) + mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, opts.timeout) if err != nil { return err } diff --git a/internal/bootstrap/enrol.go b/internal/bootstrap/enrol.go index 10c4646..e64b86c 100644 --- a/internal/bootstrap/enrol.go +++ b/internal/bootstrap/enrol.go @@ -111,7 +111,7 @@ func Enrol(ctx context.Context, o Options, sys system.System, control controlPla if err != nil { return out, fmt.Errorf( "%s would not enrol this machine: %w\n%s\n"+ - "The token is one-time and has now been spent; running this again issues "+ + "The token is one-time and may have been spent; running this again issues "+ "another, so a re-run is safe", o.Host, err, indent(strings.TrimSpace(joined))) } if !strings.Contains(joined, "enrolled as "+o.Node) { diff --git a/internal/link/enrol.go b/internal/link/enrol.go index 5999395..6cf839b 100644 --- a/internal/link/enrol.go +++ b/internal/link/enrol.go @@ -2,6 +2,7 @@ package link import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -46,13 +47,24 @@ type EnrolRequest struct { ServingKey string `json:"serving_key,omitempty"` Profile map[string]any `json:"profile,omitempty"` + + // Proof is this node's identity key signing EnrolProof over this request: that the presenter + // holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish + // on a token this key already spent (novox/hq issue 083). + Proof []byte `json:"proof,omitempty"` } // EnrolReply is what the mesh says back. type EnrolReply struct { - Accepted bool `json:"accepted"` - Node string `json:"node,omitempty"` - Queue string `json:"queue,omitempty"` + Accepted bool `json:"accepted"` + + // TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the + // token is held for a moment by another enrolment — and that nothing was spent. The same + // request is asked again (novox/hq issue 083). + TryAgain bool `json:"try_again,omitempty"` + + Node string `json:"node,omitempty"` + Queue string `json:"queue,omitempty"` // What this node keeps so it can come back on its own. Without these a restart would need a // person with a new token, which would make disconnection a crisis rather than the ordinary @@ -65,9 +77,46 @@ type EnrolReply struct { Refusal string `json:"refusal,omitempty"` } +// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it +// presents, so a proof cannot be moved to another request. The mesh builds the same bytes. +func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { + return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + + "\x00" + overlay + "\x00" + sealing + "\x00" + serving) +} + // ErrRefused is what a node gets when the mesh will not have it. var ErrRefused = errors.New("the mesh refused this enrolment") +// EnrolPatience is how long a node keeps asking while the mesh says "try again" — as long as the +// mesh holds a token for the one enrolment presenting it, so a node asking the whole time is never +// held off by its own earlier attempt. +const EnrolPatience = 2 * time.Minute + +// AskAgainAfter is the pause between asks while the mesh says "try again". +const AskAgainAfter = 3 * time.Second + +// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking. +var ErrNotNow = errors.New("the mesh could not answer this enrolment") + +// answered decides what one reply means: done, ask again, or stop with an error. Separate from the +// broker so it can be held to that by a test. +func answered(reply EnrolReply, asking time.Duration) (again bool, err error) { + switch { + case reply.Accepted: + return false, nil + case reply.TryAgain && asking < EnrolPatience: + return true, nil + case reply.TryAgain: + // Said with what to do. The mesh holds the token for this attempt's keys for as long as + // this node kept asking, so a new attempt — with keys of its own — waits that out first. + return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+ + "enrol again with it; if it is then refused, issue a new one", + ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience) + default: + return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal) + } +} + // Enrol presents this node's key and its one-time secret, and waits to be told it is known. // // The broker has already authenticated this connection: the account was created when the token @@ -75,7 +124,7 @@ var ErrRefused = errors.New("the mesh refused this enrolment") // says once it is in, and the secret travels again because the control plane must not have to ask // the broker who connected. func Enrol(ctx context.Context, address, pin, node, secret string, public []byte, - overlayKey, sealingKey, servingKey string, profile map[string]any, + overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte, timeout time.Duration) (EnrolReply, error) { config, err := PinnedConfig(pin) @@ -122,24 +171,36 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte } request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, - OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile} + OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile, + Proof: proof} body, err := json.Marshal(request) if err != nil { return EnrolReply{}, err } - correlation := fmt.Sprintf("%s-%d", node, time.Now().UnixNano()) - publish, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - if err := channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false, - amqp.Publishing{ - ContentType: "application/json", - CorrelationId: correlation, - ReplyTo: queue.Name, - Body: body, - }); err != nil { - return EnrolReply{}, fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err) + // Asked, and asked again with the same request while the mesh says "try again": the keys + // this node generated are the ones it keeps, so the same request is the same enrolment, and + // the mesh holds the token for it (novox/hq issue 083). + ask := func() (string, error) { + correlation := fmt.Sprintf("%s-%d", node, time.Now().UnixNano()) + publish, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + if err := channel.PublishWithContext(publish, Exchange, KeyEnrol, false, false, + amqp.Publishing{ + ContentType: "application/json", + CorrelationId: correlation, + ReplyTo: queue.Name, + Body: body, + }); err != nil { + return "", fmt.Errorf("cannot publish to the %s exchange: %w", Exchange, err) + } + return correlation, nil } + correlation, err := ask() + if err != nil { + return EnrolReply{}, err + } + began := time.Now() // Waited for rather than assumed. A published message that nothing answers means the control // plane is not running, and a node that carried on regardless would believe it had joined a @@ -170,10 +231,28 @@ func Enrol(ctx context.Context, address, pin, node, secret string, public []byte if err := json.Unmarshal(delivery.Body, &reply); err != nil { return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err) } - if !reply.Accepted { - return reply, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal) + again, err := answered(reply, time.Since(began)) + if err != nil { + return reply, err } - return reply, nil + if !again { + return reply, nil + } + select { + case <-ctx.Done(): + return EnrolReply{}, ctx.Err() + case <-time.After(AskAgainAfter): + } + if correlation, err = ask(); err != nil { + return EnrolReply{}, err + } + if !deadline.Stop() { + select { + case <-deadline.C: + default: + } + } + deadline.Reset(timeout) } } } diff --git a/internal/link/enrol_again_test.go b/internal/link/enrol_again_test.go new file mode 100644 index 0000000..7ce0b6f --- /dev/null +++ b/internal/link/enrol_again_test.go @@ -0,0 +1,34 @@ +package link + +import ( + "errors" + "strings" + "testing" + "time" +) + +// What one reply means to a node enrolling (novox/hq issue 083): accepted is done; "try again" is +// asked again, until the node's patience runs out and it says the token was not spent; anything +// else is a refusal. +func TestAnEnrollingNodeAsksAgainWhileTheMeshSaysNotNow(t *testing.T) { + if again, err := answered(EnrolReply{Accepted: true}, 0); again || err != nil { + t.Fatalf("an accepted enrolment was not done: again=%v err=%v", again, err) + } + if again, err := answered(EnrolReply{TryAgain: true}, time.Second); !again || err != nil { + t.Fatalf("a mesh saying not now was not asked again: again=%v err=%v", again, err) + } + again, err := answered(EnrolReply{TryAgain: true, Refusal: "restarting"}, EnrolPatience) + if again || !errors.Is(err, ErrNotNow) { + t.Fatalf("a mesh saying not now past the node's patience did not stop it: again=%v err=%v", again, err) + } + if !errorsMention(err, "not spent") { + t.Errorf("giving up did not say the token can be used again: %v", err) + } + if again, err := answered(EnrolReply{Refusal: "that token cannot be used"}, 0); again || !errors.Is(err, ErrRefused) { + t.Fatalf("a refusal was not a refusal: again=%v err=%v", again, err) + } +} + +func errorsMention(err error, what string) bool { + return err != nil && strings.Contains(err.Error(), what) +} diff --git a/internal/link/enrol_proof_test.go b/internal/link/enrol_proof_test.go new file mode 100644 index 0000000..6d6d47b --- /dev/null +++ b/internal/link/enrol_proof_test.go @@ -0,0 +1,13 @@ +package link + +import "testing" + +// The bytes a node signs when it enrols. The mesh builds the same bytes to check the signature, in +// another repository; this known answer is repeated in its test, so the two cannot drift apart +// without one of them failing (novox/hq issue 083). +func TestWhatAnEnrollingNodeSignsIsFixed(t *testing.T) { + got := string(EnrolProof("s", []byte{1, 2, 3}, "o", "e", "v")) + if want := "novox-mesh-enrol\x00s\x00AQID\x00o\x00e\x00v"; got != want { + t.Fatalf("the enrolment proof's message changed: %q, want %q", got, want) + } +}