A node holds its link open, and applies what the mesh signs
The loop the whole thing exists for: told, apply, report. `run` holds one outbound connection open and consumes the node's own queue. Every declaration is verified against the control plane's signing key before a byte of it is read as an instruction -- not once at connect, every time. The transport being pinned is a different question from the instruction being genuine, and pinning only the first would make the second transitive: a compromised broker could forge declarations, and this host applies whatever the link delivers. Malformed and forged are reported differently, because ADR 0004 requires a host to tell "this is not from the mesh I joined" from "this is broken". One means somebody is trying and the other means something needs fixing. A node now keeps what it needs to come back on its own: the broker's address and fingerprint, the signing key it believes, and its own broker password -- which the mesh issues at enrolment to replace the token's secret, so the one-time thing stays one-time and the credential it holds for years is not the one that was pasted into a terminal. Verified in the lab end to end. The node enrolled, held its link, received a signed declaration and applied it -- the file is on the machine with the right contents, and the host's own record lists both resources. That run also found issue 010, which is recorded in novox/hq: the declaration removed every container on the machine, including the control plane that sent it. Correct reconciliation, shared store, and the first thing that happens.
This commit is contained in:
+10
-1
@@ -34,7 +34,16 @@ type EnrolReply struct {
|
||||
Accepted bool `json:"accepted"`
|
||||
Node string `json:"node,omitempty"`
|
||||
Queue string `json:"queue,omitempty"`
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
|
||||
// What this node keeps so it can come back on its own. Without these a restart would need a
|
||||
// person with a new token, which would make disconnection a crisis rather than the ordinary
|
||||
// situation novox/hq ADR 0004 says it is.
|
||||
Password string `json:"password,omitempty"`
|
||||
Broker string `json:"broker,omitempty"`
|
||||
Fingerprint string `json:"fingerprint,omitempty"`
|
||||
Signer []byte `json:"signer,omitempty"`
|
||||
|
||||
Refusal string `json:"refusal,omitempty"`
|
||||
}
|
||||
|
||||
// ErrRefused is what a node gets when the mesh will not have it.
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
package link
|
||||
|
||||
// The wire formats shared with the control plane, which defines them separately because this
|
||||
// binary requires nothing present and does not import it. A test on each side asserts the field
|
||||
// names, so a rename breaks both at once rather than on a real machine months later.
|
||||
|
||||
// Routing keys a node may publish. Its broker account is scoped to this exchange and its own
|
||||
// queue, so it can say these things and nothing else.
|
||||
const (
|
||||
KeyReport = "report"
|
||||
)
|
||||
|
||||
// Signed is a declaration and the signature over it.
|
||||
//
|
||||
// novox/hq ADR 0004: the transport is verified once at connect, and **each declaration is
|
||||
// verified by its signature, every time**. The two are different questions — a node connects to
|
||||
// the broker and takes instruction from the control plane behind it, and pinning only the first
|
||||
// would make the second transitive.
|
||||
//
|
||||
// The signature is over Declaration exactly as it arrived, bytes unchanged. Re-encoding before
|
||||
// verifying would mean checking a signature over something other than what was sent, and any
|
||||
// difference in key order or spacing would break it — so the raw message is what is signed and
|
||||
// what is checked.
|
||||
type Signed struct {
|
||||
Declaration []byte `json:"declaration"`
|
||||
Signature []byte `json:"signature"`
|
||||
}
|
||||
|
||||
// Report is what a node says after applying, and it is a statement rather than a write.
|
||||
//
|
||||
// A node states; the context that owns the data writes (novox/hq ADR 0006). The difference is the
|
||||
// security boundary: something that can write cannot be prevented from writing anything, and
|
||||
// something that can only state has its blast radius bounded by what this struct can say.
|
||||
type Report struct {
|
||||
Node string `json:"node"`
|
||||
|
||||
// Applied is what this machine now owns, by resource id.
|
||||
Applied []string `json:"applied,omitempty"`
|
||||
|
||||
// Failed says what could not be applied, and why, in words for a person.
|
||||
Failed map[string]string `json:"failed,omitempty"`
|
||||
|
||||
// Refused is set when the declaration was rejected whole rather than applied in part.
|
||||
Refused string `json:"refused,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// verified runs what Run does to a delivery body, without a broker: unmarshal, check the
|
||||
// signature, and only then apply. Isolating it keeps this test about the check rather than about
|
||||
// AMQP, which is tested against a real broker in the lab.
|
||||
func verified(t *testing.T, signer ed25519.PublicKey, body []byte) (Report, bool) {
|
||||
t.Helper()
|
||||
applied := false
|
||||
report := handleBody(context.Background(), Membership{Node: "anchor", Signer: signer}, body,
|
||||
func(context.Context, []byte) Report {
|
||||
applied = true
|
||||
return Report{Applied: []string{"something"}}
|
||||
})
|
||||
return report, applied
|
||||
}
|
||||
|
||||
func signedBody(t *testing.T, private ed25519.PrivateKey, declaration string) []byte {
|
||||
t.Helper()
|
||||
raw, err := json.Marshal(Signed{
|
||||
Declaration: []byte(declaration),
|
||||
Signature: ed25519.Sign(private, []byte(declaration)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
func TestTheMeshsOwnDeclarationIsApplied(t *testing.T) {
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, applied := verified(t, public, signedBody(t, private, `{"declaration":1}`))
|
||||
if !applied {
|
||||
t.Fatalf("a declaration the mesh signed was not applied: %s", report.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAForgedDeclarationIsNeverApplied(t *testing.T) {
|
||||
// The check that stands between "the mesh changes this machine" and "anybody does". The host
|
||||
// applies whatever the link delivers, so a forged declaration is the whole machine.
|
||||
public, _, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, other, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
report, applied := verified(t, public, signedBody(t, other, `{"declaration":1}`))
|
||||
if applied {
|
||||
t.Fatal("a declaration signed by another key was applied")
|
||||
}
|
||||
if report.Refused != ErrForged.Error() {
|
||||
t.Errorf("refused, but not as a forgery: %q", report.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATamperedDeclarationIsNeverApplied(t *testing.T) {
|
||||
// A broker that changed the declaration in flight, keeping the signature. This is what makes
|
||||
// pinning the transport insufficient on its own.
|
||||
public, private, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw, err := json.Marshal(Signed{
|
||||
Declaration: []byte(`{"declaration":1,"resources":["something else entirely"]}`),
|
||||
Signature: ed25519.Sign(private, []byte(`{"declaration":1}`)),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
report, applied := verified(t, public, raw)
|
||||
if applied {
|
||||
t.Fatal("a declaration altered after signing was applied")
|
||||
}
|
||||
if report.Refused != ErrForged.Error() {
|
||||
t.Errorf("refused, but not as a forgery: %q", report.Refused)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMalformedMessageIsToldApartFromAForgery(t *testing.T) {
|
||||
// novox/hq ADR 0004 requires these to be distinguishable: one means somebody is trying, the
|
||||
// other means something is broken, and they need different responses from a person.
|
||||
public, _, err := ed25519.GenerateKey(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
report, applied := verified(t, public, []byte("this is not a message"))
|
||||
if applied {
|
||||
t.Fatal("something unparseable was applied")
|
||||
}
|
||||
if report.Refused == ErrForged.Error() {
|
||||
t.Error("a malformed message was reported as a forgery; those must be distinguishable")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
||||
// The contract with the control plane, which defines these separately. A matching test lives
|
||||
// there; rename a field on either side and both fail.
|
||||
for _, c := range []struct {
|
||||
value any
|
||||
expect []string
|
||||
}{
|
||||
{Signed{Declaration: []byte("{}"), Signature: []byte("x")}, []string{"declaration", "signature"}},
|
||||
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
|
||||
[]string{"node", "applied", "failed", "refused"}},
|
||||
} {
|
||||
raw, err := json.Marshal(c.value)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var fields map[string]any
|
||||
if err := json.Unmarshal(raw, &fields); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range c.expect {
|
||||
if _, ok := fields[want]; !ok {
|
||||
t.Errorf("%T has no %q field; the control plane uses that name", c.value, want)
|
||||
}
|
||||
}
|
||||
if len(fields) != len(c.expect) {
|
||||
t.Errorf("%T has %d fields, expected %d: %v", c.value, len(fields), len(c.expect), fields)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
package link
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ed25519"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
amqp "github.com/rabbitmq/amqp091-go"
|
||||
)
|
||||
|
||||
// ErrForged is what a node returns for a declaration whose signature is not the mesh's.
|
||||
//
|
||||
// Its own error, and it must never be confused with a malformed message. novox/hq ADR 0004
|
||||
// requires a host to tell *this is not from the mesh I joined* apart from *this is malformed*:
|
||||
// the first means somebody is trying, the second means something is broken.
|
||||
var ErrForged = errors.New("this declaration was not signed by the mesh this node joined")
|
||||
|
||||
// Membership is what a node needs to reach its mesh again, held by the caller.
|
||||
type Membership struct {
|
||||
Node string
|
||||
Broker string
|
||||
Fingerprint string
|
||||
Password string
|
||||
Signer ed25519.PublicKey
|
||||
}
|
||||
|
||||
// Applier is what the host does with a declaration that has been proved to come from the mesh.
|
||||
type Applier func(ctx context.Context, declaration []byte) Report
|
||||
|
||||
// Run holds the link open, applying what arrives and reporting what happened.
|
||||
//
|
||||
// Outbound only, and nothing listens on this machine. The connection is the node's presence in
|
||||
// the mesh: while it is up the node is enrolled, and while it is down the node is disconnected —
|
||||
// which is an ordinary situation and not a failure, so this returns rather than panicking and
|
||||
// leaves restarting to whatever supervises it.
|
||||
func Run(ctx context.Context, m Membership, apply Applier, timeout time.Duration) error {
|
||||
config, err := PinnedConfig(m.Fingerprint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dsn := fmt.Sprintf("amqps://%s:%s@%s/",
|
||||
url.QueryEscape(m.Node), url.QueryEscape(m.Password), m.Broker)
|
||||
conn, err := amqp.DialConfig(dsn, amqp.Config{
|
||||
TLSClientConfig: config,
|
||||
Dial: amqp.DefaultDial(timeout),
|
||||
// Kept short so a node that has silently lost its route notices, rather than holding a
|
||||
// connection the broker forgot about and believing it is still in the mesh.
|
||||
Heartbeat: 10 * time.Second,
|
||||
})
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrWrongCertificate) {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("cannot reach the broker at %s: %w", m.Broker, err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
channel, err := conn.Channel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer channel.Close()
|
||||
|
||||
queue := QueueFor(m.Node)
|
||||
if _, err := channel.QueueDeclare(queue, true, false, false, false, nil); err != nil {
|
||||
return fmt.Errorf("cannot declare this node's queue %s: %w", queue, err)
|
||||
}
|
||||
|
||||
// One at a time. A declaration is applied to a machine, and applying two at once would race
|
||||
// on the same filesystem — so the broker holds the next one until this one is finished,
|
||||
// where it survives a restart.
|
||||
if err := channel.Qos(1, 0, false); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
deliveries, err := channel.ConsumeWithContext(ctx, queue, "", false, false, false, false, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
closed := conn.NotifyClose(make(chan *amqp.Error, 1))
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return nil
|
||||
case reason := <-closed:
|
||||
return fmt.Errorf("the link closed: %v", reason)
|
||||
case delivery, ok := <-deliveries:
|
||||
if !ok {
|
||||
return errors.New("the broker stopped delivering")
|
||||
}
|
||||
report := handle(ctx, m, apply, delivery)
|
||||
publishReport(ctx, channel, m, report, timeout)
|
||||
// Acknowledged after the report is published. A node that dies between applying and
|
||||
// reporting leaves the declaration on the broker and applies it again on return,
|
||||
// which is safe because applying is reconciliation — it converges rather than
|
||||
// repeating.
|
||||
_ = delivery.Ack(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handle(ctx context.Context, m Membership, apply Applier, delivery amqp.Delivery) Report {
|
||||
return handleBody(ctx, m, delivery.Body, apply)
|
||||
}
|
||||
|
||||
// handleBody is the whole of deciding whether to trust a message, separated from the broker so it
|
||||
// can be tested as the security check it is rather than as message plumbing.
|
||||
func handleBody(ctx context.Context, m Membership, body []byte, apply Applier) Report {
|
||||
var signed Signed
|
||||
if err := json.Unmarshal(body, &signed); err != nil {
|
||||
return Report{Node: m.Node, Refused: "this message is not a declaration: " + err.Error()}
|
||||
}
|
||||
|
||||
// Before anything is read out of it, let alone applied. The host applies whatever the link
|
||||
// delivers, so this check is the difference between the mesh changing this machine and
|
||||
// anybody changing it.
|
||||
if !ed25519.Verify(m.Signer, signed.Declaration, signed.Signature) {
|
||||
return Report{Node: m.Node, Refused: ErrForged.Error()}
|
||||
}
|
||||
return apply(ctx, signed.Declaration)
|
||||
}
|
||||
|
||||
func publishReport(ctx context.Context, channel *amqp.Channel, m Membership, report Report,
|
||||
timeout time.Duration) {
|
||||
report.Node = m.Node
|
||||
body, err := json.Marshal(report)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
publish, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
_ = channel.PublishWithContext(publish, Exchange, KeyReport, false, false,
|
||||
amqp.Publishing{ContentType: "application/json", Body: body})
|
||||
}
|
||||
Reference in New Issue
Block a user