Merge pull request 'The node-engine without the controller's stream and lease (issue 1126 group B)' (#197) from feat/1126-removal into main
This commit was merged in pull request #197.
This commit is contained in:
@@ -171,26 +171,6 @@ carries. The claim is not. The foundation is four container services, and:
|
|||||||
So `foundation.lock` here is a real bundle with a placeholder's content. Saying that plainly
|
So `foundation.lock` here is a real bundle with a placeholder's content. Saying that plainly
|
||||||
beats shipping a host that claims a foundation it has never raised.
|
beats shipping a host that claims a foundation it has never raised.
|
||||||
|
|
||||||
## mesh-cli's socket (novox/hq ADR 0272)
|
|
||||||
|
|
||||||
The engine serves `/run/nox-mesh-host/cli.sock` for `mesh-cli`, the operator's command (`internal/meshcli`). It
|
|
||||||
reads the asking account from the kernel (`SO_PEERCRED`) and asks the controller on this node's own subject; the
|
|
||||||
controller decides what the line may do, and the engine runs nothing it receives. One line per connection, each
|
|
||||||
connection's deadline set the moment it is accepted, two lines at once per account and eight for the machine,
|
|
||||||
"busy" beyond that.
|
|
||||||
|
|
||||||
**Only a login session can be the terminal.** The engine reads the asking process's cgroup and sends its login
|
|
||||||
session's scope (`session-<id>.scope` under the account's own `user-<uid>.slice`, which logind's own record in
|
|
||||||
`/run/systemd/sessions/<id>` must say is this account's, of class `user`), or nothing, read the moment the
|
|
||||||
connection is accepted; the controller makes
|
|
||||||
nothing else the terminal. A service running as the operator — the tool runner, a unit of the account's user
|
|
||||||
manager — is at most an ordinary call.
|
|
||||||
|
|
||||||
**A process outside the machine's own namespaces is refused**, whatever its user id: the peer's mount and user
|
|
||||||
namespace must be the engine's. So `mesh-cli` run inside a flatpak, a toolbox or distrobox container, or any other
|
|
||||||
container — even one running as the operator's own id — is refused with the reason; run it from the machine's own
|
|
||||||
shell.
|
|
||||||
|
|
||||||
## A capability is detected, never assumed
|
## A capability is detected, never assumed
|
||||||
|
|
||||||
The reason this is the first thing built rather than a detail of it.
|
The reason this is the first thing built rather than a detail of it.
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"encoding/pem"
|
"encoding/pem"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -73,7 +74,7 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
|
|||||||
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil {
|
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil {
|
||||||
t.Error("a controller-signed message is accepted once the vault's key is read")
|
t.Error("a controller-signed message is accepted once the vault's key is read")
|
||||||
}
|
}
|
||||||
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
|
if got := keys().Trusts; strings.Contains(strings.Join(got, " "), ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
|
||||||
trust.VaultAuthority() != vaultCA || told != 2 {
|
trust.VaultAuthority() != vaultCA || told != 2 {
|
||||||
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
|
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -333,8 +333,8 @@ func composeFrom(current *agentNow, composer *atomic.Pointer[selfcompose.Compose
|
|||||||
// packagesReadFor bounds the verb packages' reads and its write.
|
// packagesReadFor bounds the verb packages' reads and its write.
|
||||||
const packagesReadFor = 20 * time.Second
|
const packagesReadFor = 20 * time.Second
|
||||||
|
|
||||||
// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages
|
// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages;
|
||||||
// beside the controller's mirror's key for it and their difference; with write, the key written again from what the
|
// with write, the key written again from what the
|
||||||
// composer that runs now handed, refused in words while the node composes none of its own.
|
// composer that runs now handed, refused in words while the node composes none of its own.
|
||||||
func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[selfcompose.Composer]) func(write bool) (any, error) {
|
func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[selfcompose.Composer]) func(write bool) (any, error) {
|
||||||
return func(write bool) (any, error) {
|
return func(write bool) (any, error) {
|
||||||
@@ -344,8 +344,8 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s
|
|||||||
if write {
|
if write {
|
||||||
c := composer.Load()
|
c := composer.Load()
|
||||||
if c == nil {
|
if c == nil {
|
||||||
return nil, errors.New("this node composes none of its own modules now (it applies the controller's " +
|
return nil, errors.New("this node composes none of its own modules now (its node agent is starting), " +
|
||||||
"compositions, or its node agent is starting), so it has no packages of its own to write")
|
"so it has no packages of its own to write")
|
||||||
}
|
}
|
||||||
w, err := c.WritePackages(ctx)
|
w, err := c.WritePackages(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -354,12 +354,7 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s
|
|||||||
wrote = &w
|
wrote = &w
|
||||||
}
|
}
|
||||||
key, keyErr := link.EnginePackages{Session: session}.Get(ctx, node)
|
key, keyErr := link.EnginePackages{Session: session}.Get(ctx, node)
|
||||||
var mirror []byte
|
v := link.PackagesAnswer(node, key, keyErr)
|
||||||
h, mirrorErr := session.KVHistory(ctx, link.MirrorPackagesBucket, node)
|
|
||||||
if mirrorErr == nil && len(h) > 0 && !h[len(h)-1].Deleted {
|
|
||||||
mirror = h[len(h)-1].Value
|
|
||||||
}
|
|
||||||
v := link.PackagesAnswer(node, key, mirror, keyErr, mirrorErr)
|
|
||||||
v.Wrote = wrote
|
v.Wrote = wrote
|
||||||
return v, nil
|
return v, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,17 +16,17 @@ import (
|
|||||||
|
|
||||||
// enrolThroughSeat joins this machine through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): it redeems the token
|
// enrolThroughSeat joins this machine through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): it redeems the token
|
||||||
// with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps
|
// with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps
|
||||||
// its membership and its login, dials as the login node-logins names and confirms. handled is false when the bus does not serve
|
// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this
|
||||||
// the seat to this token (the controller still holds its lease, so its enrolment subject is the way in): nothing was sent or
|
// token (link.ErrNoSeat) is said as the error: nothing was sent or kept.
|
||||||
// kept then, and the caller enrols as before.
|
|
||||||
func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity,
|
func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity,
|
||||||
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) (handled bool, err error) {
|
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error {
|
||||||
|
|
||||||
req := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public,
|
req := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public,
|
||||||
SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported}
|
SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported}
|
||||||
if found != nil {
|
if found != nil {
|
||||||
|
var err error
|
||||||
if req.Tunnel, err = json.Marshal(found); err != nil {
|
if req.Tunnel, err = json.Marshal(found); err != nil {
|
||||||
return true, err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(delivery.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey)))
|
req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(delivery.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey)))
|
||||||
@@ -61,20 +61,13 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
answer, err := j.Run(ctx)
|
answer, err := j.Run(ctx)
|
||||||
switch {
|
if err != nil {
|
||||||
case errors.Is(err, link.ErrNoSeat):
|
return err
|
||||||
// The path before the lease write: the controller's enrolment subject is granted to the token only while the
|
|
||||||
// controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said.
|
|
||||||
fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+
|
|
||||||
"subject, the path before the lease write, which the bus refuses once the lease is written\n", err)
|
|
||||||
return false, nil
|
|
||||||
case err != nil:
|
|
||||||
return true, err
|
|
||||||
}
|
}
|
||||||
fmt.Printf("\nenrolled as %s through the seat mesh-enrolment, dialled as %s and confirmed\n", answer.Node, answer.Login)
|
fmt.Printf("\nenrolled as %s through the seat mesh-enrolment, dialled as %s and confirmed\n", answer.Node, answer.Login)
|
||||||
fmt.Printf(" identity %s\n", identityPath)
|
fmt.Printf(" identity %s\n", identityPath)
|
||||||
fmt.Println(" nothing is applied until the placement of this node names a module")
|
fmt.Println(" nothing is applied until the placement of this node names a module")
|
||||||
return true, nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is
|
// keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is
|
||||||
|
|||||||
@@ -21,7 +21,6 @@ type owedBus struct {
|
|||||||
rv map[string]uint64
|
rv map[string]uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *owedBus) Report(context.Context, string, []byte) error { return nil }
|
|
||||||
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
|
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
|
||||||
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
|
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
|
||||||
|
|
||||||
|
|||||||
+12
-167
@@ -100,7 +100,7 @@ func versionAt(self, stamped string) string {
|
|||||||
const usage = `nox-mesh-host — the node-engine
|
const usage = `nox-mesh-host — the node-engine
|
||||||
|
|
||||||
A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The
|
A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The
|
||||||
operator's command is nox (mesh-cli), on any node, from their own account (novox/hq ADR 0272);
|
operator's command is nox (mesh-cli), on any node, from their own account, through the seats' verbs;
|
||||||
the subcommands here are for installing a machine, joining it, and rescuing one no mesh can
|
the subcommands here are for installing a machine, joining it, and rescuing one no mesh can
|
||||||
reach, run as root at the machine.
|
reach, run as root at the machine.
|
||||||
|
|
||||||
@@ -110,8 +110,7 @@ reach, run as root at the machine.
|
|||||||
key make this machine's tunnel key, or read the one it made, and print the public
|
key make this machine's tunnel key, or read the one it made, and print the public
|
||||||
half: what its join token is issued for (novox/hq ADR 0169)
|
half: what its join token is issued for (novox/hq ADR 0169)
|
||||||
enrol --token T join the mesh — through the tunnel when the token was issued for a key
|
enrol --token T join the mesh — through the tunnel when the token was issued for a key
|
||||||
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
|
overlay take refused: the controller that took a rekey ended and no seat takes one now
|
||||||
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
|
|
||||||
owned what this host has applied and still owns
|
owned what this host has applied and still owns
|
||||||
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
|
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
|
||||||
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
|
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
|
||||||
@@ -809,167 +808,21 @@ func enrol(ctx context.Context, opts options) error {
|
|||||||
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
|
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
|
||||||
reported := profileAsReported(detected)
|
reported := profileAsReported(detected)
|
||||||
|
|
||||||
// Signed with the identity just generated, so the mesh can tell this machine from anyone else
|
// Through the seat mesh-enrolment (novox/hq ADR 0587 §6), the one way in: the controller's enrolment subject ended
|
||||||
// who knows its public key (novox/hq issue 083).
|
// with the controller (issue 1126).
|
||||||
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
|
return enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath)
|
||||||
sealing.Public, serving.Public))
|
|
||||||
// The token says where to go and which certificate that address must present. It says nothing
|
|
||||||
// about which bus is there, and does not need to: there is one, and this host knows which
|
|
||||||
// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
|
|
||||||
//
|
|
||||||
// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
|
|
||||||
// while two buses existed and became a refusal the moment one did: an empty transport is not
|
|
||||||
// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
|
|
||||||
// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
|
|
||||||
// changed.
|
|
||||||
// Through the seat mesh-enrolment when the bus serves it to this token (novox/hq ADR 0587 §6): the controller's enrolment
|
|
||||||
// subject below stays for as long as the controller holds its lease.
|
|
||||||
if handled, err := enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath); handled || err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
reply, err := link.Enrol(ctx,
|
|
||||||
link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
|
|
||||||
*name, token.Secret,
|
|
||||||
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
|
|
||||||
opts.timeout)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// The mesh's name for this node wins over what the machine called itself: the token was
|
|
||||||
// issued for a node record, and that record is what the identity binds to.
|
|
||||||
mine.Node = reply.Node
|
|
||||||
mine.Membership = identity.Membership{
|
|
||||||
Broker: firstNonEmpty(reply.Broker, token.Broker),
|
|
||||||
Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
|
|
||||||
Signer: firstNonEmpty2(reply.Signer, token.Signer),
|
|
||||||
Password: reply.Password,
|
|
||||||
// **Which bus this membership is for, said rather than left empty** (novox/hq
|
|
||||||
// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
|
|
||||||
// is not this one's — so a node enrolled without it came up and reconnected for ever
|
|
||||||
// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
|
|
||||||
// reply does not carry it because there is one bus and the host knows which (ADR 0131);
|
|
||||||
// what was missing was writing that down where the link reads it.
|
|
||||||
Transport: link.OnNATS,
|
|
||||||
}
|
|
||||||
if mine.Membership.Password == "" {
|
|
||||||
// The mesh did not replace the token's secret, so it is still this node's broker
|
|
||||||
// password. Said rather than silently kept: a one-time secret living on as a credential
|
|
||||||
// is worth knowing about.
|
|
||||||
mine.Membership.Password = token.Secret
|
|
||||||
fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " +
|
|
||||||
"remains this node's credential")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Saved only now, and only once the mesh has said it knows this node. A node holding an
|
|
||||||
// identity the mesh has never recorded would believe it had joined and be believed by
|
|
||||||
// nobody — worse than not having joined, because nothing would look wrong.
|
|
||||||
if err := identity.Save(identityPath, mine); err != nil {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"the mesh accepted this node as %q and its identity could not be saved: %w\n"+
|
|
||||||
"That token is spent, so getting back needs a new one", reply.Node, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if !mine.Membership.Joined() {
|
|
||||||
return fmt.Errorf(
|
|
||||||
"the mesh accepted this node as %q but did not say how to reach it again, so this "+
|
|
||||||
"identity could not be used after a restart. Nothing was saved", reply.Node)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Written before the identity, so a node that dies between the two has a key file with no
|
|
||||||
// identity — which enrols again cleanly — rather than an identity naming a key that is not
|
|
||||||
// there, which looks joined and cannot come up.
|
|
||||||
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
|
||||||
[]byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
|
|
||||||
return fmt.Errorf("cannot write this node's overlay key: %w", err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(identity.SealingKeyPath(opts.state),
|
|
||||||
[]byte(sealing.Private+"\n"), 0o600); err != nil {
|
|
||||||
return fmt.Errorf("cannot write this node's sealing key: %w", err)
|
|
||||||
}
|
|
||||||
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
|
|
||||||
return fmt.Errorf("cannot write this node's serving key: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
fmt.Printf("\nenrolled as %s\n", reply.Node)
|
|
||||||
fmt.Printf(" identity %s\n", identityPath)
|
|
||||||
fmt.Printf(" queue %s\n", reply.Queue)
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
|
// overlayCommand is `mesh-host overlay take`: a node taking the tunnel found on its machine over after it enrolled
|
||||||
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
|
// (novox/hq ADR 0105). Refused, and nothing is changed: the rekey it sent was taken by the controller alone, which
|
||||||
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
|
// ended by its unassignment (issue 1126), and no seat takes one now; a key written here that no part of the mesh knows
|
||||||
// every credential the mesh sealed to it.
|
// would raise the mesh's interface on a key its hub does not peer.
|
||||||
func overlayCommand(ctx context.Context, opts options) error {
|
func overlayCommand(_ context.Context, opts options) error {
|
||||||
if opts.file != "take" {
|
if opts.file != "take" {
|
||||||
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
|
||||||
}
|
}
|
||||||
identityPath := identity.Path(opts.state)
|
return errors.New("overlay take is refused: the controller that took a rekey ended (issue 1126) and no seat " +
|
||||||
mine, err := identity.Load(identityPath)
|
"takes one now. Nothing was changed")
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("%w. Nothing was changed", err)
|
|
||||||
}
|
|
||||||
taken, rekey, err := rekeyOnto(mine, found)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
|
|
||||||
fmt.Printf(" identity, sealing and serving keys are untouched\n")
|
|
||||||
|
|
||||||
// Told first, then written: a mesh told and a machine not yet written is put right by running
|
|
||||||
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
|
|
||||||
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
|
|
||||||
// on a key the mesh does not know at the next restart.
|
|
||||||
if err := link.Publish(ctx, link.Membership{
|
|
||||||
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
|
|
||||||
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
|
|
||||||
Transport: mine.Membership.Transport,
|
|
||||||
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
|
|
||||||
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
|
|
||||||
}
|
|
||||||
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
|
|
||||||
|
|
||||||
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
|
|
||||||
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
|
|
||||||
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
|
|
||||||
}
|
|
||||||
if err := identity.Save(identityPath, taken); err != nil {
|
|
||||||
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
|
|
||||||
}
|
|
||||||
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
|
|
||||||
identity.OverlayKeyPath(opts.state))
|
|
||||||
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
|
|
||||||
mine.Node, found.Port, mine.Node)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
|
|
||||||
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
|
|
||||||
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
|
|
||||||
// is the one before the take, so the mesh can tell a repeat from a replay.
|
|
||||||
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
|
|
||||||
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
|
|
||||||
if err != nil {
|
|
||||||
return identity.Identity{}, link.Rekey{}, err
|
|
||||||
}
|
|
||||||
previous := mine.Overlay.Public
|
|
||||||
if previous == overlay.Public && mine.OverlayBefore != "" {
|
|
||||||
previous = mine.OverlayBefore
|
|
||||||
}
|
|
||||||
taken := mine
|
|
||||||
taken.Overlay = overlay
|
|
||||||
if previous != overlay.Public {
|
|
||||||
taken.OverlayBefore = previous
|
|
||||||
}
|
|
||||||
presented := carried(found)
|
|
||||||
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
|
|
||||||
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
|
|
||||||
return taken, rekey, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
|
||||||
@@ -1120,9 +973,6 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
Membership: membership,
|
Membership: membership,
|
||||||
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
|
||||||
Keys: recordKeys(opts.state, trust.Trusts),
|
Keys: recordKeys(opts.state, trust.Trusts),
|
||||||
// And no controller's key: every ask the controller would sign is refused, never verified with the membership's
|
|
||||||
// signer (novox/hq ADR 0563 §7; issue 1126, the controller ended by its unassignment).
|
|
||||||
ControllerKey: noControllerKey,
|
|
||||||
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
|
||||||
HaveWritten: proof.haveWritten,
|
HaveWritten: proof.haveWritten,
|
||||||
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
// And the operator's hand-over of a directory used as found, carried by the controller from the
|
||||||
@@ -1213,9 +1063,6 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
}, BusyAskEvery)
|
}, BusyAskEvery)
|
||||||
// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
|
// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
|
||||||
go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
|
go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
|
||||||
// And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller
|
|
||||||
// (novox/hq ADR 0272).
|
|
||||||
go serveMeshCLI(aside, queue, say)
|
|
||||||
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
|
||||||
// 0349 §4).
|
// 0349 §4).
|
||||||
go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
|
go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
|
||||||
@@ -1236,8 +1083,6 @@ func runLink(ctx context.Context, opts options) error {
|
|||||||
// be a control surface on every machine, reachable by anything that can reach the machine, in
|
// be a control surface on every machine, reachable by anything that can reach the machine, in
|
||||||
// exchange for saving twenty seconds — and the whole security argument rests on there not being
|
// exchange for saving twenty seconds — and the whole security argument rests on there not being
|
||||||
// one. A signal is delivered by the service manager to a process it already supervises.
|
// one. A signal is delivered by the service manager to a process it already supervises.
|
||||||
// (mesh-cli's socket is no such surface: it has no network address and nothing sent to it acts on
|
|
||||||
// the machine — it is asked of the controller, novox/hq ADR 0272 §3.)
|
|
||||||
//
|
//
|
||||||
// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*,
|
// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*,
|
||||||
// and nothing here is being reloaded from a file that a different signal would suit better.
|
// and nothing here is being reloaded from a file that a different signal would suit better.
|
||||||
|
|||||||
@@ -1,25 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/cli"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/meshcli"
|
|
||||||
)
|
|
||||||
|
|
||||||
// serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account
|
|
||||||
// from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served
|
|
||||||
// is said once, and this engine goes on applying; mesh-cli then says the engine does not answer.
|
|
||||||
func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) {
|
|
||||||
if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil {
|
|
||||||
say("mesh-cli is not answered on this machine: " + err.Error())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// forwardOver asks the controller over the queue's link, and follows the line to what came of it.
|
|
||||||
func forwardOver(queue *link.Session) meshcli.Forward {
|
|
||||||
return func(ctx context.Context, asked meshcli.Asked) cli.Answer {
|
|
||||||
return meshcli.AskTheController(ctx, queue.AskController, asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+11
-47
@@ -3,7 +3,6 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -375,42 +374,6 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// operatorKeysFile is the file beside the node's state that holds the operator's public keys, one ed25519 key in base64 a
|
|
||||||
// line (a line that starts with # is a note). The switch's last declaration writes it as a file resource, beside the
|
|
||||||
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
|
|
||||||
const operatorKeysFile = "operator-keys"
|
|
||||||
|
|
||||||
// trustedKeys are the keys a composition in the node's composition bucket may be signed by: the one given (none for the
|
|
||||||
// agent; the enrolment's signer for the user-list follower, which reads only where the bus's user-list file is from
|
|
||||||
// such a composition, never its list) and the operator's, read from operatorKeysFile each time it is asked, so a key
|
|
||||||
// that arrives while the agent runs is trusted at the next composition. A file that cannot be read trusts the one given
|
|
||||||
// alone, and says so.
|
|
||||||
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
|
|
||||||
return func() []ed25519.PublicKey {
|
|
||||||
keys := []ed25519.PublicKey{}
|
|
||||||
if controller := controllerNow(); len(controller) == ed25519.PublicKeySize {
|
|
||||||
keys = append(keys, controller)
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
|
|
||||||
if err != nil {
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
for _, line := range strings.Split(string(raw), "\n") {
|
|
||||||
line = strings.TrimSpace(line)
|
|
||||||
if line == "" || strings.HasPrefix(line, "#") {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
key, err := base64.StdEncoding.DecodeString(line)
|
|
||||||
if err != nil || len(key) != ed25519.PublicKeySize {
|
|
||||||
fmt.Fprintf(os.Stderr, "mesh-host: a line of %s is not an ed25519 public key in base64, and is not trusted\n", operatorKeysFile)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
keys = append(keys, ed25519.PublicKey(key))
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
|
||||||
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
|
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
|
||||||
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
|
||||||
@@ -511,10 +474,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
// The bus's user list, the grants part's alone (ADR 0547 §8, clause 11; issue 1126), written into the file the
|
// The bus's user list, the grants part's alone (ADR 0547 §8, clause 11; issue 1126), written into the file the
|
||||||
// bus module's composition in the node's bucket declares: on every node, and a node whose compositions carry no
|
// bus module this node runs declares, as its agent holds what runs (its own composition, or the declaration kept
|
||||||
// list writes nothing. Those compositions were signed with the enrolment's signer and nobody writes them now; they
|
// before the switch): never read from the node's composition bucket, which the controller wrote. On every node;
|
||||||
// are read for the file's place alone, never for a list, until the place is read elsewhere (milestone B17).
|
// a node that runs no list writes nothing. Looked at again whenever what runs changed (RunsChanged, Settled).
|
||||||
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, func() ed25519.PublicKey { return signer }), say, func() {
|
users := userListFollower(opts, session, func() []delivery.Composed {
|
||||||
|
if now := current.get(); now != nil {
|
||||||
|
return now.Everything()
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}, say, func() {
|
||||||
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
|
||||||
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
// seconds: what failed in them is tried again once that is over, not at its next wait.
|
||||||
time.AfterFunc(reloadSettles, func() {
|
time.AfterFunc(reloadSettles, func() {
|
||||||
@@ -558,12 +526,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
Settled: func(module string) {
|
Settled: func(module string) {
|
||||||
knockFilling(module+" settled", say)
|
knockFilling(module+" settled", say)
|
||||||
judgeWhatRunsNow(opts.state, signer)
|
judgeWhatRunsNow(opts.state, signer)
|
||||||
|
users.RunsChanged()
|
||||||
},
|
},
|
||||||
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
|
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
|
||||||
// a refusal from the engine's start does not wait for some module to settle (issue 924).
|
// a refusal from the engine's start does not wait for some module to settle (issue 924).
|
||||||
RunsChanged: func(why string) {
|
RunsChanged: func(why string) {
|
||||||
knockFilling(why, say)
|
knockFilling(why, say)
|
||||||
judgeWhatRunsNow(opts.state, signer)
|
judgeWhatRunsNow(opts.state, signer)
|
||||||
|
users.RunsChanged()
|
||||||
},
|
},
|
||||||
Say: say,
|
Say: say,
|
||||||
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
|
||||||
@@ -648,14 +618,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
shadowHooks := hooks
|
shadowHooks := hooks
|
||||||
shadowHooks.refused, shadowHooks.certify = nil, nil
|
shadowHooks.refused, shadowHooks.certify = nil, nil
|
||||||
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
shadowEnv := composerEnv(session, node, say, shadowHooks)
|
||||||
// And, after each answer, what runs of an own secret the answer says the vault holds another value of: handed to
|
|
||||||
// the vault only while a node applied the controller's compositions (issue 1126, cause C4; ownresync), so nothing now.
|
|
||||||
composes := &composesNow{}
|
|
||||||
resync := ownResync(session, node, say, shadowEnv, composes)
|
|
||||||
shadow := selfcompose.NewShadow(shadowEnv,
|
shadow := selfcompose.NewShadow(shadowEnv,
|
||||||
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
||||||
session.NoteCandidates(ctx, delivery.WithValuesDiffer(answers))
|
session.NoteCandidates(ctx, delivery.WithValuesDiffer(answers))
|
||||||
go resync.take(ctx, answers)
|
|
||||||
})
|
})
|
||||||
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
|
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
|
||||||
// Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's
|
// Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's
|
||||||
@@ -664,7 +629,6 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
|
|||||||
composer := selfComposer(session, node, hooks, say)
|
composer := selfComposer(session, node, hooks, say)
|
||||||
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
|
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
|
||||||
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
|
||||||
composes.set(delivery.ComposesOwn)
|
|
||||||
env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy
|
env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy
|
||||||
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
|
||||||
session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
|
session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ package main
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
@@ -45,7 +44,6 @@ type itBed struct {
|
|||||||
opts options
|
opts options
|
||||||
session *link.Session
|
session *link.Session
|
||||||
controll ed25519.PrivateKey
|
controll ed25519.PrivateKey
|
||||||
operator ed25519.PrivateKey
|
|
||||||
said []string
|
said []string
|
||||||
// refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met.
|
// refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met.
|
||||||
refusals []string
|
refusals []string
|
||||||
@@ -137,20 +135,12 @@ func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, b.controll, _ = ed25519.GenerateKey(nil)
|
_, b.controll, _ = ed25519.GenerateKey(nil)
|
||||||
_, b.operator, _ = ed25519.GenerateKey(nil)
|
|
||||||
return b
|
return b
|
||||||
}
|
}
|
||||||
|
|
||||||
// start runs the node agent under a session on the bus, trusting the controller's key, and the operator's when asked.
|
// start runs the node agent under a session on the bus.
|
||||||
func (b *itBed) start(trustOperator bool) (stop func()) {
|
func (b *itBed) start() (stop func()) {
|
||||||
b.t.Helper()
|
b.t.Helper()
|
||||||
if trustOperator {
|
|
||||||
pub := b.operator.Public().(ed25519.PublicKey)
|
|
||||||
if err := os.WriteFile(filepath.Join(filepath.Dir(b.opts.state), operatorKeysFile),
|
|
||||||
[]byte("# the operator\n"+base64.StdEncoding.EncodeToString(pub)+"\n"), 0o600); err != nil {
|
|
||||||
b.t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
|
||||||
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
|
||||||
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
|
||||||
@@ -440,10 +430,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
|||||||
builds := kvOf(delivery.BuilderBuildsBucket)
|
builds := kvOf(delivery.BuilderBuildsBucket)
|
||||||
candidates := kvOf(delivery.PlacementCandidatesBucket)
|
candidates := kvOf(delivery.PlacementCandidatesBucket)
|
||||||
packages := kvOf(link.PackagesBucket)
|
packages := kvOf(link.PackagesBucket)
|
||||||
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
|
|
||||||
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path := filepath.Join(b.dir, "web.conf")
|
path := filepath.Join(b.dir, "web.conf")
|
||||||
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
|
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
|
||||||
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` +
|
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` +
|
||||||
@@ -497,7 +483,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
stop := b.start(false)
|
stop := b.start()
|
||||||
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
|
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
|
||||||
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
|
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
|
||||||
time.Sleep(300 * time.Millisecond)
|
time.Sleep(300 * time.Millisecond)
|
||||||
|
|||||||
@@ -26,12 +26,10 @@ const (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
|
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
|
||||||
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of
|
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is.
|
||||||
// the user list on the link open now, told when the key is first heard or changes.
|
|
||||||
type heardKey struct {
|
type heardKey struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
pem string
|
pem string
|
||||||
kick func()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// vaultKey is the vault's key this engine heard.
|
// vaultKey is the vault's key this engine heard.
|
||||||
@@ -43,8 +41,7 @@ func (h *heardKey) get() string {
|
|||||||
return h.pem
|
return h.pem
|
||||||
}
|
}
|
||||||
|
|
||||||
// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer
|
// heard keeps the key a node-key answer gave. An answer with no key (a vault before this one) changes nothing.
|
||||||
// with no key (a vault before this one) changes nothing.
|
|
||||||
func (h *heardKey) heard(answer json.RawMessage) {
|
func (h *heardKey) heard(answer json.RawMessage) {
|
||||||
var a struct {
|
var a struct {
|
||||||
VaultKey string `json:"vault_key"`
|
VaultKey string `json:"vault_key"`
|
||||||
@@ -53,24 +50,8 @@ func (h *heardKey) heard(answer json.RawMessage) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
h.mu.Lock()
|
h.mu.Lock()
|
||||||
changed := a.VaultKey != h.pem
|
|
||||||
h.pem = a.VaultKey
|
h.pem = a.VaultKey
|
||||||
kick := h.kick
|
|
||||||
h.mu.Unlock()
|
h.mu.Unlock()
|
||||||
if changed && kick != nil {
|
|
||||||
kick()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// following names the follower told when the key changes, and tells it at once when a key is known.
|
|
||||||
func (h *heardKey) following(kick func()) {
|
|
||||||
h.mu.Lock()
|
|
||||||
h.kick = kick
|
|
||||||
known := h.pem != ""
|
|
||||||
h.mu.Unlock()
|
|
||||||
if known && kick != nil {
|
|
||||||
kick()
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
|
||||||
|
|||||||
@@ -162,29 +162,22 @@ func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
|
||||||
// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one
|
// answer gives is kept; an answer without one (an older vault) or with no public key changes nothing (ADR 0547 clause
|
||||||
// (an older vault) or with no public key changes nothing (ADR 0547 clause 9).
|
// 9).
|
||||||
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
|
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
|
||||||
if link.NodeKeyContract != 13 {
|
if link.NodeKeyContract != 13 {
|
||||||
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
|
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
|
||||||
}
|
}
|
||||||
h := &heardKey{}
|
h := &heardKey{}
|
||||||
kicks := 0
|
|
||||||
h.following(func() { kicks++ })
|
|
||||||
h.heard(json.RawMessage(`{"recorded":true}`))
|
h.heard(json.RawMessage(`{"recorded":true}`))
|
||||||
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
|
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
|
||||||
if h.get() != "" || kicks != 0 {
|
if h.get() != "" {
|
||||||
t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks)
|
t.Fatalf("an answer with no key was kept: %q", h.get())
|
||||||
}
|
}
|
||||||
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
|
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
|
||||||
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
|
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
|
||||||
h.heard(answer)
|
h.heard(answer)
|
||||||
h.heard(answer)
|
if h.get() != pem {
|
||||||
if h.get() != pem || kicks != 1 {
|
t.Fatalf("key %q, want the key", h.get())
|
||||||
t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks)
|
|
||||||
}
|
|
||||||
h.following(func() { kicks += 10 })
|
|
||||||
if kicks != 11 {
|
|
||||||
t.Fatalf("a follower named after the key was heard was not told: %d", kicks)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,94 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/ownresync"
|
|
||||||
"github.com/novox/mesh-host/internal/selfcompose"
|
|
||||||
)
|
|
||||||
|
|
||||||
// composesNow is whose compositions this run of the agent applies, once decided ("" before).
|
|
||||||
type composesNow struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
now string
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *composesNow) set(s string) { c.mu.Lock(); c.now = s; c.mu.Unlock() }
|
|
||||||
func (c *composesNow) get() string { c.mu.Lock(); defer c.mu.Unlock(); return c.now }
|
|
||||||
|
|
||||||
// resyncStep is the step that hands the vault what runs of an own secret a shadow answer says another value of (issue
|
|
||||||
// 1126, cause C4; internal/ownresync): one pass at a time, and the last answers taken again once the agent has decided
|
|
||||||
// whose compositions it applies.
|
|
||||||
type resyncStep struct {
|
|
||||||
r *ownresync.Resync
|
|
||||||
one sync.Mutex
|
|
||||||
mu sync.Mutex
|
|
||||||
last map[string]delivery.CandidateAnswer
|
|
||||||
// tries and pending are the step taken again after a bound when a read or an ask the bus did not answer stopped it
|
|
||||||
// (Outcome.Retry, issue 1126, 2026-10-28), one wait at a time.
|
|
||||||
tries int
|
|
||||||
pending bool
|
|
||||||
}
|
|
||||||
|
|
||||||
func ownResync(session *link.Session, node string, say func(string), env selfcompose.Env, composes *composesNow) *resyncStep {
|
|
||||||
return &resyncStep{r: ownresync.New(ownresync.Env{Node: node, Composes: composes.get, Running: env.Running,
|
|
||||||
Get: env.Get, Unseal: env.Unseal, Seal: identity.Seal, Say: say,
|
|
||||||
Ask: func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error) {
|
|
||||||
return session.AskVault(ctx, verb, args)
|
|
||||||
}})}
|
|
||||||
}
|
|
||||||
|
|
||||||
// take keeps answers as the last and takes them.
|
|
||||||
func (s *resyncStep) take(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
|
|
||||||
s.mu.Lock()
|
|
||||||
s.last = answers
|
|
||||||
s.mu.Unlock()
|
|
||||||
s.again(ctx)
|
|
||||||
}
|
|
||||||
|
|
||||||
// again takes the last answers, and once more after a bound (selfcompose.LaterWait) while the bus did not answer for one.
|
|
||||||
func (s *resyncStep) again(ctx context.Context) {
|
|
||||||
s.one.Lock()
|
|
||||||
defer s.one.Unlock()
|
|
||||||
s.mu.Lock()
|
|
||||||
last := s.last
|
|
||||||
s.mu.Unlock()
|
|
||||||
if last == nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
retry := false
|
|
||||||
for _, o := range s.r.Take(ctx, last) {
|
|
||||||
retry = retry || o.Retry
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if !retry {
|
|
||||||
s.tries = 0
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if s.pending || ctx.Err() != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
wait := selfcompose.LaterWait(s.tries)
|
|
||||||
s.tries++
|
|
||||||
s.pending = true
|
|
||||||
go func() {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
case <-time.After(wait):
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
s.pending = false
|
|
||||||
s.mu.Unlock()
|
|
||||||
if ctx.Err() == nil {
|
|
||||||
s.again(ctx)
|
|
||||||
}
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
@@ -1,94 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/ed25519"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
"github.com/novox/mesh-host/internal/tunnel"
|
|
||||||
)
|
|
||||||
|
|
||||||
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
|
|
||||||
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
|
|
||||||
// proof signed by the identity key, over the previous key, the new one and the tunnel.
|
|
||||||
|
|
||||||
func anEnrolledNode(t *testing.T) identity.Identity {
|
|
||||||
t.Helper()
|
|
||||||
mine, err := identity.Generate("anchor")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Overlay, err = identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
|
|
||||||
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
|
|
||||||
return mine
|
|
||||||
}
|
|
||||||
|
|
||||||
func aFoundTunnel(t *testing.T) tunnel.Found {
|
|
||||||
t.Helper()
|
|
||||||
private, err := identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
|
|
||||||
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
|
|
||||||
return found
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
|
|
||||||
mine := anEnrolledNode(t)
|
|
||||||
found := aFoundTunnel(t)
|
|
||||||
before := mine.Overlay.Public
|
|
||||||
|
|
||||||
taken, rekey, err := rekeyOnto(mine, found)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
|
|
||||||
t.Fatal("the overlay key is not the tunnel's")
|
|
||||||
}
|
|
||||||
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
|
|
||||||
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
|
|
||||||
taken.Membership.Broker != mine.Membership.Broker {
|
|
||||||
t.Fatal("something other than the overlay key moved")
|
|
||||||
}
|
|
||||||
if taken.OverlayBefore != before {
|
|
||||||
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
|
|
||||||
}
|
|
||||||
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
|
|
||||||
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
|
|
||||||
t.Fatalf("the rekey does not say what moved: %+v", rekey)
|
|
||||||
}
|
|
||||||
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
|
|
||||||
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
|
||||||
t.Fatal("the rekey is not signed by this node's identity key over what it says")
|
|
||||||
}
|
|
||||||
if ed25519.Verify(ed25519.PublicKey(mine.Public),
|
|
||||||
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
|
|
||||||
t.Fatal("the proof is not bound to the node")
|
|
||||||
}
|
|
||||||
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
|
|
||||||
if strings.Contains(said, found.PrivateKey()) {
|
|
||||||
t.Fatal("the private key travels")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
|
|
||||||
// names is still the one before the take, so the mesh can tell a repeat from a replay.
|
|
||||||
again, second, err := rekeyOnto(taken, found)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
|
|
||||||
t.Fatalf("a take run again does not name the key before the first: %+v", second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -36,10 +36,6 @@ type meshTrust struct {
|
|||||||
// newMeshTrust trusts nothing until the vault's key is read.
|
// newMeshTrust trusts nothing until the vault's key is read.
|
||||||
func newMeshTrust() *meshTrust { return &meshTrust{} }
|
func newMeshTrust() *meshTrust { return &meshTrust{} }
|
||||||
|
|
||||||
// noControllerKey is the controller's key as this engine trusts it: none. A link told it refuses every ask signed as
|
|
||||||
// the controller's, where a link told nothing would verify it with the membership's signer (link.HandOverCheck).
|
|
||||||
func noControllerKey() ed25519.PublicKey { return nil }
|
|
||||||
|
|
||||||
// onChange names who is told when what it trusts changed.
|
// onChange names who is told when what it trusts changed.
|
||||||
func (t *meshTrust) onChange(f func()) {
|
func (t *meshTrust) onChange(f func()) {
|
||||||
t.mu.Lock()
|
t.mu.Lock()
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -26,13 +25,12 @@ import (
|
|||||||
|
|
||||||
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
|
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
|
||||||
// agent trusts, the write alone, and the have.
|
// agent trusts, the write alone, and the have.
|
||||||
func userListFollower(opts options, session *link.Session, composedBucket string, trusted func() []ed25519.PublicKey,
|
func userListFollower(opts options, session *link.Session, running func() []delivery.Composed,
|
||||||
say link.Announce, wrote func()) *userlist.Follower {
|
say link.Announce, wrote func()) *userlist.Follower {
|
||||||
return userlist.New(userlist.Env{
|
return userlist.New(userlist.Env{
|
||||||
Node: session.Membership.Node,
|
Node: session.Membership.Node,
|
||||||
Composed: bucketWatch{session, composedBucket},
|
Running: running,
|
||||||
Trusted: trusted,
|
OnDisk: userListOnDisk,
|
||||||
OnDisk: userListOnDisk,
|
|
||||||
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
|
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
|
||||||
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
|
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
|
||||||
if changed && wrote != nil {
|
if changed && wrote != nil {
|
||||||
|
|||||||
@@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool {
|
|||||||
|
|
||||||
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
|
||||||
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
|
||||||
host, _ := os.Hostname()
|
|
||||||
w := &witness.Watcher{
|
w := &witness.Watcher{
|
||||||
Root: apply.DaemonRoot(), Node: node, Host: host,
|
Root: apply.DaemonRoot(), Node: node,
|
||||||
Asker: func() link.Asker { return queue.Asker() },
|
Asker: func() link.Asker { return queue.Asker() },
|
||||||
Run: witness.Runner(apply.ExecRunner),
|
Run: witness.Runner(apply.ExecRunner),
|
||||||
Hold: func(f func()) {
|
Hold: func(f func()) {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
|
|||||||
go 1.26.0
|
go 1.26.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
|
||||||
github.com/nats-io/nats-server/v2 v2.11.17
|
github.com/nats-io/nats-server/v2 v2.11.17
|
||||||
github.com/nats-io/nats.go v1.54.0
|
github.com/nats-io/nats.go v1.54.0
|
||||||
golang.org/x/crypto v0.57.0
|
golang.org/x/crypto v0.57.0
|
||||||
|
|||||||
@@ -1,9 +1,5 @@
|
|||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
|
|
||||||
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
|
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
|
||||||
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
|
||||||
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
|
||||||
|
|||||||
@@ -72,36 +72,36 @@ func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
|
|||||||
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
|
||||||
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
|
||||||
|
|
||||||
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"})
|
oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"})
|
||||||
newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"})
|
newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"})
|
||||||
controller := func(body []byte, digest, env string) string {
|
runtime := func(body []byte, digest, env string) string {
|
||||||
return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) +
|
return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
|
||||||
`","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}`
|
`","digest":"` + digest + `","run":["./node-tools"],"env":{"X":"` + env + `"}}`
|
||||||
}
|
}
|
||||||
_, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{},
|
_, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest, "1")), store.State{},
|
||||||
store.OriginDeclared, run, nil, nil)
|
store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state,
|
_, state, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "1")), state,
|
||||||
store.OriginDeclared, run, nil, nil)
|
store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run),
|
if _, err := witness.Restore(context.Background(), bundles, "node-tools", "never answered", witness.Runner(run),
|
||||||
time.Now()); err != nil {
|
time.Now()); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
// The mesh still declares the new build, and its unit changes: the process is re-placed.
|
// The mesh still declares the new build, and its unit changes: the process is re-placed.
|
||||||
report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state,
|
report, _, err := Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "2")), state,
|
||||||
store.OriginDeclared, run, nil, nil)
|
store.OriginDeclared, run, nil, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" {
|
if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "old\n" {
|
||||||
t.Fatalf("the rolled-back build was placed again: %q", got)
|
t.Fatalf("the rolled-back build was placed again: %q", got)
|
||||||
}
|
}
|
||||||
if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") {
|
if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "rolled back on this machine") {
|
||||||
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -35,10 +35,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/declaration"
|
"git.novox.be/novox/mesh-sdk/go/declaration"
|
||||||
"github.com/novox/mesh-host/internal/meshcli"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Socket is where the node-engine accepts a write of a filled file, beside mesh-cli's socket.
|
// Socket is where the node-engine accepts a write of a filled file.
|
||||||
const Socket = "/run/nox-mesh-host/fill.sock"
|
const Socket = "/run/nox-mesh-host/fill.sock"
|
||||||
|
|
||||||
// MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive.
|
// MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive.
|
||||||
@@ -293,11 +292,11 @@ func UIDOf(account string) (uint32, error) {
|
|||||||
return uint32(id), nil
|
return uint32(id), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// peerOf and sameNamespaces are the kernel's word on the asking process, as mesh-cli's socket reads it. Variables so
|
// peerOf and sameNamespaces are the kernel's word on the asking process (peer_linux.go). Variables so
|
||||||
// a test can say.
|
// a test can say.
|
||||||
var (
|
var (
|
||||||
peerOf = meshcli.PeerCredentials
|
peerOf = peerCredentials
|
||||||
sameNamespaces = meshcli.SameNamespaces
|
sameNamespaces = namespacesOf
|
||||||
)
|
)
|
||||||
|
|
||||||
// within bounds how long an accepted connection may take to say its request, and refusedWithin how long a busy one
|
// within bounds how long an accepted connection may take to say its request, and refusedWithin how long a busy one
|
||||||
@@ -352,7 +351,7 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error
|
|||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||||
return fmt.Errorf("the filled files' socket directory: %w", err)
|
return fmt.Errorf("the filled files' socket directory: %w", err)
|
||||||
}
|
}
|
||||||
if problem := meshcli.NotOwnDirectory(dir); problem != "" {
|
if problem := notOwnDirectory(dir); problem != "" {
|
||||||
return fmt.Errorf("filled files are not written on request on this machine: %s", problem)
|
return fmt.Errorf("filled files are not written on request on this machine: %s", problem)
|
||||||
}
|
}
|
||||||
if err := os.Chmod(dir, 0o755); err != nil {
|
if err := os.Chmod(dir, 0o755); err != nil {
|
||||||
|
|||||||
@@ -532,7 +532,7 @@ func TestTheSocketGoesOnAfterAnAcceptFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// The socket's directory is given the mode the socket needs, as mesh-cli's is.
|
// The socket's directory is given the mode the socket needs.
|
||||||
func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) {
|
func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) {
|
||||||
dir, err := os.MkdirTemp("", "fill")
|
dir, err := os.MkdirTemp("", "fill")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build linux
|
//go:build linux
|
||||||
|
|
||||||
package meshcli
|
package filling
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
//go:build !linux
|
//go:build !linux
|
||||||
|
|
||||||
package meshcli
|
package filling
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The enrolment conversation, as the host's own words for it.
|
|
||||||
//
|
|
||||||
// **Its own seam rather than part of Link**, because almost nothing about it is the same. The
|
|
||||||
// credential is a one-time secret rather than this node's own; there is no declaration to hear; the
|
|
||||||
// whole exchange is a single question asked and possibly asked again. And the stakes differ: a node
|
|
||||||
// that fails here is not in the mesh at all, where a node that fails in Link has merely lost touch
|
|
||||||
// with one it belongs to.
|
|
||||||
|
|
||||||
// Approach is how a node reaches a mesh it does not yet belong to.
|
|
||||||
//
|
|
||||||
// The three things a token carries about where to go, and nothing about who is asking: the address,
|
|
||||||
// the certificate that address must present, and which bus is at the other end — the mesh's own
|
|
||||||
// (hearing.go), and a token naming any other is refused before anything is sent.
|
|
||||||
type Approach struct {
|
|
||||||
Address string
|
|
||||||
Fingerprint string
|
|
||||||
Transport string
|
|
||||||
}
|
|
||||||
|
|
||||||
// Asking is one open enrolment conversation.
|
|
||||||
type Asking interface {
|
|
||||||
// Ask puts the request to the mesh and waits for one answer, or says why none came.
|
|
||||||
//
|
|
||||||
// Called again, with the same bytes, while the mesh says "try again": the keys this node
|
|
||||||
// generated are the ones it keeps, so the same request is the same enrolment and the mesh holds
|
|
||||||
// the token for it (novox/hq issue 083).
|
|
||||||
Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error)
|
|
||||||
|
|
||||||
// Close lets go of the connection made with the token.
|
|
||||||
Close()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Present opens an enrolment conversation with the mesh.
|
|
||||||
//
|
|
||||||
// The connection is made before anything is sent, and the certificate is checked while it is being
|
|
||||||
// made — so a node pointed at the wrong bus finds out before its token has left the machine (ADR
|
|
||||||
// 0004).
|
|
||||||
func Present(ctx context.Context, to Approach, node, secret string,
|
|
||||||
timeout time.Duration) (Asking, error) {
|
|
||||||
|
|
||||||
if to.Transport != OnNATS {
|
|
||||||
return nil, fmt.Errorf("this token is for the %q bus, and the mesh's bus is %s", to.Transport, OnNATS)
|
|
||||||
}
|
|
||||||
return presentNats(ctx, to, node, secret, timeout)
|
|
||||||
}
|
|
||||||
@@ -1,186 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/rand"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The enrolment conversation on the bus being built.
|
|
||||||
//
|
|
||||||
// **The reply address is the whole of what changes**, and it changes for a reason the transport
|
|
||||||
// forces rather than a preference. Core NATS request/reply puts the caller's inbox in the message's
|
|
||||||
// reply field and a plain responder answers it — but this request goes into a stream, and a message
|
|
||||||
// a JetStream consumer delivers has had that field claimed for the consumer's own ack address. So by
|
|
||||||
// the time the controller reads the request, the transport's reply field names where the
|
|
||||||
// *controller* must acknowledge. Verified against a running server (design 25 §2).
|
|
||||||
//
|
|
||||||
// The address therefore travels as a field of the request, and this subscribes it before publishing:
|
|
||||||
// a node that published first could miss an answer to a question nobody was listening for.
|
|
||||||
|
|
||||||
// enrolInbox is where a node enrolling waits.
|
|
||||||
//
|
|
||||||
// Under `_INBOX.enrol.<node>.`, which is exactly what its enrolment user may subscribe and no
|
|
||||||
// wider — so an answer sealed to one machine cannot be read by another enrolling beside it. The
|
|
||||||
// random tail is this attempt's own: a reply left over from an attempt that timed out is not the
|
|
||||||
// answer to this question, which is what the correlation id does on the other transport.
|
|
||||||
func enrolInbox(node string) (string, error) {
|
|
||||||
tail := make([]byte, 8)
|
|
||||||
if _, err := rand.Read(tail); err != nil {
|
|
||||||
return "", fmt.Errorf("cannot make a reply address: %w", err)
|
|
||||||
}
|
|
||||||
return "_INBOX.enrol." + node + "." + hex.EncodeToString(tail), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
type natsAsking struct {
|
|
||||||
conn *nats.Conn
|
|
||||||
js nats.JetStreamContext
|
|
||||||
inbox string
|
|
||||||
answers *nats.Subscription
|
|
||||||
lost chan error
|
|
||||||
}
|
|
||||||
|
|
||||||
func presentNats(_ context.Context, to Approach, node, secret string,
|
|
||||||
timeout time.Duration) (Asking, error) {
|
|
||||||
|
|
||||||
config, err := PinnedConfig(to.Fingerprint)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
inbox, err := enrolInbox(node)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
lost := make(chan error, 1)
|
|
||||||
// The user is this token's own — `enrol.<node>`, which may publish the enrolment subject and
|
|
||||||
// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
|
|
||||||
// string the request claims, so the server proves somebody holds the token and the request
|
|
||||||
// proves the same thing to the controller without it having to ask the server who connected.
|
|
||||||
// **Its own inbox space, because that is the only one it may listen in** (novox/hq
|
|
||||||
// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
|
|
||||||
// client picks, and the client's default is `_INBOX.<random>` — which this user may not
|
|
||||||
// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
|
|
||||||
// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
|
|
||||||
// pick its inboxes there; the reply address below is in the same space for the same reason.
|
|
||||||
conn, err := nats.Connect(natsURL(to.Address),
|
|
||||||
nats.Secure(config),
|
|
||||||
nats.CustomInboxPrefix("_INBOX.enrol."+node),
|
|
||||||
nats.UserInfo("enrol."+node, secret),
|
|
||||||
nats.Name("mesh-host/enrol/"+node),
|
|
||||||
nats.Timeout(timeout),
|
|
||||||
nats.NoReconnect(),
|
|
||||||
nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
|
|
||||||
select {
|
|
||||||
case lost <- fmt.Errorf("the bus closed the connection: %w", err):
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
}),
|
|
||||||
)
|
|
||||||
if err != nil {
|
|
||||||
if errors.Is(err, ErrWrongCertificate) {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
// Not quoted back with the credential: the secret is one-time and still a secret.
|
|
||||||
return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", to.Address, node, err)
|
|
||||||
}
|
|
||||||
js, err := conn.JetStream()
|
|
||||||
if err != nil {
|
|
||||||
conn.Close()
|
|
||||||
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", to.Address, err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Subscribed before anything is published, so an answer cannot arrive before there is anywhere
|
|
||||||
// for it to land.
|
|
||||||
answers, err := conn.SubscribeSync(inbox)
|
|
||||||
if err != nil {
|
|
||||||
conn.Close()
|
|
||||||
return nil, fmt.Errorf("this node cannot listen for the mesh's answer: %w", err)
|
|
||||||
}
|
|
||||||
if err := conn.Flush(); err != nil {
|
|
||||||
conn.Close()
|
|
||||||
return nil, fmt.Errorf("this node's reply address did not reach the bus: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: lost}, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *natsAsking) Close() {
|
|
||||||
if a.answers != nil {
|
|
||||||
_ = a.answers.Unsubscribe()
|
|
||||||
}
|
|
||||||
if a.conn != nil {
|
|
||||||
a.conn.Close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Ask publishes the request with this attempt's reply address written into it, and waits there.
|
|
||||||
func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) {
|
|
||||||
addressed, err := withReplyTo(request, a.inbox)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
publish, cancel := context.WithTimeout(ctx, wait)
|
|
||||||
defer cancel()
|
|
||||||
// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
|
|
||||||
// assumed, because the node has nothing else to go on.
|
|
||||||
//
|
|
||||||
// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
|
|
||||||
// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
|
|
||||||
// credential, which replaced the first, which is the one the node had already been given. The
|
|
||||||
// machine then reconnected for ever as a user whose password the mesh had rotated out from under
|
|
||||||
// it, and the controller's log said "enrolled anchor" twice in the same second.
|
|
||||||
//
|
|
||||||
// The id is the message: the same bytes carry the same id, so the stream discards the client's
|
|
||||||
// own retry, and a genuine second attempt — which carries a new reply address — is a different
|
|
||||||
// message and is let through.
|
|
||||||
sum := sha256.Sum256(addressed)
|
|
||||||
if _, err := a.js.Publish(EnrolSubject, addressed,
|
|
||||||
nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
|
|
||||||
return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Waited for rather than assumed. A published message that nothing answers means the controller
|
|
||||||
// is not running, and a node that carried on regardless would believe it had joined a mesh that
|
|
||||||
// has never heard of it.
|
|
||||||
//
|
|
||||||
// **The wait may legitimately be several store-window cycles long**: the controller naks the
|
|
||||||
// request with a delay while its store is restarting, and the node is waiting on the other side
|
|
||||||
// of that — which is exactly the combination that would have delivered the answer to a caller
|
|
||||||
// who had given up, had the address travelled in the transport's field.
|
|
||||||
answered, cancelAnswer := context.WithTimeout(ctx, wait)
|
|
||||||
defer cancelAnswer()
|
|
||||||
for {
|
|
||||||
msg, err := a.answers.NextMsgWithContext(answered)
|
|
||||||
switch {
|
|
||||||
case err == nil:
|
|
||||||
return msg.Data, nil
|
|
||||||
case errors.Is(err, context.DeadlineExceeded):
|
|
||||||
select {
|
|
||||||
case reason := <-a.lost:
|
|
||||||
return nil, reason
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"the bus accepted this node's connection and nothing answered within %s. The mesh's "+
|
|
||||||
"bus is running and its controller is not", wait)
|
|
||||||
case errors.Is(err, context.Canceled):
|
|
||||||
return nil, ctx.Err()
|
|
||||||
default:
|
|
||||||
select {
|
|
||||||
case reason := <-a.lost:
|
|
||||||
return nil, reason
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
return nil, fmt.Errorf("waiting for the mesh's answer: %w", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,192 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The enrolment round trip against a real server.
|
|
||||||
//
|
|
||||||
// **This is the test that keeps a reason from becoming folklore.** The reply address travels in the
|
|
||||||
// request's payload because a JetStream consumer's delivery has had the transport's reply field
|
|
||||||
// claimed for its own ack address — which is a fact about a server, not a rule anybody can check by
|
|
||||||
// reading. Both halves are asserted here: that the field really is eaten, and that the answer
|
|
||||||
// reaches the node anyway.
|
|
||||||
//
|
|
||||||
// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js
|
|
||||||
// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNatsAnEnrolment
|
|
||||||
|
|
||||||
// asking is a conversation on a bus with no TLS. Built directly rather than through Present because
|
|
||||||
// the pin is what Present adds and PinnedConfig's own tests cover it; what is under test here is the
|
|
||||||
// address the answer comes back on.
|
|
||||||
func asking(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, node string) *natsAsking {
|
|
||||||
t.Helper()
|
|
||||||
inbox, err := enrolInbox(node)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
answers, err := conn.SubscribeSync(inbox)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := conn.Flush(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
a := &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: make(chan error, 1)}
|
|
||||||
// **Not a.Close.** Close closes the connection, and this connection is the test's own, shared with
|
|
||||||
// the stand-in. Cleanups run last-in first-out and the stand-in was made first, so its cleanup
|
|
||||||
// would run on a closed connection: Unsubscribe returns ErrConnectionClosed without deleting the
|
|
||||||
// consumer it made, and every request after that fails. The consumer then outlives the test on a
|
|
||||||
// workqueue stream, which allows one consumer per filter, and the next test cannot subscribe
|
|
||||||
// (issue 604). The connection is closed by aBus, last of all.
|
|
||||||
t.Cleanup(func() { _ = answers.Unsubscribe() })
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
|
|
||||||
// theMeshAnswers stands in for the controller: it consumes the enrolment off the stream, reads the
|
|
||||||
// reply address out of the payload — never from the transport field — and answers there. It reports
|
|
||||||
// what the transport field actually held, which is the claim design 25 §2 rests on.
|
|
||||||
func theMeshAnswers(t *testing.T, conn *nats.Conn, js nats.JetStreamContext,
|
|
||||||
reply EnrolReply) <-chan string {
|
|
||||||
t.Helper()
|
|
||||||
sawReplyField := make(chan string, 1)
|
|
||||||
// **A consumer of this test's own, gone before it starts and gone when it ends.** The stand-in
|
|
||||||
// used to be one durable name shared by every test, released by Unsubscribe alone. Unsubscribe
|
|
||||||
// only asks the server to let go: it deletes the consumer only when this subscription made it,
|
|
||||||
// and the server drops the binding on its own time. A test that started in that window (or one
|
|
||||||
// that found the consumer left by an earlier run on a long-lived bus) was refused with "consumer
|
|
||||||
// is already bound to a subscription". Per-test names remove the sharing; the delete-and-wait
|
|
||||||
// before and after removes the dependence on the server's timing.
|
|
||||||
durable := standinName(t)
|
|
||||||
stream, err := js.StreamNameBySubject(EnrolSubject)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
standinGone(t, js, stream, durable)
|
|
||||||
sub, err := js.Subscribe(EnrolSubject, func(msg *nats.Msg) {
|
|
||||||
select {
|
|
||||||
case sawReplyField <- msg.Reply:
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
var addressed struct {
|
|
||||||
ReplyTo string `json:"reply_to"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(msg.Data, &addressed); err != nil || addressed.ReplyTo == "" {
|
|
||||||
_ = msg.Ack()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(reply)
|
|
||||||
// Published explicitly to the address the payload named, never msg.Respond — which would
|
|
||||||
// send it to whatever the transport's reply field holds, and that is the point.
|
|
||||||
_ = conn.Publish(addressed.ReplyTo, body)
|
|
||||||
_ = msg.Ack()
|
|
||||||
}, nats.Durable(durable), nats.ManualAck())
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Cleanup(func() {
|
|
||||||
_ = sub.Unsubscribe()
|
|
||||||
standinGone(t, js, stream, durable)
|
|
||||||
})
|
|
||||||
return sawReplyField
|
|
||||||
}
|
|
||||||
|
|
||||||
// standinName is the stand-in's durable name for one test: the test's name with every character a
|
|
||||||
// consumer name may not hold replaced.
|
|
||||||
func standinName(t *testing.T) string {
|
|
||||||
t.Helper()
|
|
||||||
return "controller-standin-" + strings.Map(func(r rune) rune {
|
|
||||||
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
|
|
||||||
return r
|
|
||||||
}
|
|
||||||
return '_'
|
|
||||||
}, t.Name())
|
|
||||||
}
|
|
||||||
|
|
||||||
// standinGone deletes the durable consumer if it exists and waits until the server no longer reports
|
|
||||||
// it, so that nothing after this call can meet its binding.
|
|
||||||
func standinGone(t *testing.T, js nats.JetStreamContext, stream, durable string) {
|
|
||||||
t.Helper()
|
|
||||||
deadline := time.Now().Add(5 * time.Second)
|
|
||||||
for {
|
|
||||||
_, err := js.ConsumerInfo(stream, durable)
|
|
||||||
if errors.Is(err, nats.ErrConsumerNotFound) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if errors.Is(err, nats.ErrConnectionClosed) {
|
|
||||||
t.Fatalf("the connection closed before consumer %s was deleted: %v", durable, err)
|
|
||||||
}
|
|
||||||
_ = js.DeleteConsumer(stream, durable)
|
|
||||||
if time.Now().After(deadline) {
|
|
||||||
t.Fatalf("consumer %s on %s still exists 5s after it was deleted", durable, stream)
|
|
||||||
}
|
|
||||||
time.Sleep(20 * time.Millisecond)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// An enrolment is answered on the address the request carried, and the transport's own reply field
|
|
||||||
// held something else entirely.
|
|
||||||
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
|
|
||||||
conn, js := aBus(t)
|
|
||||||
const node = "joining"
|
|
||||||
|
|
||||||
sawReplyField := theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node, Password: "p"})
|
|
||||||
a := asking(t, conn, js, node)
|
|
||||||
|
|
||||||
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
|
|
||||||
answer, err := a.Ask(context.Background(), request, 8*time.Second)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("no answer reached the node: %v", err)
|
|
||||||
}
|
|
||||||
var reply EnrolReply
|
|
||||||
if err := json.Unmarshal(answer, &reply); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if !reply.Accepted || reply.Node != node {
|
|
||||||
t.Fatalf("the answer was not the mesh's: %+v", reply)
|
|
||||||
}
|
|
||||||
|
|
||||||
// And the field the answer would have gone to, had it used the transport's: the consumer's own
|
|
||||||
// ack address. If a future server stopped doing this, the payload-borne address would still
|
|
||||||
// work and this line is what would say the reason had changed.
|
|
||||||
select {
|
|
||||||
case field := <-sawReplyField:
|
|
||||||
if field == a.inbox {
|
|
||||||
t.Fatalf("the transport's reply field held this node's inbox (%s), so the payload "+
|
|
||||||
"address is no longer load-bearing — check design 25 §2 before relying on it", field)
|
|
||||||
}
|
|
||||||
if field == "" {
|
|
||||||
t.Fatal("the transport's reply field was empty rather than claimed, which is a third " +
|
|
||||||
"behaviour from the two design 25 §2 describes")
|
|
||||||
}
|
|
||||||
case <-time.After(2 * time.Second):
|
|
||||||
t.Fatal("the stand-in never saw the request")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A request that names no reply address is not answered, and the node says so as a mesh that is not
|
|
||||||
// running rather than hanging. The controller has nowhere to send an answer, which is the failure
|
|
||||||
// the payload field exists to make impossible — asserted so that a request built without it fails
|
|
||||||
// loudly here rather than quietly on a machine.
|
|
||||||
func TestNatsAnEnrolmentWithNoReplyAddressIsNotAnswered(t *testing.T) {
|
|
||||||
conn, js := aBus(t)
|
|
||||||
const node = "silent"
|
|
||||||
|
|
||||||
theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node})
|
|
||||||
a := asking(t, conn, js, node)
|
|
||||||
|
|
||||||
// Published without going through Ask, so the reply address is genuinely absent.
|
|
||||||
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
|
|
||||||
if _, err := js.Publish(EnrolSubject, request); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := a.Ask(context.Background(), []byte(`{"node":"`+node+`"}`), 0); err == nil {
|
|
||||||
t.Fatal("a node with no answer coming was told it had one")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
+5
-40
@@ -10,28 +10,14 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Bus is what a host needs of the mesh's bus, in the mesh's own words.
|
// Bus is the link open now as the session holds it. What it can do is asked of it by what it implements (HaveBus,
|
||||||
//
|
// HealthBus, ToolBus, FilesBus): the report it once published into the controller's CONTROL stream went with the
|
||||||
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
|
// controller (issue 1126), and what the node applied is said in its haves.
|
||||||
// the same kind of statement and the difference is the whole of this interface — one must arrive
|
type Bus interface{}
|
||||||
// and one must not be insisted on.
|
|
||||||
//
|
|
||||||
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
|
|
||||||
// interface over its own client libraries, not a contract shared with the controller. The two
|
|
||||||
// agree because a conformance fixture holds them to one envelope, which is the only kind of
|
|
||||||
// agreement that survives being in different repositories.
|
|
||||||
type Bus interface {
|
|
||||||
// Report says what this node applied. **It must arrive.** A report that fails leaves the
|
|
||||||
// mesh believing the node never answered while the node believes it did, and the two go on
|
|
||||||
// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
|
|
||||||
// finding. Returns false when it could not be delivered, so the caller can say so.
|
|
||||||
Report(ctx context.Context, node string, body []byte) error
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- The bus the mesh runs on today -----------------------------------------------------------
|
// --- The bus the mesh runs on today -----------------------------------------------------------
|
||||||
|
|
||||||
// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
|
// OverNATS is the bus as a connection.
|
||||||
// the controller's store restarting; a heartbeat does not, because it must not.
|
|
||||||
type OverNATS struct {
|
type OverNATS struct {
|
||||||
Conn *nats.Conn
|
Conn *nats.Conn
|
||||||
JS nats.JetStreamContext
|
JS nats.JetStreamContext
|
||||||
@@ -39,11 +25,6 @@ type OverNATS struct {
|
|||||||
Node string
|
Node string
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReportSubject is this node's own, and no other node's: a host's account may
|
|
||||||
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
|
|
||||||
// which node a report is about.
|
|
||||||
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
|
|
||||||
|
|
||||||
// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
|
// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
|
||||||
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
|
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
|
||||||
// statement lost is said again within a minute while anything is not healthy.
|
// statement lost is said again within a minute while anything is not healthy.
|
||||||
@@ -65,11 +46,6 @@ func (b OverNATS) Health(ctx context.Context, node string, body []byte) error {
|
|||||||
return b.Conn.FlushWithContext(flush)
|
return b.Conn.FlushWithContext(flush)
|
||||||
}
|
}
|
||||||
|
|
||||||
// CLISubject is where this node's engine asks the controller a line `mesh-cli` was given on this machine
|
|
||||||
// (novox/hq ADR 0272 §3): its own node's, inside the grant every engine already has (`mesh.control.<node>.>`), so the
|
|
||||||
// bus server is the authority on which node asked. Core NATS, never the CONTROL stream, which names its own subjects.
|
|
||||||
func CLISubject(node string) string { return "mesh.control." + node + ".cli" }
|
|
||||||
|
|
||||||
// ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared
|
// ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared
|
||||||
// tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other.
|
// tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other.
|
||||||
func ToolSubject(module, tool, node string) string {
|
func ToolSubject(module, tool, node string) string {
|
||||||
@@ -90,17 +66,6 @@ func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte,
|
|||||||
return msg.Data, nil
|
return msg.Data, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
|
|
||||||
// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
|
|
||||||
// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
|
|
||||||
// the message is redelivered; a publish the bus never accepted must fail here rather than
|
|
||||||
// be assumed.
|
|
||||||
if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
|
|
||||||
return fmt.Errorf("reporting: %w", err)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes.
|
// HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes.
|
||||||
// **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket
|
// **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket
|
||||||
// is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every
|
// is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every
|
||||||
|
|||||||
@@ -17,10 +17,6 @@ import (
|
|||||||
// mesh-host.packages (delivery.EnginePackagesRead).
|
// mesh-host.packages (delivery.EnginePackagesRead).
|
||||||
const PackagesBucket = "mesh-host_" + delivery.EnginePackagesState
|
const PackagesBucket = "mesh-host_" + delivery.EnginePackagesState
|
||||||
|
|
||||||
// MirrorPackagesBucket is the controller's declared-packages mirror (ADR 0507 §5), which the engine reads its own node's
|
|
||||||
// key of until the lease write, to compare (the node seat's verb packages).
|
|
||||||
const MirrorPackagesBucket = "mesh-controller_declared-packages"
|
|
||||||
|
|
||||||
// packagesStore binds PackagesBucket over the link open now: bound, never created.
|
// packagesStore binds PackagesBucket over the link open now: bound, never created.
|
||||||
func (l *natsLink) packagesStore() (nats.KeyValue, error) {
|
func (l *natsLink) packagesStore() (nats.KeyValue, error) {
|
||||||
kv, err := l.js.KeyValue(PackagesBucket)
|
kv, err := l.js.KeyValue(PackagesBucket)
|
||||||
@@ -83,51 +79,33 @@ func (p EnginePackages) Put(_ context.Context, key string, value []byte) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
// PackagesView is the node seat's verb packages' answer (ADR 0565 §6): this node's key of the engine's state packages,
|
// PackagesView is the node seat's verb packages' answer (ADR 0565 §6): this node's key of the engine's state packages.
|
||||||
// and until the lease write the controller's mirror's key for the node and every package they hold differently.
|
// The controller's mirror it was once compared with ended with the controller (issue 1126).
|
||||||
type PackagesView struct {
|
type PackagesView struct {
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
// Key is the node's key as the bus holds it (Source); null when the engine has written none.
|
// Key is the node's key as the bus holds it (Source); null when the engine has written none.
|
||||||
Source string `json:"source"`
|
Source string `json:"source"`
|
||||||
Key *delivery.DeclaredPackages `json:"key"`
|
Key *delivery.DeclaredPackages `json:"key"`
|
||||||
KeyError string `json:"key_error,omitempty"`
|
KeyError string `json:"key_error,omitempty"`
|
||||||
// Mirror is the controller's mirror's key for the node (MirrorSource); null when it holds none or cannot be read.
|
|
||||||
MirrorSource string `json:"mirror_source"`
|
|
||||||
Mirror *delivery.DeclaredPackages `json:"mirror"`
|
|
||||||
MirrorError string `json:"mirror_error,omitempty"`
|
|
||||||
// Differences is every package the two hold differently, read only when both are; Agree when there is none.
|
|
||||||
Differences []delivery.PackageDifference `json:"differences"`
|
|
||||||
Agree bool `json:"agree"`
|
|
||||||
// Wrote says, for write, whether the key was written again (false: it already held what the engine composed).
|
// Wrote says, for write, whether the key was written again (false: it already held what the engine composed).
|
||||||
Wrote *bool `json:"wrote,omitempty"`
|
Wrote *bool `json:"wrote,omitempty"`
|
||||||
Note string `json:"note,omitempty"`
|
Note string `json:"note,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// PackagesAnswer is the verb's answer from the two keys as read (nil: no key), each with its read's error.
|
// PackagesAnswer is the verb's answer from the key as read (nil: no key), with its read's error.
|
||||||
func PackagesAnswer(node string, key, mirror []byte, keyErr, mirrorErr error) PackagesView {
|
func PackagesAnswer(node string, key []byte, keyErr error) PackagesView {
|
||||||
v := PackagesView{Node: node, Source: delivery.EnginePackagesRead, MirrorSource: "mesh-controller.declared-packages",
|
v := PackagesView{Node: node, Source: delivery.EnginePackagesRead}
|
||||||
Differences: []delivery.PackageDifference{}}
|
if keyErr == nil && key != nil {
|
||||||
read := func(raw []byte, err error, into **delivery.DeclaredPackages, said *string) {
|
var d delivery.DeclaredPackages
|
||||||
if err == nil && raw != nil {
|
if keyErr = json.Unmarshal(key, &d); keyErr == nil {
|
||||||
var d delivery.DeclaredPackages
|
v.Key = &d
|
||||||
if err = json.Unmarshal(raw, &d); err == nil {
|
|
||||||
*into = &d
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
*said = err.Error()
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
read(key, keyErr, &v.Key, &v.KeyError)
|
|
||||||
read(mirror, mirrorErr, &v.Mirror, &v.MirrorError)
|
|
||||||
switch {
|
switch {
|
||||||
case v.Key != nil && v.Mirror != nil:
|
case keyErr != nil:
|
||||||
v.Differences = delivery.DiffDeclaredPackages(*v.Key, *v.Mirror)
|
v.KeyError = keyErr.Error()
|
||||||
v.Agree = len(v.Differences) == 0
|
case v.Key == nil:
|
||||||
case v.Key == nil && v.KeyError == "":
|
v.Note = "the engine has written no key for this node yet"
|
||||||
v.Note = "the engine has written no key for this node yet: pacman reads the controller's mirror for it"
|
|
||||||
case v.Mirror == nil && v.MirrorError == "":
|
|
||||||
v.Note = "the controller's mirror holds no key for this node: nothing to compare"
|
|
||||||
}
|
}
|
||||||
return v
|
return v
|
||||||
}
|
}
|
||||||
|
|||||||
+10
-180
@@ -1,70 +1,21 @@
|
|||||||
package link
|
package link
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
|
||||||
"encoding/base64"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The wire format shared with the control plane, which defines it separately because this binary
|
// What a joining machine keeps of the enrolment's words: the tunnel it found, as it travels, and how long it asks
|
||||||
// requires nothing present and does not import it. A test on each side asserts the field names.
|
// while the seat mesh-enrolment says to ask again (Redeem). The enrolment through the controller's subject on the
|
||||||
const (
|
// CONTROL stream ended with the controller (issue 1126).
|
||||||
Exchange = "mesh"
|
|
||||||
KeyEnrol = "enrol"
|
|
||||||
)
|
|
||||||
|
|
||||||
// QueueFor is the queue this node consumes from — the only one its account may read.
|
// Approach is how a node reaches a mesh it does not yet belong to: the address, the certificate that address must
|
||||||
func QueueFor(node string) string { return "node." + node }
|
// present, and which bus is at the other end, the mesh's own (hearing.go); a token naming any other is refused before
|
||||||
|
// anything is sent.
|
||||||
// EnrolRequest is what this node says when joining.
|
type Approach struct {
|
||||||
type EnrolRequest struct {
|
Address string
|
||||||
Node string `json:"node"`
|
Fingerprint string
|
||||||
Secret string `json:"secret"`
|
Transport string
|
||||||
PublicKey []byte `json:"public_key"`
|
|
||||||
|
|
||||||
// OverlayKey is the public half of this node's key on the private network — a different key
|
|
||||||
// from PublicKey above, generated at the same moment and for a different purpose.
|
|
||||||
//
|
|
||||||
// Sent with enrolment because the overlay is the first declaration a node receives, and the
|
|
||||||
// mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled
|
|
||||||
// and unreachable for a round trip, which is the state everything else here works to avoid.
|
|
||||||
OverlayKey string `json:"overlay_key,omitempty"`
|
|
||||||
|
|
||||||
// SealingKey is the public half of the key secrets are sealed to. A third key, and the
|
|
||||||
// reasoning is the same one twice over: the mesh must be able to send this node something
|
|
||||||
// nothing else can read, and it must never be able to read it either.
|
|
||||||
SealingKey string `json:"sealing_key,omitempty"`
|
|
||||||
|
|
||||||
// ServingKey is the public half of the key this node serves TLS with on its internal name.
|
|
||||||
// The mesh signs a certificate binding it; the private half never leaves the machine, so
|
|
||||||
// there is nothing to seal and nothing that could be stolen from the mesh's copy.
|
|
||||||
ServingKey string `json:"serving_key,omitempty"`
|
|
||||||
|
|
||||||
Profile map[string]any `json:"profile,omitempty"`
|
|
||||||
|
|
||||||
// Proof is this node's identity key signing EnrolProof over this request: that the presenter
|
|
||||||
// holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish
|
|
||||||
// on a token this key already spent (novox/hq issue 083).
|
|
||||||
Proof []byte `json:"proof,omitempty"`
|
|
||||||
|
|
||||||
// ReplyTo is where the mesh's answer goes, as a field of the request rather than the
|
|
||||||
// transport's own reply address (design 25 §2). Written by the transport that needs it —
|
|
||||||
// withReplyTo, once per attempt — because a request going into a stream has had the transport's
|
|
||||||
// reply field claimed for the consumer's ack address before the controller ever reads it.
|
|
||||||
//
|
|
||||||
// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
|
|
||||||
// field means what it has always meant. Named here so both sides of the wire hold the same
|
|
||||||
// field name, which is what the shape test on each side is for.
|
|
||||||
ReplyTo string `json:"reply_to,omitempty"`
|
|
||||||
|
|
||||||
// Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR
|
|
||||||
// 0105): everything about it but that key. Sent with the keys because it is one of them —
|
|
||||||
// OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the
|
|
||||||
// hub's address, the range and the carried peers from it before the first declaration.
|
|
||||||
Tunnel *Tunnel `json:"tunnel,omitempty"`
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tunnel is a found tunnel as it travels: no private key.
|
// Tunnel is a found tunnel as it travels: no private key.
|
||||||
@@ -86,36 +37,6 @@ type TunnelPeer struct {
|
|||||||
Address string `json:"address"`
|
Address string `json:"address"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// EnrolReply is what the mesh says back.
|
|
||||||
type EnrolReply struct {
|
|
||||||
Accepted bool `json:"accepted"`
|
|
||||||
|
|
||||||
// TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the
|
|
||||||
// token is held for a moment by another enrolment — and that nothing was spent. The same
|
|
||||||
// request is asked again (novox/hq issue 083).
|
|
||||||
TryAgain bool `json:"try_again,omitempty"`
|
|
||||||
|
|
||||||
Node string `json:"node,omitempty"`
|
|
||||||
Queue string `json:"queue,omitempty"`
|
|
||||||
|
|
||||||
// What this node keeps so it can come back on its own. Without these a restart would need a
|
|
||||||
// person with a new token, which would make disconnection a crisis rather than the ordinary
|
|
||||||
// situation novox/hq ADR 0004 says it is.
|
|
||||||
Password string `json:"password,omitempty"`
|
|
||||||
Broker string `json:"broker,omitempty"`
|
|
||||||
Fingerprint string `json:"fingerprint,omitempty"`
|
|
||||||
Signer []byte `json:"signer,omitempty"`
|
|
||||||
|
|
||||||
Refusal string `json:"refusal,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it
|
|
||||||
// presents, so a proof cannot be moved to another request. The mesh builds the same bytes.
|
|
||||||
func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte {
|
|
||||||
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
|
|
||||||
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrRefused is what a node gets when the mesh will not have it.
|
// ErrRefused is what a node gets when the mesh will not have it.
|
||||||
var ErrRefused = errors.New("the mesh refused this enrolment")
|
var ErrRefused = errors.New("the mesh refused this enrolment")
|
||||||
|
|
||||||
@@ -126,94 +47,3 @@ const EnrolPatience = 2 * time.Minute
|
|||||||
|
|
||||||
// AskAgainAfter is the pause between asks while the mesh says "try again".
|
// AskAgainAfter is the pause between asks while the mesh says "try again".
|
||||||
const AskAgainAfter = 3 * time.Second
|
const AskAgainAfter = 3 * time.Second
|
||||||
|
|
||||||
// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking.
|
|
||||||
var ErrNotNow = errors.New("the mesh could not answer this enrolment")
|
|
||||||
|
|
||||||
// answered decides what one reply means: done, ask again, or stop with an error. Separate from the
|
|
||||||
// broker so it can be held to that by a test.
|
|
||||||
func answered(reply EnrolReply, asking time.Duration) (again bool, err error) {
|
|
||||||
switch {
|
|
||||||
case reply.Accepted:
|
|
||||||
return false, nil
|
|
||||||
case reply.TryAgain && asking < EnrolPatience:
|
|
||||||
return true, nil
|
|
||||||
case reply.TryAgain:
|
|
||||||
// Said with what to do. The mesh holds the token for this attempt's keys for as long as
|
|
||||||
// this node kept asking, so a new attempt — with keys of its own — waits that out first.
|
|
||||||
return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+
|
|
||||||
"enrol again with it; if it is then refused, issue a new one",
|
|
||||||
ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience)
|
|
||||||
default:
|
|
||||||
return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Enrol presents this node's key and its one-time secret, and waits to be told it is known.
|
|
||||||
//
|
|
||||||
// The broker has already authenticated this connection: the account was created when the token
|
|
||||||
// was issued and the secret is its password. So this is not how the node gets in — it is what it
|
|
||||||
// says once it is in, and the secret travels again because the control plane must not have to ask
|
|
||||||
// the broker who connected.
|
|
||||||
func Enrol(ctx context.Context, to Approach, node, secret string, public []byte,
|
|
||||||
overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
|
|
||||||
tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) {
|
|
||||||
|
|
||||||
asking, err := Present(ctx, to, node, secret, timeout)
|
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
defer asking.Close()
|
|
||||||
|
|
||||||
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
|
|
||||||
OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
|
|
||||||
Proof: proof, Tunnel: tunnel}
|
|
||||||
body, err := json.Marshal(request)
|
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Asked, and asked again with the same request while the mesh says "try again": the keys this
|
|
||||||
// node generated are the ones it keeps, so the same request is the same enrolment, and the mesh
|
|
||||||
// holds the token for it (novox/hq issue 083).
|
|
||||||
began := time.Now()
|
|
||||||
for {
|
|
||||||
answer, err := asking.Ask(ctx, body, timeout)
|
|
||||||
if err != nil {
|
|
||||||
return EnrolReply{}, err
|
|
||||||
}
|
|
||||||
var reply EnrolReply
|
|
||||||
if err := json.Unmarshal(answer, &reply); err != nil {
|
|
||||||
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
|
|
||||||
}
|
|
||||||
again, err := answered(reply, time.Since(began))
|
|
||||||
if err != nil {
|
|
||||||
return reply, err
|
|
||||||
}
|
|
||||||
if !again {
|
|
||||||
return reply, nil
|
|
||||||
}
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return EnrolReply{}, ctx.Err()
|
|
||||||
case <-time.After(AskAgainAfter):
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// withReplyTo writes this attempt's reply address into the request, as a field of its own.
|
|
||||||
//
|
|
||||||
// **Written into the bytes rather than carried beside them**, because the whole point is that the
|
|
||||||
// address survives a stream: a JetStream consumer's delivery has had the transport's reply field
|
|
||||||
// claimed for its own ack address, so a reply address that is not in the payload is one the
|
|
||||||
// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the
|
|
||||||
// field before marshalling, so one request can be asked again with a fresh address each time without
|
|
||||||
// the caller knowing that is what happens.
|
|
||||||
func withReplyTo(request []byte, inbox string) ([]byte, error) {
|
|
||||||
var fields map[string]any
|
|
||||||
if err := json.Unmarshal(request, &fields); err != nil {
|
|
||||||
return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err)
|
|
||||||
}
|
|
||||||
fields["reply_to"] = inbox
|
|
||||||
return json.Marshal(fields)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -1,34 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What one reply means to a node enrolling (novox/hq issue 083): accepted is done; "try again" is
|
|
||||||
// asked again, until the node's patience runs out and it says the token was not spent; anything
|
|
||||||
// else is a refusal.
|
|
||||||
func TestAnEnrollingNodeAsksAgainWhileTheMeshSaysNotNow(t *testing.T) {
|
|
||||||
if again, err := answered(EnrolReply{Accepted: true}, 0); again || err != nil {
|
|
||||||
t.Fatalf("an accepted enrolment was not done: again=%v err=%v", again, err)
|
|
||||||
}
|
|
||||||
if again, err := answered(EnrolReply{TryAgain: true}, time.Second); !again || err != nil {
|
|
||||||
t.Fatalf("a mesh saying not now was not asked again: again=%v err=%v", again, err)
|
|
||||||
}
|
|
||||||
again, err := answered(EnrolReply{TryAgain: true, Refusal: "restarting"}, EnrolPatience)
|
|
||||||
if again || !errors.Is(err, ErrNotNow) {
|
|
||||||
t.Fatalf("a mesh saying not now past the node's patience did not stop it: again=%v err=%v", again, err)
|
|
||||||
}
|
|
||||||
if !errorsMention(err, "not spent") {
|
|
||||||
t.Errorf("giving up did not say the token can be used again: %v", err)
|
|
||||||
}
|
|
||||||
if again, err := answered(EnrolReply{Refusal: "that token cannot be used"}, 0); again || !errors.Is(err, ErrRefused) {
|
|
||||||
t.Fatalf("a refusal was not a refusal: again=%v err=%v", again, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func errorsMention(err error, what string) bool {
|
|
||||||
return err != nil && strings.Contains(err.Error(), what)
|
|
||||||
}
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
// The bytes a node signs when it enrols. The mesh builds the same bytes to check the signature, in
|
|
||||||
// another repository; this known answer is repeated in its test, so the two cannot drift apart
|
|
||||||
// without one of them failing (novox/hq issue 083).
|
|
||||||
func TestWhatAnEnrollingNodeSignsIsFixed(t *testing.T) {
|
|
||||||
got := string(EnrolProof("s", []byte{1, 2, 3}, "o", "e", "v"))
|
|
||||||
if want := "novox-mesh-enrol\x00s\x00AQID\x00o\x00e\x00v"; got != want {
|
|
||||||
t.Fatalf("the enrolment proof's message changed: %q, want %q", got, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -18,8 +18,8 @@ import (
|
|||||||
// Joining a mesh through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): the node-engine dials the bus as enrol.<node>
|
// Joining a mesh through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): the node-engine dials the bus as enrol.<node>
|
||||||
// with the certificate's fingerprint pinned, calls `redeem` with the public halves of the keys it made where it runs and a
|
// with the certificate's fingerprint pinned, calls `redeem` with the public halves of the keys it made where it runs and a
|
||||||
// proof signed with its identity key, refuses an answer not signed by the key its token carries, keeps its membership, dials
|
// proof signed with its identity key, refuses an answer not signed by the key its token carries, keeps its membership, dials
|
||||||
// again as the login node-logins names for it, and calls `confirm`. The controller's enrolment subject (Enrol) stays for as long
|
// again as the login node-logins names for it, and calls `confirm`. It is the one way in: the controller's enrolment subject
|
||||||
// as the controller holds its lease.
|
// ended with the controller (issue 1126).
|
||||||
|
|
||||||
// EnrolmentSeat is the seat that adopts a new machine, and its two verbs a node-engine calls.
|
// EnrolmentSeat is the seat that adopts a new machine, and its two verbs a node-engine calls.
|
||||||
const (
|
const (
|
||||||
@@ -33,8 +33,8 @@ func CalledSeatSubject(seat, verb, user string) string {
|
|||||||
return "mesh.seat." + seat + ".call." + verb + "." + strings.ReplaceAll(user, ".", "~")
|
return "mesh.seat." + seat + ".call." + verb + "." + strings.ReplaceAll(user, ".", "~")
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call (the
|
// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call, or
|
||||||
// controller still holds its lease) or no holder answers it.
|
// no holder answers it.
|
||||||
var ErrNoSeat = errors.New("the seat mesh-enrolment is not served to this machine's token")
|
var ErrNoSeat = errors.New("the seat mesh-enrolment is not served to this machine's token")
|
||||||
|
|
||||||
// VerifyRedeemAnswer holds an answer to what it must be before anything is kept (ADR 0587 §6): it names the node asked for,
|
// VerifyRedeemAnswer holds an answer to what it must be before anything is kept (ADR 0587 §6): it names the node asked for,
|
||||||
@@ -80,8 +80,8 @@ func seatAnswer(reply []byte) (json.RawMessage, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Redeem calls the seat's redeem as enrol.<node>, with the token's secret as the bus password, on the pinned certificate. It
|
// Redeem calls the seat's redeem as enrol.<node>, with the token's secret as the bus password, on the pinned certificate. It
|
||||||
// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), for
|
// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), , for
|
||||||
// as long as a machine asking the controller would (EnrolPatience). An answer is returned as the holder gave it: the caller
|
// EnrolPatience. An answer is returned as the holder gave it: the caller
|
||||||
// verifies it (VerifyRedeemAnswer) before it keeps anything.
|
// verifies it (VerifyRedeemAnswer) before it keeps anything.
|
||||||
func Redeem(ctx context.Context, to Approach, node, secret string, req delivery.Redeem, timeout time.Duration) (delivery.RedeemAnswer, error) {
|
func Redeem(ctx context.Context, to Approach, node, secret string, req delivery.Redeem, timeout time.Duration) (delivery.RedeemAnswer, error) {
|
||||||
if to.Transport != OnNATS {
|
if to.Transport != OnNATS {
|
||||||
|
|||||||
@@ -1,67 +0,0 @@
|
|||||||
package link
|
|
||||||
|
|
||||||
import (
|
|
||||||
"encoding/json"
|
|
||||||
"os"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/identity"
|
|
||||||
)
|
|
||||||
|
|
||||||
// What this node says when it joins, written out so the mesh's own suite can accept it.
|
|
||||||
//
|
|
||||||
// The two ends are separate structs in separate repositories. Every field here is one somebody
|
|
||||||
// could rename on one side, and the failure would be silent: enrolment succeeds, a key is simply
|
|
||||||
// absent, and the node looks joined until the first thing sealed to it cannot be opened. That is
|
|
||||||
// exactly the shape of fault this project keeps finding late.
|
|
||||||
//
|
|
||||||
// Skipped unless MESH_ENROL_OUT names a file, so this is a check somebody runs deliberately
|
|
||||||
// rather than a dependency between two repositories.
|
|
||||||
func TestWhatThisNodeSaysWhenItJoins(t *testing.T) {
|
|
||||||
path := os.Getenv("MESH_ENROL_OUT")
|
|
||||||
if path == "" {
|
|
||||||
t.Skip("set MESH_ENROL_OUT to write the enrolment request the mesh's suite reads")
|
|
||||||
}
|
|
||||||
|
|
||||||
// A real one. Generated the way enrolment generates them rather than typed as literals, so a
|
|
||||||
// key that stopped being a key would be caught here rather than travelling.
|
|
||||||
mine, err := identity.Generate("workstation")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
overlay, err := identity.GenerateOverlayKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
sealing, err := identity.GenerateSealingKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
serving, err := identity.GenerateServingKey()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
request := EnrolRequest{
|
|
||||||
Node: "workstation",
|
|
||||||
Secret: "a-one-time-secret",
|
|
||||||
PublicKey: mine.Public,
|
|
||||||
OverlayKey: overlay.Public,
|
|
||||||
SealingKey: sealing.Public,
|
|
||||||
ServingKey: serving.Public,
|
|
||||||
Profile: map[string]any{"seat": true},
|
|
||||||
}
|
|
||||||
body, err := json.MarshalIndent(request, "", " ")
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(path, append(body, '\n'), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
// The private half of the sealing key goes beside it, so the mesh's suite can prove what it
|
|
||||||
// sealed is openable rather than merely present.
|
|
||||||
if err := os.WriteFile(path+".sealing-private", []byte(sealing.Private), 0o600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
t.Logf("wrote %s", path)
|
|
||||||
}
|
|
||||||
@@ -76,13 +76,9 @@ const MaxHandOverBy = 200
|
|||||||
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
|
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
|
||||||
type HandOverCheck struct {
|
type HandOverCheck struct {
|
||||||
Signer ed25519.PublicKey
|
Signer ed25519.PublicKey
|
||||||
// SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer:
|
Node string
|
||||||
// nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7,
|
Now func() time.Time
|
||||||
// ADR 0550 §2).
|
Nonces *Nonces
|
||||||
SignerNow func() ed25519.PublicKey
|
|
||||||
Node string
|
|
||||||
Now func() time.Time
|
|
||||||
Nonces *Nonces
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory —
|
// Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory —
|
||||||
@@ -162,11 +158,6 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod
|
|||||||
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
|
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
|
||||||
}
|
}
|
||||||
signer := check.Signer
|
signer := check.Signer
|
||||||
if check.SignerNow != nil {
|
|
||||||
if signer = check.SignerNow(); signer == nil {
|
|
||||||
return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(signer) != ed25519.PublicKeySize ||
|
if len(signer) != ed25519.PublicKeySize ||
|
||||||
!ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
!ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) {
|
||||||
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
|
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
|
||||||
|
|||||||
@@ -148,20 +148,14 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing.
|
|||||||
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
|
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
|
||||||
t.Fatalf("the handler was given %v", given)
|
t.Fatalf("the handler was given %v", given)
|
||||||
}
|
}
|
||||||
// ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key
|
// Issue 1126: a link opened on the mesh has no signer (the controller that signed these asks ended), so an ask signed
|
||||||
// is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over.
|
// by the membership's key, the controller's, is refused and nothing is handed over.
|
||||||
trusted := m.Signer
|
|
||||||
following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node,
|
|
||||||
Now: func() time.Time { return now }, Nonces: &Nonces{}}
|
|
||||||
before := len(given)
|
before := len(given)
|
||||||
trusted = nil
|
opened := (&natsLink{node: m.Node}).check()
|
||||||
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) }))
|
opened.Now, opened.Nonces = func() time.Time { return now }, &Nonces{}
|
||||||
if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before {
|
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Nonce = strings.Repeat("cd", 16) }))
|
||||||
t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a)
|
if a := AnswerHandOver(opened, handle, late); !strings.Contains(a.Refused, "not signed by the mesh") || len(given) != before {
|
||||||
}
|
t.Fatalf("a link opened on the mesh answered an ask signed by the controller's key: %+v", a)
|
||||||
trusted = m.Signer
|
|
||||||
if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 {
|
|
||||||
t.Fatalf("while the controller holds the lease its ask was answered %+v", a)
|
|
||||||
}
|
}
|
||||||
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
|
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
|
||||||
now = now.Add(HandOverGood + time.Second)
|
now = now.Add(HandOverGood + time.Second)
|
||||||
@@ -192,23 +186,17 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) {
|
|||||||
// handingLink is a quiet link that hears hand-overs, keeping the handler it was given.
|
// handingLink is a quiet link that hears hand-overs, keeping the handler it was given.
|
||||||
type handingLink struct {
|
type handingLink struct {
|
||||||
*quietLink
|
*quietLink
|
||||||
given chan HandOverFunc
|
given chan HandOverFunc
|
||||||
signer chan func() ed25519.PublicKey
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
|
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
|
||||||
|
|
||||||
// FollowSigner keeps what the link is told to verify the controller's asks with.
|
|
||||||
func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f }
|
|
||||||
|
|
||||||
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
|
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
|
||||||
// judgement, and nothing else.
|
// judgement, and nothing else.
|
||||||
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
||||||
m, _ := aMember(t)
|
m, _ := aMember(t)
|
||||||
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)}
|
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)}
|
||||||
q := &Session{Membership: m, Timeout: time.Second}
|
q := &Session{Membership: m, Timeout: time.Second}
|
||||||
// The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7).
|
|
||||||
q.ControllerKey = func() ed25519.PublicKey { return nil }
|
|
||||||
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
|
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
|
||||||
ctx, stop := context.WithCancel(context.Background())
|
ctx, stop := context.WithCancel(context.Background())
|
||||||
served := make(chan error, 1)
|
served := make(chan error, 1)
|
||||||
@@ -221,14 +209,6 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
|||||||
case <-time.After(5 * time.Second):
|
case <-time.After(5 * time.Second):
|
||||||
t.Fatal("the link was never given the queue's hand-over")
|
t.Fatal("the link was never given the queue's hand-over")
|
||||||
}
|
}
|
||||||
select {
|
|
||||||
case f := <-l.signer:
|
|
||||||
if f() != nil {
|
|
||||||
t.Fatal("the link was given another key than the queue's")
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("the link was never given the key the engine trusts now")
|
|
||||||
}
|
|
||||||
stop()
|
stop()
|
||||||
<-served
|
<-served
|
||||||
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
|
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
|
||||||
@@ -241,15 +221,13 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
|||||||
return map[string]any{"module": module, "digest": digest, "diff": diff}, nil
|
return map[string]any{"module": module, "digest": digest, "diff": diff}, nil
|
||||||
},
|
},
|
||||||
Compose: func(module string) (any, error) { return map[string]any{"composed": true, "retried": module}, nil },
|
Compose: func(module string) (any, error) { return map[string]any{"composed": true, "retried": module}, nil },
|
||||||
// ADR 0565 §6, §15: the node's key beside the controller's mirror's, the mirror's from its writer's test.
|
// ADR 0565 §6, §15: the node's key of the engine's state packages.
|
||||||
Packages: func(write bool) (any, error) {
|
Packages: func(write bool) (any, error) {
|
||||||
key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`)
|
key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`)
|
||||||
mirror := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false},` +
|
|
||||||
`{"package":"nano","modules":["web"],"absent":true}]}`)
|
|
||||||
if !write {
|
if !write {
|
||||||
return PackagesAnswer("n1", key, mirror, nil, nil), nil
|
return PackagesAnswer("n1", key, nil), nil
|
||||||
}
|
}
|
||||||
v := PackagesAnswer("n1", nil, nil, nil, errors.New("not granted"))
|
v := PackagesAnswer("n1", nil, nil)
|
||||||
wrote := true
|
wrote := true
|
||||||
v.Wrote = &wrote
|
v.Wrote = &wrote
|
||||||
return v, nil
|
return v, nil
|
||||||
@@ -265,11 +243,10 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
|
|||||||
// The recovery verb compose (issue 774) makes a pass now, with a module's apply tried again when named.
|
// The recovery verb compose (issue 774) makes a pass now, with a module's apply tried again when named.
|
||||||
{"compose", EngineToolSubject("compose", "n1"), `"composed":true,"retried":""`},
|
{"compose", EngineToolSubject("compose", "n1"), `"composed":true,"retried":""`},
|
||||||
{"compose", EngineToolSubject("compose", "n1") + " " + `{"module":"route-proxy"}`, `"retried":"route-proxy"`},
|
{"compose", EngineToolSubject("compose", "n1") + " " + `{"module":"route-proxy"}`, `"retried":"route-proxy"`},
|
||||||
// The read verb packages (ADR 0565 §6) names both sources and their difference; with write, what it wrote, and a
|
// The read verb packages (ADR 0565 §6) names its source; with write, what it wrote, and a key the engine has not
|
||||||
// key the engine has not written, or a mirror it may not read, is said and not an error.
|
// written is said and not an error.
|
||||||
{"packages", EngineToolSubject("packages", "n1"), `"differences":[{"package":"nano","absent":true,"engine":[],"mirror":["web"]}],"agree":false`},
|
|
||||||
{"packages", EngineToolSubject("packages", "n1"), `"source":"mesh-host.packages","key":{"node":"n1"`},
|
{"packages", EngineToolSubject("packages", "n1"), `"source":"mesh-host.packages","key":{"node":"n1"`},
|
||||||
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"mirror":null,"mirror_error":"not granted","differences":[],"agree":false,"wrote":true`},
|
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"key":null,"wrote":true`},
|
||||||
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`},
|
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`},
|
||||||
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
|
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
|
||||||
} {
|
} {
|
||||||
|
|||||||
@@ -756,9 +756,8 @@ func (q *Session) NoteCandidates(ctx context.Context, answers map[string]deliver
|
|||||||
q.writeHaves(ctx, bus)
|
q.writeHaves(ctx, bus)
|
||||||
}
|
}
|
||||||
|
|
||||||
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn or
|
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn) and why: written
|
||||||
// delivery.ComposesController) and why: written at each start of its agent, so a node returning to the controller's
|
// at each start of its agent, where every have is read (issue 1126).
|
||||||
// compositions says so where every have is read (issue 1126).
|
|
||||||
func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
|
func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
|
||||||
q.init()
|
q.init()
|
||||||
s := &q.haves
|
s := &q.haves
|
||||||
|
|||||||
@@ -24,7 +24,7 @@ type haveLink struct {
|
|||||||
hmu sync.Mutex
|
hmu sync.Mutex
|
||||||
kv map[string]haveEntry
|
kv map[string]haveEntry
|
||||||
sets []string // keys written, in order
|
sets []string // keys written, in order
|
||||||
events []string // "report" and "have <key>", in the order they happened
|
events []string // "have <key>", in the order they happened
|
||||||
failing bool
|
failing bool
|
||||||
binds int
|
binds int
|
||||||
setTries int
|
setTries int
|
||||||
@@ -40,16 +40,6 @@ type haveEntry struct {
|
|||||||
|
|
||||||
func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }
|
func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }
|
||||||
|
|
||||||
func (l *haveLink) Report(ctx context.Context, node string, body []byte) error {
|
|
||||||
err := l.quietLink.Report(ctx, node, body)
|
|
||||||
if err == nil {
|
|
||||||
l.hmu.Lock()
|
|
||||||
l.events = append(l.events, "report")
|
|
||||||
l.hmu.Unlock()
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *haveLink) Health(context.Context, string, []byte) error { return nil }
|
func (l *haveLink) Health(context.Context, string, []byte) error { return nil }
|
||||||
|
|
||||||
func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
|
func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
|
||||||
@@ -241,9 +231,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters
|
// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile.
|
||||||
// and profile.
|
func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
||||||
func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
|
||||||
l := newHaveLink()
|
l := newHaveLink()
|
||||||
q, _ := aHaveQueue(l)
|
q, _ := aHaveQueue(l)
|
||||||
rep := appliedModules()
|
rep := appliedModules()
|
||||||
@@ -253,13 +242,6 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
|||||||
rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
|
rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
|
||||||
rep.Profile = map[string]any{"cores": 8}
|
rep.Profile = map[string]any{"cores": 8}
|
||||||
q.SayFacts(context.Background(), rep)
|
q.SayFacts(context.Background(), rep)
|
||||||
if len(l.events) < 2 || l.events[0] != "report" {
|
|
||||||
t.Fatalf("the report was not first: %v", l.events)
|
|
||||||
}
|
|
||||||
raw, _ := json.Marshal(l.said()[0])
|
|
||||||
if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") {
|
|
||||||
t.Errorf("the report carries the module facts: %s", raw)
|
|
||||||
}
|
|
||||||
got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value)
|
got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -310,16 +292,13 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
|
// 1132/AC7: a write the bucket refuses is said in the log, and the next apply writes again.
|
||||||
func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
|
func TestAFailedHaveWriteIsSaidAndWrittenAgain(t *testing.T) {
|
||||||
l := newHaveLink()
|
l := newHaveLink()
|
||||||
l.failing = true
|
l.failing = true
|
||||||
q, said := aHaveQueue(l)
|
q, said := aHaveQueue(l)
|
||||||
if !q.SayFacts(context.Background(), appliedModules()) {
|
if !q.SayFacts(context.Background(), appliedModules()) {
|
||||||
t.Fatal("the report did not go out")
|
t.Fatal("the facts were not said with a link open")
|
||||||
}
|
|
||||||
if len(l.said()) != 1 {
|
|
||||||
t.Fatalf("reports sent: %d", len(l.said()))
|
|
||||||
}
|
}
|
||||||
if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
|
if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
|
||||||
t.Errorf("the failure is not in the log: %v", *said)
|
t.Errorf("the failure is not in the log: %v", *said)
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"regexp"
|
"regexp"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
@@ -18,11 +17,6 @@ import (
|
|||||||
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
|
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
|
||||||
// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts.
|
// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts.
|
||||||
|
|
||||||
// EnrolSubject is where a joining machine asks. One subject for every node, because a machine
|
|
||||||
// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is
|
|
||||||
// the whole of what its enrolment user may do.
|
|
||||||
const EnrolSubject = "mesh.control.enrol"
|
|
||||||
|
|
||||||
// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream.
|
// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream.
|
||||||
// The answer is the facts event on its report subject — nobody's inbox is answered.
|
// The answer is the facts event on its report subject — nobody's inbox is answered.
|
||||||
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
|
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
|
||||||
@@ -39,14 +33,11 @@ func natsURL(address string) string {
|
|||||||
|
|
||||||
// natsLink is this node's connection as a JetStream subscription.
|
// natsLink is this node's connection as a JetStream subscription.
|
||||||
type natsLink struct {
|
type natsLink struct {
|
||||||
// mu guards signerNow (FollowSigner).
|
conn *nats.Conn
|
||||||
mu sync.Mutex
|
js nats.JetStreamContext
|
||||||
signerNow func() ed25519.PublicKey
|
asking *nats.Subscription
|
||||||
conn *nats.Conn
|
asked chan struct{}
|
||||||
js nats.JetStreamContext
|
handing *nats.Subscription
|
||||||
asking *nats.Subscription
|
|
||||||
asked chan struct{}
|
|
||||||
handing *nats.Subscription
|
|
||||||
// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
|
// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
|
||||||
searching *nats.Subscription
|
searching *nats.Subscription
|
||||||
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
|
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
|
||||||
@@ -117,7 +108,9 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
l := &natsLink{
|
l := &natsLink{
|
||||||
conn: conn, js: js, node: m.Node, user: user, signer: m.Signer,
|
// No signer: the asks these subjects carry were signed by the controller, which ended by its unassignment
|
||||||
|
// (issue 1126), so a link opened on the mesh believes none of them (ADR 0563 §7, ADR 0550 §2).
|
||||||
|
conn: conn, js: js, node: m.Node, user: user,
|
||||||
lost: make(chan error, 1),
|
lost: make(chan error, 1),
|
||||||
asked: make(chan struct{}, 1),
|
asked: make(chan struct{}, 1),
|
||||||
refused: make(chan string, 64),
|
refused: make(chan string, 64),
|
||||||
@@ -163,19 +156,9 @@ func (l *natsLink) hearAsks() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the
|
// check is how a signed ask is believed on this link: by the link's signer, which a link opened on the mesh has none of.
|
||||||
// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2).
|
|
||||||
func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) {
|
|
||||||
l.mu.Lock()
|
|
||||||
l.signerNow = f
|
|
||||||
l.mu.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust.
|
|
||||||
func (l *natsLink) check() HandOverCheck {
|
func (l *natsLink) check() HandOverCheck {
|
||||||
l.mu.Lock()
|
return HandOverCheck{Signer: l.signer, Node: l.node}
|
||||||
defer l.mu.Unlock()
|
|
||||||
return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
|
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
|
||||||
@@ -323,10 +306,6 @@ func (l *natsLink) Close() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *natsLink) Report(ctx context.Context, node string, body []byte) error {
|
|
||||||
return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *natsLink) Health(ctx context.Context, node string, body []byte) error {
|
func (l *natsLink) Health(ctx context.Context, node string, body []byte) error {
|
||||||
return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body)
|
return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,6 @@
|
|||||||
package link
|
package link
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
"git.novox.be/novox/mesh-sdk/go/delivery"
|
||||||
@@ -47,13 +45,6 @@ type Report struct {
|
|||||||
// machine saying what is true of it, not asking for anything.
|
// machine saying what is true of it, not asking for anything.
|
||||||
Carried []int `json:"carried,omitempty"`
|
Carried []int `json:"carried,omitempty"`
|
||||||
|
|
||||||
// Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand
|
|
||||||
// among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine,
|
|
||||||
// the sequence one higher for every facts event, so the controller refuses older facts by this order. On the
|
|
||||||
// wire as three top-level keys.
|
|
||||||
Writer string `json:"writer,omitempty"`
|
|
||||||
Order
|
|
||||||
|
|
||||||
// Held is what this adopted node found and is keeping as it was until its module is taken
|
// Held is what this adopted node found and is keeping as it was until its module is taken
|
||||||
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
|
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
|
||||||
Held []Held `json:"held,omitempty"`
|
Held []Held `json:"held,omitempty"`
|
||||||
@@ -142,10 +133,6 @@ type Report struct {
|
|||||||
// that does not load is a machine filtering nothing while its unit reports success.
|
// that does not load is a machine filtering nothing while its unit reports success.
|
||||||
Outward []string `json:"outward,omitempty"`
|
Outward []string `json:"outward,omitempty"`
|
||||||
|
|
||||||
// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
|
|
||||||
// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
|
|
||||||
Rekey *Rekey `json:"rekey,omitempty"`
|
|
||||||
|
|
||||||
// Rollbacks is what this machine's witnesses decided about a core build that was not healthy
|
// Rollbacks is what this machine's witnesses decided about a core build that was not healthy
|
||||||
// in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the
|
// in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the
|
||||||
// engine, the engine about the controller and the node tools. **Said on every report while it
|
// engine, the engine about the controller and the node tools. **Said on every report while it
|
||||||
@@ -488,9 +475,8 @@ const WitnessContract = 1
|
|||||||
|
|
||||||
// The core components a witness judges (to-be 45 §8), as a rollback names them.
|
// The core components a witness judges (to-be 45 §8), as a rollback names them.
|
||||||
const (
|
const (
|
||||||
ComponentEngine = "node-engine"
|
ComponentEngine = "node-engine"
|
||||||
ComponentController = "controller"
|
ComponentNodeTools = "node-tools"
|
||||||
ComponentNodeTools = "node-tools"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// What a witness concluded, as a rollback says it.
|
// What a witness concluded, as a rollback says it.
|
||||||
@@ -506,7 +492,7 @@ const (
|
|||||||
// RestoreFailed is a restoration that was attempted and did not complete.
|
// RestoreFailed is a restoration that was attempted and did not complete.
|
||||||
RestoreFailed = "restore-failed"
|
RestoreFailed = "restore-failed"
|
||||||
// Unwitnessed is a build whose health could not be judged at all within the bound — the
|
// Unwitnessed is a build whose health could not be judged at all within the bound — the
|
||||||
// witness could not ask (no grant, no lease bucket) — so it is neither proved nor rolled back.
|
// witness could not ask (no grant, the bus away) — so it is neither proved nor rolled back.
|
||||||
Unwitnessed = "unwitnessed"
|
Unwitnessed = "unwitnessed"
|
||||||
// Halted is the node-engine's launcher giving up: the build it would go back to fails too, so
|
// Halted is the node-engine's launcher giving up: the build it would go back to fails too, so
|
||||||
// the fault is the machine's and not a build's.
|
// the fault is the machine's and not a build's.
|
||||||
@@ -531,26 +517,6 @@ type Rollback struct {
|
|||||||
At time.Time `json:"at"`
|
At time.Time `json:"at"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its
|
|
||||||
// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one
|
|
||||||
// with nothing kept on disk.
|
|
||||||
type Order struct {
|
|
||||||
Epoch int64 `json:"epoch,omitempty"`
|
|
||||||
Sequence int64 `json:"sequence,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a
|
|
||||||
// Stringer promoted onto every report would print each one as its order alone.
|
|
||||||
func (o Order) Words() string {
|
|
||||||
switch {
|
|
||||||
case o.Epoch > 0:
|
|
||||||
return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10)
|
|
||||||
case o.Sequence > 0:
|
|
||||||
return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch"
|
|
||||||
}
|
|
||||||
return "no order"
|
|
||||||
}
|
|
||||||
|
|
||||||
// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried
|
// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried
|
||||||
// states below; Note is what the host did about it, when it did something.
|
// states below; Note is what the host did about it, when it did something.
|
||||||
type CarriedTunnel struct {
|
type CarriedTunnel struct {
|
||||||
@@ -573,36 +539,6 @@ const (
|
|||||||
CarriedDown = "down"
|
CarriedDown = "down"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling
|
|
||||||
// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel
|
|
||||||
// over, since re-enrolling would rotate every key it holds. Signed with the identity key over
|
|
||||||
// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key.
|
|
||||||
type Rekey struct {
|
|
||||||
// Previous is the overlay key this node held until now, as the mesh records it; the mesh
|
|
||||||
// refuses a rekey naming another, which is how a replayed one is refused.
|
|
||||||
Previous string `json:"previous"`
|
|
||||||
OverlayKey string `json:"overlay_key"`
|
|
||||||
Tunnel *Tunnel `json:"tunnel"`
|
|
||||||
Proof []byte `json:"proof"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes
|
|
||||||
// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel.
|
|
||||||
// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof).
|
|
||||||
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
|
|
||||||
var t Tunnel
|
|
||||||
if tunnel != nil {
|
|
||||||
t = *tunnel
|
|
||||||
}
|
|
||||||
peers := make([]string, 0, len(t.Peers))
|
|
||||||
for _, p := range t.Peers {
|
|
||||||
peers = append(peers, p.PublicKey+"@"+p.Address)
|
|
||||||
}
|
|
||||||
return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
|
|
||||||
t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
|
|
||||||
t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
|
|
||||||
}
|
|
||||||
|
|
||||||
// Held is one file or container found on an adopted node and kept as it was.
|
// Held is one file or container found on an adopted node and kept as it was.
|
||||||
type Held struct {
|
type Held struct {
|
||||||
ID string `json:"id"`
|
ID string `json:"id"`
|
||||||
|
|||||||
@@ -12,9 +12,6 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
|||||||
value any
|
value any
|
||||||
expect []string
|
expect []string
|
||||||
}{
|
}{
|
||||||
// novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence.
|
|
||||||
{Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}},
|
|
||||||
[]string{"node", "writer", "epoch", "sequence"}},
|
|
||||||
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
|
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
|
||||||
[]string{"node", "applied", "failed", "refused"}},
|
[]string{"node", "applied", "failed", "refused"}},
|
||||||
// novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what
|
// novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what
|
||||||
@@ -27,7 +24,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
|
|||||||
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
|
||||||
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
|
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
|
||||||
// witness contract this host keeps.
|
// witness contract this host keeps.
|
||||||
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract},
|
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract},
|
||||||
[]string{"node", "rollbacks", "witness"}},
|
[]string{"node", "rollbacks", "witness"}},
|
||||||
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
|
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
|
||||||
[]string{"component", "from", "to", "outcome", "why", "at"}},
|
[]string{"component", "from", "to", "outcome", "why", "at"}},
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package link
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"time"
|
"time"
|
||||||
@@ -199,12 +198,6 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
|||||||
}
|
}
|
||||||
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
|
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
|
||||||
// 356), answered on the link the question came over.
|
// 356), answered on the link the question came over.
|
||||||
// Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7).
|
|
||||||
if f, ok := link.(interface {
|
|
||||||
FollowSigner(func() ed25519.PublicKey)
|
|
||||||
}); ok && queue.ControllerKey != nil {
|
|
||||||
f.FollowSigner(queue.ControllerKey)
|
|
||||||
}
|
|
||||||
if h, ok := link.(HandOvers); ok {
|
if h, ok := link.(HandOvers); ok {
|
||||||
h.HearHandOvers(queue.HandOver)
|
h.HearHandOvers(queue.HandOver)
|
||||||
}
|
}
|
||||||
@@ -290,41 +283,3 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// publishReport tells the mesh what this node did, and says whether the broker took it.
|
|
||||||
// Publish sends one report on this node's own connection and returns: the one-shot path for a
|
|
||||||
// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same
|
|
||||||
// account, the same pinned certificate and the same exchange as the running host's reports.
|
|
||||||
func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error {
|
|
||||||
link, err := Open(ctx, m, timeout)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer link.Close()
|
|
||||||
var said string
|
|
||||||
if !publishReport(ctx, link, m, report, func(s string) { said = s }, timeout) {
|
|
||||||
return errors.New(said)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func publishReport(ctx context.Context, bus Bus, m Membership, report Report,
|
|
||||||
say Announce, timeout time.Duration) bool {
|
|
||||||
report.Node = m.Node
|
|
||||||
body, err := json.Marshal(report)
|
|
||||||
if err != nil {
|
|
||||||
say("cannot encode this node's own report: " + err.Error())
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
publish, cancel := context.WithTimeout(ctx, timeout)
|
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
|
|
||||||
// node never answered, while the node believes it did — and the two would go on disagreeing
|
|
||||||
// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
|
|
||||||
if err := bus.Report(publish, m.Node, body); err != nil {
|
|
||||||
say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package link
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@@ -14,16 +13,11 @@ import (
|
|||||||
"git.novox.be/novox/mesh-sdk/go/health"
|
"git.novox.be/novox/mesh-sdk/go/health"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program
|
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this
|
||||||
// that wrote them, beside its epoch and sequence.
|
// machine over it, and what the machine says on it unasked: its health and its haves.
|
||||||
const Writer = "node-engine"
|
|
||||||
|
|
||||||
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks
|
|
||||||
// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts.
|
|
||||||
//
|
//
|
||||||
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an
|
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here
|
||||||
// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles
|
// receives, orders or reconciles a whole declaration.
|
||||||
// a whole declaration.
|
|
||||||
type Session struct {
|
type Session struct {
|
||||||
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
|
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
|
||||||
Fallback func() *delivery.LoginFallback
|
Fallback func() *delivery.LoginFallback
|
||||||
@@ -54,9 +48,6 @@ type Session struct {
|
|||||||
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
|
||||||
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
|
||||||
MoveSecret MoveSecretFunc
|
MoveSecret MoveSecretFunc
|
||||||
// ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq
|
|
||||||
// ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's.
|
|
||||||
ControllerKey func() ed25519.PublicKey
|
|
||||||
// Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the
|
// Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the
|
||||||
// operator asked and heard over it. Nil serves no seat.
|
// operator asked and heard over it. Nil serves no seat.
|
||||||
Acts *HandActs
|
Acts *HandActs
|
||||||
@@ -88,12 +79,8 @@ type Session struct {
|
|||||||
haveAgainPending bool
|
haveAgainPending bool
|
||||||
haveAgainTries int
|
haveAgainTries int
|
||||||
|
|
||||||
// saying is held while a facts event is made and said, so they leave in the order they are made.
|
// saying is held while a facts event is noted, so the haves are written in the order the facts are made.
|
||||||
saying sync.Mutex
|
saying sync.Mutex
|
||||||
// epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one
|
|
||||||
// without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2).
|
|
||||||
epoch int64
|
|
||||||
sequence int64
|
|
||||||
|
|
||||||
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
|
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
|
||||||
haves haveState
|
haves haveState
|
||||||
@@ -137,7 +124,6 @@ func (q *Session) init() {
|
|||||||
if q.Say == nil {
|
if q.Say == nil {
|
||||||
q.Say = func(string) {}
|
q.Say = func(string) {}
|
||||||
}
|
}
|
||||||
q.epoch = time.Now().UnixMilli()
|
|
||||||
q.changed = make(chan struct{})
|
q.changed = make(chan struct{})
|
||||||
q.owed = make(chan struct{}, 1)
|
q.owed = make(chan struct{}, 1)
|
||||||
})
|
})
|
||||||
@@ -160,8 +146,10 @@ func (q *Session) oweHealth() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them.
|
// SayFacts says this node's facts in its haves on the link open now, in the order they are made, and whether a link
|
||||||
// A facts event made with no link open is not kept: the next link says the facts as they are then.
|
// was open to say them on. A facts event made with no link open is not kept: the next link says the facts as they are
|
||||||
|
// then. They are said in the haves alone: the CONTROL stream they were also published on was the controller's, which
|
||||||
|
// ended by its unassignment (issue 1126). A write the bucket refuses is said and written again (issue 1132).
|
||||||
func (q *Session) SayFacts(ctx context.Context, r Report) bool {
|
func (q *Session) SayFacts(ctx context.Context, r Report) bool {
|
||||||
q.init()
|
q.init()
|
||||||
q.saying.Lock()
|
q.saying.Lock()
|
||||||
@@ -172,14 +160,7 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool {
|
|||||||
if bus == nil {
|
if bus == nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
q.sequence++
|
|
||||||
r.Node = q.Membership.Node
|
r.Node = q.Membership.Node
|
||||||
r.Writer = Writer
|
|
||||||
r.Order = Order{Epoch: q.epoch, Sequence: q.sequence}
|
|
||||||
if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
// After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132).
|
|
||||||
if q.noteApply(r) {
|
if q.noteApply(r) {
|
||||||
q.writeHaves(ctx, bus)
|
q.writeHaves(ctx, bus)
|
||||||
}
|
}
|
||||||
@@ -309,23 +290,6 @@ func (q *Session) AskTool(ctx context.Context, module, tool string) (health.Answ
|
|||||||
return ReadToolAnswer(reply)
|
return ReadToolAnswer(reply)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over
|
|
||||||
// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for.
|
|
||||||
func (q *Session) AskController(ctx context.Context, body []byte) ([]byte, error) {
|
|
||||||
q.init()
|
|
||||||
q.mu.Lock()
|
|
||||||
bus := q.bus
|
|
||||||
q.mu.Unlock()
|
|
||||||
if bus == nil {
|
|
||||||
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
|
|
||||||
}
|
|
||||||
asker, ok := bus.(ToolBus)
|
|
||||||
if !ok {
|
|
||||||
return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked)
|
|
||||||
}
|
|
||||||
return asker.Ask(ctx, CLISubject(q.Membership.Node), body)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any
|
// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any
|
||||||
// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272).
|
// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272).
|
||||||
var ErrNotAsked = errors.New("not asked")
|
var ErrNotAsked = errors.New("not asked")
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package link
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
"crypto/ed25519"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
@@ -15,42 +14,16 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
)
|
)
|
||||||
|
|
||||||
// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled,
|
// quietLink is a link with no broker behind it.
|
||||||
// as the bus's is ("reporting: context canceled"), or while refusing.
|
|
||||||
type quietLink struct {
|
type quietLink struct {
|
||||||
mu sync.Mutex
|
lost chan error
|
||||||
reports []Report
|
|
||||||
refusing bool
|
|
||||||
lost chan error
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
|
func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
|
||||||
|
|
||||||
func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error {
|
|
||||||
if err := ctx.Err(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
if l.refusing {
|
|
||||||
return errors.New("refused")
|
|
||||||
}
|
|
||||||
var r Report
|
|
||||||
if err := json.Unmarshal(body, &r); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
l.reports = append(l.reports, r)
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
func (l *quietLink) Lost() <-chan error { return l.lost }
|
func (l *quietLink) Lost() <-chan error { return l.lost }
|
||||||
func (l *quietLink) Close() {}
|
func (l *quietLink) Close() {}
|
||||||
|
|
||||||
func (l *quietLink) said() []Report {
|
|
||||||
l.mu.Lock()
|
|
||||||
defer l.mu.Unlock()
|
|
||||||
return append([]Report(nil), l.reports...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
|
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
public, private, err := ed25519.GenerateKey(nil)
|
public, private, err := ed25519.GenerateKey(nil)
|
||||||
@@ -60,40 +33,6 @@ func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
|
|||||||
return Membership{Node: "n1", Signer: public}, private
|
return Membership{Node: "n1", Signer: public}, private
|
||||||
}
|
}
|
||||||
|
|
||||||
// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539
|
|
||||||
// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts.
|
|
||||||
func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) {
|
|
||||||
m, _ := aMember(t)
|
|
||||||
l := newQuietLink()
|
|
||||||
s := &Session{Membership: m, Timeout: time.Second}
|
|
||||||
s.Attach(context.Background(), l)
|
|
||||||
|
|
||||||
if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) {
|
|
||||||
t.Fatal("the facts were not taken")
|
|
||||||
}
|
|
||||||
got := l.said()
|
|
||||||
if len(got) != 2 {
|
|
||||||
t.Fatalf("%d facts events were said", len(got))
|
|
||||||
}
|
|
||||||
for _, r := range got {
|
|
||||||
if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 {
|
|
||||||
t.Fatalf("the facts do not say who wrote them: %+v", r)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 {
|
|
||||||
t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order)
|
|
||||||
}
|
|
||||||
|
|
||||||
// A later start orders after this one with nothing kept on disk.
|
|
||||||
time.Sleep(2 * time.Millisecond)
|
|
||||||
later := &Session{Membership: m, Timeout: time.Second}
|
|
||||||
later.Attach(context.Background(), l)
|
|
||||||
later.SayFacts(context.Background(), Report{})
|
|
||||||
if last := l.said()[2]; last.Epoch <= got[1].Epoch {
|
|
||||||
t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then.
|
// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then.
|
||||||
func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
|
func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
|
||||||
m, _ := aMember(t)
|
m, _ := aMember(t)
|
||||||
@@ -104,7 +43,7 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
|
|||||||
}
|
}
|
||||||
s.Attach(context.Background(), l)
|
s.Attach(context.Background(), l)
|
||||||
s.Detach(l)
|
s.Detach(l)
|
||||||
if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 {
|
if s.Linked() || s.SayFacts(context.Background(), Report{}) {
|
||||||
t.Fatal("facts were said on a link that was let go")
|
t.Fatal("facts were said on a link that was let go")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -140,16 +79,6 @@ func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
|
|
||||||
Retention: nats.WorkQueuePolicy}
|
|
||||||
if _, err := js.StreamInfo(want.Name); err != nil {
|
|
||||||
if _, err := js.AddStream(want); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := js.PurgeStream(want.Name); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return conn, js
|
return conn, js
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+3
-5
@@ -3,16 +3,15 @@
|
|||||||
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
|
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
|
||||||
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
|
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
|
||||||
#
|
#
|
||||||
# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a
|
# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host
|
||||||
# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool
|
# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime.
|
||||||
# runtime, `admin` for the controller writing the lease.
|
|
||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
accounts {
|
accounts {
|
||||||
MESH {
|
MESH {
|
||||||
jetstream: enabled
|
jetstream: enabled
|
||||||
users = [
|
users = [
|
||||||
{ user: "node.anchor", password: "a", permissions: {
|
{ user: "node.anchor", password: "a", permissions: {
|
||||||
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] }
|
publish: { allow: ["$SRV.PING.node-tools.anchor"] }
|
||||||
subscribe: { allow: ["_INBOX.node.anchor.>"] }
|
subscribe: { allow: ["_INBOX.node.anchor.>"] }
|
||||||
} }
|
} }
|
||||||
{ user: "node.bare", password: "b", permissions: {
|
{ user: "node.bare", password: "b", permissions: {
|
||||||
@@ -23,7 +22,6 @@ accounts {
|
|||||||
subscribe: { allow: ["$SRV.PING.node-tools.>"] }
|
subscribe: { allow: ["$SRV.PING.node-tools.>"] }
|
||||||
allow_responses: { max: 1, ttl: "1m" }
|
allow_responses: { max: 1, ttl: "1m" }
|
||||||
} }
|
} }
|
||||||
{ user: "admin", password: "x" }
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,14 +10,11 @@ import (
|
|||||||
"github.com/nats-io/nats.go"
|
"github.com/nats-io/nats.go"
|
||||||
)
|
)
|
||||||
|
|
||||||
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the
|
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this
|
||||||
// node tools it placed on this machine, by what the bus says about them rather than by what they
|
// machine, by what the bus says about them rather than by what they say about themselves.
|
||||||
// say about themselves.
|
|
||||||
//
|
//
|
||||||
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read
|
// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's
|
||||||
// by JetStream's direct get — one subject, no stream information, nothing written; and this
|
// PING by its name and this machine's, so only this machine's runtime can answer it.
|
||||||
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
|
|
||||||
// so only this machine's runtime can answer it.
|
|
||||||
|
|
||||||
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
|
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
|
||||||
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
|
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
|
||||||
@@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered")
|
|||||||
|
|
||||||
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
|
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
|
||||||
type Asker interface {
|
type Asker interface {
|
||||||
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
|
|
||||||
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
|
|
||||||
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
|
|
||||||
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
|
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
|
||||||
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
|
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
|
||||||
Ping(ctx context.Context, service, id string) error
|
Ping(ctx context.Context, service, id string) error
|
||||||
}
|
}
|
||||||
|
|
||||||
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
|
|
||||||
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
|
|
||||||
func DirectGetSubject(bucket, key string) string {
|
|
||||||
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
|
|
||||||
}
|
|
||||||
|
|
||||||
// PingSubject is the services protocol's PING of one instance of one service.
|
// PingSubject is the services protocol's PING of one instance of one service.
|
||||||
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
|
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
|
||||||
|
|
||||||
@@ -58,36 +46,6 @@ func (q *Session) Asker() Asker {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
|
|
||||||
subject := DirectGetSubject(bucket, key)
|
|
||||||
before := l.conn.LastError()
|
|
||||||
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
|
|
||||||
switch {
|
|
||||||
case errors.Is(err, nats.ErrNoResponders):
|
|
||||||
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
|
|
||||||
case err != nil:
|
|
||||||
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before))
|
|
||||||
}
|
|
||||||
if msg.Header != nil {
|
|
||||||
switch status := msg.Header.Get("Status"); status {
|
|
||||||
case "":
|
|
||||||
case "404":
|
|
||||||
return nil, false, nil
|
|
||||||
default:
|
|
||||||
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
|
|
||||||
bucket, status, msg.Header.Get("Description"))
|
|
||||||
}
|
|
||||||
switch msg.Header.Get("KV-Operation") {
|
|
||||||
case "DEL", "PURGE":
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(msg.Data) == 0 {
|
|
||||||
return nil, false, nil
|
|
||||||
}
|
|
||||||
return msg.Data, true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
|
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
|
||||||
subject := PingSubject(service, id)
|
subject := PingSubject(service, id)
|
||||||
before := l.conn.LastError()
|
before := l.conn.LastError()
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package link
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"os"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -12,7 +11,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
|
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
|
||||||
// whether the direct get and the PING work as the two subjects named, and whether a host without them
|
// whether the PING works as the subject named, and whether a host without them
|
||||||
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
|
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
|
||||||
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
||||||
url := os.Getenv("MESH_TEST_NATS_GRANTS")
|
url := os.Getenv("MESH_TEST_NATS_GRANTS")
|
||||||
@@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
|||||||
t.Cleanup(conn.Close)
|
t.Cleanup(conn.Close)
|
||||||
return conn
|
return conn
|
||||||
}
|
}
|
||||||
admin := dial("admin", "x", "_INBOX.admin")
|
|
||||||
js, err := admin.JetStream()
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = js.DeleteKeyValue("mesh-controller_lease")
|
|
||||||
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
|
|
||||||
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
|
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
|
||||||
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
|
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
|
||||||
ask := func() (context.Context, context.CancelFunc) {
|
ask := func() (context.Context, context.CancelFunc) {
|
||||||
return context.WithTimeout(context.Background(), 2*time.Second)
|
return context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Nobody holds it.
|
|
||||||
ctx, cancel := ask()
|
|
||||||
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
|
|
||||||
t.Fatalf("an empty lease read as %v, %v", found, err)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
// Held.
|
|
||||||
if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel = ask()
|
|
||||||
value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder")
|
|
||||||
cancel()
|
|
||||||
if err != nil || !found {
|
|
||||||
t.Fatalf("a held lease read as %v, %v", found, err)
|
|
||||||
}
|
|
||||||
var holder struct {
|
|
||||||
Instance string `json:"instance"`
|
|
||||||
}
|
|
||||||
if json.Unmarshal(value, &holder); holder.Instance == "" {
|
|
||||||
t.Fatalf("the lease's value is %s", value)
|
|
||||||
}
|
|
||||||
// Given back.
|
|
||||||
if err := kv.Delete("holder"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel = ask()
|
|
||||||
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
|
|
||||||
t.Fatalf("a lease given back read as %v, %v", found, err)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
// A host without the grant cannot ask — which is not "nobody holds it".
|
|
||||||
ctx, cancel = ask()
|
|
||||||
if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
|
|
||||||
t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
|
|
||||||
// No runtime on the bus: not there to answer.
|
// No runtime on the bus: not there to answer.
|
||||||
ctx, cancel = ask()
|
ctx, cancel := ask()
|
||||||
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
|
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
|
||||||
t.Fatalf("a PING nobody serves is %v, want no answer", err)
|
t.Fatalf("a PING nobody serves is %v, want no answer", err)
|
||||||
}
|
}
|
||||||
@@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
|
|||||||
t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
|
t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
|
||||||
}
|
}
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
// No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it".
|
|
||||||
if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
ctx, cancel = ask()
|
|
||||||
if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
|
|
||||||
t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err)
|
|
||||||
}
|
|
||||||
cancel()
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,92 +0,0 @@
|
|||||||
package meshcli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/cli"
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
|
||||||
)
|
|
||||||
|
|
||||||
// The controller answers a line as it answers every call (novox/hq ADR 0272, review): in an envelope, `result` or
|
|
||||||
// `error`, and within its window for an answer — so a line still running is answered `running` with its call, and
|
|
||||||
// followed by the same account on the same node until it ends. Its side holds the same field names (mesh-controller
|
|
||||||
// internal/link, the mesh-cli field names' test).
|
|
||||||
|
|
||||||
// ErrNotAsked is the link's: a request it knows it never delivered.
|
|
||||||
var ErrNotAsked = link.ErrNotAsked
|
|
||||||
|
|
||||||
// AskFunc asks the controller one request on this node's subject and returns its reply.
|
|
||||||
type AskFunc func(ctx context.Context, body []byte) ([]byte, error)
|
|
||||||
|
|
||||||
// followEvery is how often a line still running is asked about again.
|
|
||||||
var followEvery = 2 * time.Second
|
|
||||||
|
|
||||||
// envelope is a call's answer: its result, or its error.
|
|
||||||
type envelope struct {
|
|
||||||
Result json.RawMessage `json:"result"`
|
|
||||||
Error string `json:"error"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// runningResult is what a call still running answers in its result.
|
|
||||||
type runningResult struct {
|
|
||||||
Running bool `json:"running"`
|
|
||||||
Call string `json:"call"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// AskTheController asks the line and follows it to its answer, within ctx. What it says is one of three things,
|
|
||||||
// never confused: what the line printed; that nothing ran, and why (the request was never taken); or that what came
|
|
||||||
// of it is not known — it may have run — naming the call to look at when one is known.
|
|
||||||
func AskTheController(ctx context.Context, ask AskFunc, asked Asked) cli.Answer {
|
|
||||||
body, _ := json.Marshal(asked)
|
|
||||||
reply, err := ask(ctx, body)
|
|
||||||
if err != nil {
|
|
||||||
// **Only a request the link knows it never asked ran nothing**: no link open, nobody listening. Any other
|
|
||||||
// failure may come after the request was delivered, and the line may have run (review of ADR 0272).
|
|
||||||
if errors.Is(err, ErrNotAsked) {
|
|
||||||
return cli.Refusal("the controller was not reached, so nothing ran: " + err.Error())
|
|
||||||
}
|
|
||||||
return cli.NotKnown("", "no answer from the controller ("+err.Error()+")")
|
|
||||||
}
|
|
||||||
call := ""
|
|
||||||
for {
|
|
||||||
var env envelope
|
|
||||||
if err := json.Unmarshal(reply, &env); err != nil {
|
|
||||||
return cli.NotKnown(call, "the controller's answer could not be read")
|
|
||||||
}
|
|
||||||
if env.Error != "" {
|
|
||||||
return cli.NotKnown(call, "the controller answered "+env.Error)
|
|
||||||
}
|
|
||||||
var r runningResult
|
|
||||||
_ = json.Unmarshal(env.Result, &r)
|
|
||||||
if !r.Running {
|
|
||||||
var a cli.Answer
|
|
||||||
if err := json.Unmarshal(env.Result, &a); err != nil {
|
|
||||||
return cli.NotKnown(call, "the controller's answer could not be read")
|
|
||||||
}
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
if r.Call != "" {
|
|
||||||
call = r.Call
|
|
||||||
}
|
|
||||||
if call == "" {
|
|
||||||
return cli.NotKnown("", "the controller says the line is running and names no call to follow")
|
|
||||||
}
|
|
||||||
// Asked again until it ends, through a link that may drop and come back: a failed follow is tried
|
|
||||||
// again, and only ctx ending stops it.
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-ctx.Done():
|
|
||||||
return cli.NotKnown(call, fmt.Sprintf("no answer within the time mesh-cli waits (%s)", cli.AskWithin))
|
|
||||||
case <-time.After(followEvery):
|
|
||||||
}
|
|
||||||
body, _ := json.Marshal(Asked{Follow: call, Account: asked.Account, UID: asked.UID, Session: asked.Session})
|
|
||||||
if reply, err = ask(ctx, body); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
package meshcli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
// controllerSays is a controller answering each request in turn from a script, and recording what it was asked.
|
|
||||||
type controllerSays struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
script []func(Asked) ([]byte, error)
|
|
||||||
asked []Asked
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *controllerSays) ask(_ context.Context, body []byte) ([]byte, error) {
|
|
||||||
c.mu.Lock()
|
|
||||||
defer c.mu.Unlock()
|
|
||||||
var a Asked
|
|
||||||
_ = json.Unmarshal(body, &a)
|
|
||||||
c.asked = append(c.asked, a)
|
|
||||||
if len(c.script) == 0 {
|
|
||||||
return nil, context.DeadlineExceeded
|
|
||||||
}
|
|
||||||
next := c.script[0]
|
|
||||||
if len(c.script) > 1 {
|
|
||||||
c.script = c.script[1:]
|
|
||||||
}
|
|
||||||
return next(a)
|
|
||||||
}
|
|
||||||
|
|
||||||
func result(v any) func(Asked) ([]byte, error) {
|
|
||||||
return func(Asked) ([]byte, error) { return json.Marshal(map[string]any{"result": v}) }
|
|
||||||
}
|
|
||||||
|
|
||||||
func fastFollow(t *testing.T) {
|
|
||||||
was := followEvery
|
|
||||||
followEvery = time.Millisecond
|
|
||||||
t.Cleanup(func() { followEvery = was })
|
|
||||||
}
|
|
||||||
|
|
||||||
// A line still running when the controller first answers is followed, by its call, to what it printed.
|
|
||||||
func TestALineStillRunningIsFollowedToItsAnswer(t *testing.T) {
|
|
||||||
fastFollow(t)
|
|
||||||
running := result(map[string]any{"running": true, "call": "call-9"})
|
|
||||||
c := &controllerSays{script: []func(Asked) ([]byte, error){running, running, running,
|
|
||||||
result(map[string]any{"stdout": []byte("pushed\n"), "exit": 0, "terminal": true})}}
|
|
||||||
a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000})
|
|
||||||
if string(a.Stdout) != "pushed\n" || !a.Terminal || a.Refused != "" || a.Unknown != "" {
|
|
||||||
t.Fatalf("followed to %+v", a)
|
|
||||||
}
|
|
||||||
for _, f := range c.asked[1:] {
|
|
||||||
if f.Follow != "call-9" || f.Account != "op" || len(f.Line) != 0 {
|
|
||||||
t.Fatalf("a follow asked %+v", f)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **No answer in time is never "nothing ran"** (review of ADR 0272): the line was asked and may have run, and the
|
|
||||||
// call to look at is named when it is known.
|
|
||||||
func TestNoAnswerInTimeSaysTheLineMayHaveRun(t *testing.T) {
|
|
||||||
fastFollow(t)
|
|
||||||
c := &controllerSays{}
|
|
||||||
a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"push"}, Account: "op"})
|
|
||||||
if a.Refused != "" || !strings.Contains(a.Unknown, "may have run") || a.Exit == 0 {
|
|
||||||
t.Fatalf("a request with no answer reads %+v", a)
|
|
||||||
}
|
|
||||||
c = &controllerSays{script: []func(Asked) ([]byte, error){result(map[string]any{"running": true, "call": "call-3"}),
|
|
||||||
func(Asked) ([]byte, error) { return nil, context.DeadlineExceeded }}}
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
|
|
||||||
defer cancel()
|
|
||||||
a = AskTheController(ctx, c.ask, Asked{Line: []string{"push"}, Account: "op"})
|
|
||||||
if a.Refused != "" || !strings.Contains(a.Unknown, "call-3") || !strings.Contains(a.Unknown, "may have run") {
|
|
||||||
t.Fatalf("a running line never answered reads %+v", a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A request the bus never carried to a controller ran nothing, and says so.
|
|
||||||
func TestARequestNobodyTookSaysNothingRan(t *testing.T) {
|
|
||||||
c := &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) {
|
|
||||||
return nil, fmt.Errorf("%w: nothing on this machine answers mesh.control.laptop.cli", ErrNotAsked)
|
|
||||||
}}}
|
|
||||||
a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"status"}, Account: "op"})
|
|
||||||
if !strings.Contains(a.Refused, "nothing ran") || a.Unknown != "" {
|
|
||||||
t.Fatalf("a request nobody took reads %+v", a)
|
|
||||||
}
|
|
||||||
c = &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) {
|
|
||||||
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
|
|
||||||
}}}
|
|
||||||
if a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"status"}}); !strings.Contains(a.Refused, "nothing ran") {
|
|
||||||
t.Fatalf("no link reads %+v", a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Any error once the request may have been delivered is not known, never "nothing ran"** (review of ADR 0272):
|
|
||||||
// only a request the link says it never asked — no link open, nobody listening — ran nothing.
|
|
||||||
func TestOnlyARequestNeverAskedSaysNothingRan(t *testing.T) {
|
|
||||||
for _, c := range []struct {
|
|
||||||
err error
|
|
||||||
nothing bool
|
|
||||||
}{
|
|
||||||
{fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked), true},
|
|
||||||
{fmt.Errorf("%w: nothing on this machine answers mesh.control.laptop.cli", ErrNotAsked), true},
|
|
||||||
{errors.New("nats: connection closed"), false},
|
|
||||||
{errors.New("mesh.control.laptop.cli: permissions violation"), false},
|
|
||||||
{context.DeadlineExceeded, false},
|
|
||||||
} {
|
|
||||||
err := c.err
|
|
||||||
ctl := &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) { return nil, err }}}
|
|
||||||
a := AskTheController(context.Background(), ctl.ask, Asked{Line: []string{"push"}, Account: "op"})
|
|
||||||
if c.nothing && (!strings.Contains(a.Refused, "nothing ran") || a.Unknown != "") {
|
|
||||||
t.Errorf("%v reads %+v; it was never asked", err, a)
|
|
||||||
}
|
|
||||||
if !c.nothing && (a.Refused != "" || !strings.Contains(a.Unknown, "may have run")) {
|
|
||||||
t.Errorf("%v reads %+v; it may have been asked", err, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,42 +0,0 @@
|
|||||||
package meshcli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"net"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/cli"
|
|
||||||
)
|
|
||||||
|
|
||||||
// **A connection that says nothing is bounded from the moment it is accepted** (review of ADR 0272): its deadline is
|
|
||||||
// set before any lookup, so it is answered and closed within readWithin, holds no slot after, and asks nothing.
|
|
||||||
func TestAConnectionThatSaysNothingIsAnsweredWithinItsDeadline(t *testing.T) {
|
|
||||||
was := readWithin
|
|
||||||
readWithin = 200 * time.Millisecond
|
|
||||||
t.Cleanup(func() { readWithin = was })
|
|
||||||
asked := false
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
asked = true
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
conn, err := net.Dial("unix", socket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
|
||||||
a, err := cli.ReadAnswer(conn)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("a silent connection was not answered within its deadline: %v", err)
|
|
||||||
}
|
|
||||||
if asked || a.Refused == "" || !strings.Contains(a.Refused, "nothing was asked") {
|
|
||||||
t.Fatalf("a silent connection was asked (%v) or answered %+v", asked, a)
|
|
||||||
}
|
|
||||||
// Its slot was given back: the account's next line is not busy.
|
|
||||||
b, err := cli.Ask(context.Background(), socket, []string{"status"})
|
|
||||||
if err != nil || strings.Contains(b.Refused, "busy") || !asked {
|
|
||||||
t.Fatalf("the next line after a silent one reads %+v (%v), asked %v", b, err, asked)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,339 +0,0 @@
|
|||||||
// Package meshcli is the node-engine's side of `mesh-cli` (novox/hq ADR 0272): a socket on this machine that any
|
|
||||||
// account may connect to, the account read from the kernel, and the line asked of the controller.
|
|
||||||
//
|
|
||||||
// **The node-engine vouches; it does not decide.** It adds what only it can know — which account on this machine
|
|
||||||
// is asking, from the kernel's peer credentials, never from anything the program says — and sends the request out
|
|
||||||
// over its own link, on its own node's subject. The controller judges whether the line runs as its terminal, as an
|
|
||||||
// ordinary call, or not at all (ADR 0272 §4). Nothing received here is run or applied on the machine.
|
|
||||||
//
|
|
||||||
// **No listening control surface** (novox/hq ADR 0004, ADR 0272 §3): the socket has no network address, so
|
|
||||||
// nothing on another machine reaches it, and nothing sent to it acts on this one.
|
|
||||||
package meshcli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"context"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"os/user"
|
|
||||||
"path/filepath"
|
|
||||||
"strconv"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/cli"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Asked is what the node-engine sends the controller on its own node's subject: the line, and the account the
|
|
||||||
// kernel says is asking. The controller's side holds the same field names (a test on each side).
|
|
||||||
type Asked struct {
|
|
||||||
Line []string `json:"line,omitempty"`
|
|
||||||
// Account is the login of the user id the kernel gave, from this machine's user database.
|
|
||||||
Account string `json:"account"`
|
|
||||||
// UID is the user id itself, so a renamed or duplicated login is still the same id.
|
|
||||||
UID uint32 `json:"uid"`
|
|
||||||
// Follow is a call a line runs as, asked about again by the account that asked it (AskTheController).
|
|
||||||
Follow string `json:"follow,omitempty"`
|
|
||||||
// Session is the login session the asking process runs in — its systemd scope, `session-<id>.scope` under the
|
|
||||||
// account's own slice — as the kernel's cgroup says it; empty for anything else: a system service, a user unit,
|
|
||||||
// a process another account's session started. Only a login session is a person at a terminal (novox/hq ADR
|
|
||||||
// 0272): the controller makes nothing else the terminal.
|
|
||||||
Session string `json:"session,omitempty"`
|
|
||||||
// Stdin is what mesh-cli's standard input held, at most cli.MaxStdin: carried to the controller with the line
|
|
||||||
// and nowhere else. The engine says it in no journal, keeps it in no state, and never sends it again: a follow
|
|
||||||
// carries none. The controller gives it to a line that runs as its terminal alone.
|
|
||||||
Stdin []byte `json:"stdin,omitempty"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// Forward asks the controller and follows the line to what came of it (AskTheController): what it printed, that
|
|
||||||
// nothing ran and why, or that it is not known.
|
|
||||||
type Forward func(ctx context.Context, asked Asked) cli.Answer
|
|
||||||
|
|
||||||
// readWithin bounds how long a connection may take to say its line: a program that connects and says nothing
|
|
||||||
// holds nothing for long. A variable so a test need not wait.
|
|
||||||
var readWithin = 30 * time.Second
|
|
||||||
|
|
||||||
// refusedWithin bounds how long the engine waits for the request of a line it refuses before it is asked: a client
|
|
||||||
// that has connected writes its request within microseconds, so a refused connection holds nothing for long. Never
|
|
||||||
// longer than readWithin. A variable so a test can say.
|
|
||||||
var refusedWithin = 1 * time.Second
|
|
||||||
|
|
||||||
// refusalsWaiting bounds how many refusals wait for their request at the same time, on the whole engine; drainCap
|
|
||||||
// bounds how much of a refused request is read, and it is read into a small buffer and dropped. The socket is open
|
|
||||||
// to every account, so a refusal must cost the engine little however many connections ask (review of ADR 0272).
|
|
||||||
// Variables so a test can say.
|
|
||||||
var (
|
|
||||||
refusalsWaiting = make(chan struct{}, 32)
|
|
||||||
drainCap = 4 << 10
|
|
||||||
)
|
|
||||||
|
|
||||||
// drain reads and drops what a refused client wrote, up to the end of its request line, the end of the stream, the
|
|
||||||
// connection's deadline or drainCap bytes, whichever comes first. Nothing is kept.
|
|
||||||
func drain(r io.Reader) {
|
|
||||||
var buf [512]byte
|
|
||||||
for read := 0; read < drainCap; {
|
|
||||||
n, err := r.Read(buf[:min(len(buf), drainCap-read)])
|
|
||||||
read += n
|
|
||||||
if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// atOnce bounds how many lines are asked at the same time, and perAccount how many of them one account's: a person
|
|
||||||
// types one at a time, so more is a program on the machine. One over either bound is answered busy at once — the
|
|
||||||
// engine never stops accepting for it, so one account cannot hold the socket from every other (review of ADR 0272).
|
|
||||||
var (
|
|
||||||
atOnce = 8
|
|
||||||
perAccount = 2
|
|
||||||
)
|
|
||||||
|
|
||||||
// slots counts the lines being asked, in all and per user id.
|
|
||||||
type slots struct {
|
|
||||||
mu sync.Mutex
|
|
||||||
all int
|
|
||||||
byUID map[uint32]int
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *slots) take(uid uint32) bool {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
if s.all >= atOnce || s.byUID[uid] >= perAccount {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
s.all++
|
|
||||||
s.byUID[uid]++
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *slots) give(uid uint32) {
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
s.all--
|
|
||||||
if s.byUID[uid]--; s.byUID[uid] <= 0 {
|
|
||||||
delete(s.byUID, uid)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// peerOf reads the connecting process's credentials from the kernel: its user id and process id. A variable so a
|
|
||||||
// test can say what the kernel would; it is the only source of the account.
|
|
||||||
var peerOf = peerCredentials
|
|
||||||
|
|
||||||
// sessionOf reads the login session a process runs in, from the kernel. A variable so a test can say.
|
|
||||||
var sessionOf = func(pid int32, uid uint32) string {
|
|
||||||
raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid))
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return userSessionIn(string(raw), uid)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sessionsDir is where logind keeps a record of each session it holds. A variable so a test can say.
|
|
||||||
var sessionsDir = "/run/systemd/sessions"
|
|
||||||
|
|
||||||
// userSessionIn is the login session a cgroup names, when logind's own record of it says it is a person's: this
|
|
||||||
// account's (UID), of class `user` — not a greeter's or the user manager's — and of this very scope. The cgroup
|
|
||||||
// alone names a scope; only the record says whose session it is. No record, or one that does not say so, is no
|
|
||||||
// session.
|
|
||||||
func userSessionIn(cgroup string, uid uint32) string {
|
|
||||||
scope := loginSessionIn(cgroup, uid)
|
|
||||||
id, ok := strings.CutPrefix(strings.TrimSuffix(scope, ".scope"), "session-")
|
|
||||||
if !ok || id == "" || strings.ContainsAny(id, "/.") {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
raw, err := os.ReadFile(filepath.Join(sessionsDir, id))
|
|
||||||
if err != nil {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
// **The record is systemd's private format**, not an interface it promises: `KEY=value` lines logind writes for
|
|
||||||
// itself. Only UID, CLASS and SCOPE are read, each at its first occurrence — logind writes each once, and a later
|
|
||||||
// line saying otherwise is not the record's word. A systemd that changes the format, renames a key or stops
|
|
||||||
// writing the file fails closed here: no session is read, so nobody is the terminal until this reads it again.
|
|
||||||
record := map[string]string{}
|
|
||||||
for _, line := range strings.Split(string(raw), "\n") {
|
|
||||||
k, v, ok := strings.Cut(line, "=")
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if _, said := record[k]; !said {
|
|
||||||
record[k] = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if record["UID"] != strconv.FormatUint(uint64(uid), 10) || record["CLASS"] != "user" || record["SCOPE"] != scope {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return scope
|
|
||||||
}
|
|
||||||
|
|
||||||
// loginSessionIn is the login session scope a process's cgroup names, for its own account: the unified hierarchy's
|
|
||||||
// line, `0::/user.slice/user-<uid>.slice/session-<id>.scope`, and nothing below or beside it. Anything else is no
|
|
||||||
// login session — a service under system.slice, a unit under the account's user manager (user@<uid>.service), a
|
|
||||||
// scope of another account's.
|
|
||||||
func loginSessionIn(cgroup string, uid uint32) string {
|
|
||||||
for _, line := range strings.Split(cgroup, "\n") {
|
|
||||||
path, ok := strings.CutPrefix(line, "0::")
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
prefix := fmt.Sprintf("/user.slice/user-%d.slice/", uid)
|
|
||||||
scope, ok := strings.CutPrefix(path, prefix)
|
|
||||||
if !ok || strings.Contains(scope, "/") || !strings.HasPrefix(scope, "session-") || !strings.HasSuffix(scope, ".scope") {
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
return scope
|
|
||||||
}
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// sameNamespaces refuses a peer whose mount or user namespace is not the engine's. A variable so a test can say.
|
|
||||||
var sameNamespaces = namespacesOf
|
|
||||||
|
|
||||||
// lookupAccount names a user id from this machine's user database.
|
|
||||||
var lookupAccount = func(uid uint32) (string, error) {
|
|
||||||
u, err := user.LookupId(strconv.FormatUint(uint64(uid), 10))
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
return u.Username, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Serve listens on socket until ctx ends, and answers each connection: one request, one answer.
|
|
||||||
//
|
|
||||||
// The directory is made 0755 and the socket 0666: every account on the machine may ask, because the account is
|
|
||||||
// what is vouched for and the controller decides what it may do. A file at the socket's path that is not a socket
|
|
||||||
// is never removed: the engine says so and serves nothing.
|
|
||||||
func Serve(ctx context.Context, socket string, forward Forward, say func(string)) error {
|
|
||||||
dir := filepath.Dir(socket)
|
|
||||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
|
||||||
return fmt.Errorf("mesh-cli's socket directory: %w", err)
|
|
||||||
}
|
|
||||||
// The directory is the engine's: a link, or one another account owns, could put the socket where that
|
|
||||||
// account replaces it with its own. Said, and nothing served.
|
|
||||||
if problem := notOwnDirectory(dir); problem != "" {
|
|
||||||
return fmt.Errorf("mesh-cli is not served on this machine: %s", problem)
|
|
||||||
}
|
|
||||||
if err := os.Chmod(dir, 0o755); err != nil {
|
|
||||||
return fmt.Errorf("mesh-cli's socket directory: %w", err)
|
|
||||||
}
|
|
||||||
if info, err := os.Lstat(socket); err == nil {
|
|
||||||
if info.Mode()&os.ModeSocket == 0 {
|
|
||||||
return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; "+
|
|
||||||
"mesh-cli is not served on this machine until it is moved", socket)
|
|
||||||
}
|
|
||||||
// A socket left by the engine before this one: nothing listens on it any more.
|
|
||||||
if err := os.Remove(socket); err != nil {
|
|
||||||
return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("listening for mesh-cli at %s: %w", socket, err)
|
|
||||||
}
|
|
||||||
if err := os.Chmod(socket, 0o666); err != nil {
|
|
||||||
l.Close()
|
|
||||||
return fmt.Errorf("opening mesh-cli's socket to every account: %w", err)
|
|
||||||
}
|
|
||||||
go func() {
|
|
||||||
<-ctx.Done()
|
|
||||||
l.Close()
|
|
||||||
}()
|
|
||||||
held := &slots{byUID: map[uint32]int{}}
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
defer wg.Wait()
|
|
||||||
for {
|
|
||||||
conn, err := l.AcceptUnix()
|
|
||||||
if err != nil {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
var ne net.Error
|
|
||||||
if errors.As(err, &ne) && ne.Timeout() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
return fmt.Errorf("mesh-cli's socket: %w", err)
|
|
||||||
}
|
|
||||||
// Bounded from the first byte: nothing below may wait on a connection for longer than this before the
|
|
||||||
// line is read, whatever the peer does (review of ADR 0272).
|
|
||||||
readBy := time.Now().Add(readWithin)
|
|
||||||
_ = conn.SetDeadline(readBy)
|
|
||||||
wg.Add(1)
|
|
||||||
go func() {
|
|
||||||
defer wg.Done()
|
|
||||||
answer(ctx, conn, readBy, held, forward, say)
|
|
||||||
}()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// answer serves one connection; readBy is the deadline set when it was accepted, for reading its line.
|
|
||||||
func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, held *slots, forward Forward, say func(string)) {
|
|
||||||
defer conn.Close()
|
|
||||||
reply := func(a cli.Answer) {
|
|
||||||
_ = conn.SetWriteDeadline(time.Now().Add(readWithin))
|
|
||||||
if err := cli.WriteAnswer(conn, a); err != nil && say != nil {
|
|
||||||
say(fmt.Sprintf("mesh-cli: the answer could not be written back: %v", err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// A refusal is written only after the request it answers is read (novox/hq issue 434). The client writes its
|
|
||||||
// request first and reads the answer after; an engine that answers and closes before that write has landed
|
|
||||||
// makes the write fail with a broken pipe, and the client never reads the refusal waiting for it — under load a
|
|
||||||
// line beyond its account's bound read as an error instead of busy. The wait ends at refusedWithin, or earlier
|
|
||||||
// at the deadline set when the connection was accepted; what was written is drained and never asked. Only
|
|
||||||
// refusalsWaiting refusals wait at once: over that the refusal is written at once, as before, and only then can a
|
|
||||||
// client still find its write broken.
|
|
||||||
refuse := func(why string) {
|
|
||||||
select {
|
|
||||||
case refusalsWaiting <- struct{}{}:
|
|
||||||
by := time.Now().Add(refusedWithin)
|
|
||||||
if readBy.Before(by) {
|
|
||||||
by = readBy
|
|
||||||
}
|
|
||||||
_ = conn.SetReadDeadline(by)
|
|
||||||
drain(conn)
|
|
||||||
<-refusalsWaiting
|
|
||||||
default:
|
|
||||||
}
|
|
||||||
reply(cli.Refusal(why))
|
|
||||||
}
|
|
||||||
uid, pid, err := peerOf(conn)
|
|
||||||
if err != nil {
|
|
||||||
// Never sent as anyone: an account the kernel did not give is no account at all.
|
|
||||||
refuse("the node-engine could not read which account is asking, so nothing was asked: " + err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// The user id is this machine's account only in this machine's namespaces: a container running with the
|
|
||||||
// operator's id shares the number and nothing else (review of ADR 0272).
|
|
||||||
if err := sameNamespaces(pid); err != nil {
|
|
||||||
refuse("the asking process does not run in this machine's own namespaces, so it is not this " +
|
|
||||||
"machine's account and nothing was asked: " + err.Error())
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// And the login session, now: judged as the peer was when it connected, before it can exec or exit while its
|
|
||||||
// line is read (review of ADR 0272).
|
|
||||||
session := sessionOf(pid, uid)
|
|
||||||
// Looked up before a slot is taken, so a refused account never holds one while it is refused.
|
|
||||||
account, err := lookupAccount(uid)
|
|
||||||
if err != nil || account == "" {
|
|
||||||
refuse(fmt.Sprintf("user id %d is not in this machine's user database, so nothing was asked", uid))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if !held.take(uid) {
|
|
||||||
refuse(fmt.Sprintf("busy: user id %d has %d lines being asked already, or the machine %d; ask "+
|
|
||||||
"again when one has ended. Nothing was asked", uid, perAccount, atOnce))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
defer held.give(uid)
|
|
||||||
req, err := cli.ReadRequest(conn)
|
|
||||||
if err != nil {
|
|
||||||
reply(cli.Refusal("the request could not be read, so nothing was asked: " + err.Error()))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
// From here the wait is the controller's: as long as mesh-cli waits, and no longer.
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(cli.AskWithin))
|
|
||||||
asking, cancel := context.WithTimeout(ctx, cli.AskWithin-30*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
reply(forward(asking, Asked{Line: req.Line, Account: account, UID: uid, Session: session, Stdin: req.Stdin}))
|
|
||||||
}
|
|
||||||
@@ -1,534 +0,0 @@
|
|||||||
package meshcli
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"io"
|
|
||||||
"net"
|
|
||||||
"os"
|
|
||||||
"os/user"
|
|
||||||
"path/filepath"
|
|
||||||
"sort"
|
|
||||||
"strings"
|
|
||||||
"sync"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/cli"
|
|
||||||
)
|
|
||||||
|
|
||||||
// serving starts Serve on a socket of the test's own, and stops it when the test ends.
|
|
||||||
func serving(t *testing.T, forward Forward) string {
|
|
||||||
t.Helper()
|
|
||||||
socket := filepath.Join(t.TempDir(), "run", "cli.sock")
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() { done <- Serve(ctx, socket, forward, func(string) {}) }()
|
|
||||||
t.Cleanup(func() {
|
|
||||||
cancel()
|
|
||||||
if err := <-done; err != nil {
|
|
||||||
t.Errorf("Serve ended with %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
for i := 0; i < 200; i++ {
|
|
||||||
if _, err := os.Stat(socket); err == nil {
|
|
||||||
return socket
|
|
||||||
}
|
|
||||||
time.Sleep(5 * time.Millisecond)
|
|
||||||
}
|
|
||||||
t.Fatal("the socket never appeared")
|
|
||||||
return ""
|
|
||||||
}
|
|
||||||
|
|
||||||
// The account sent is the one the kernel gives for the connecting process, and the answer comes back as it was.
|
|
||||||
func TestTheAccountAskingIsTheKernelsAndTheAnswerComesBack(t *testing.T) {
|
|
||||||
var got Asked
|
|
||||||
socket := serving(t, func(_ context.Context, asked Asked) cli.Answer {
|
|
||||||
got = asked
|
|
||||||
return cli.Answer{Stdout: []byte("done\n"), Exit: 0, Terminal: true, Why: "the terminal"}
|
|
||||||
})
|
|
||||||
a, err := cli.Ask(context.Background(), socket, []string{"settings", "show", "zsh"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if string(a.Stdout) != "done\n" || !a.Terminal || a.Refused != "" {
|
|
||||||
t.Fatalf("the answer came back as %+v", a)
|
|
||||||
}
|
|
||||||
me, err := user.Current()
|
|
||||||
if err != nil {
|
|
||||||
t.Skipf("this machine cannot name the test's account: %v", err)
|
|
||||||
}
|
|
||||||
if int(got.UID) != os.Getuid() || got.Account != me.Username {
|
|
||||||
t.Fatalf("asked as %d %q, and the kernel says %d %q", got.UID, got.Account, os.Getuid(), me.Username)
|
|
||||||
}
|
|
||||||
if strings.Join(got.Line, " ") != "settings show zsh" {
|
|
||||||
t.Fatalf("the line was sent as %q", got.Line)
|
|
||||||
}
|
|
||||||
if info, err := os.Stat(socket); err != nil || info.Mode().Perm() != 0o666 {
|
|
||||||
t.Fatalf("the socket is not open to every account: %v %v", info.Mode(), err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Credentials the kernel did not give are no account: refused, and nothing is asked.
|
|
||||||
func TestAConnectionWhoseAccountCannotBeReadIsNeverAskedAsAnyone(t *testing.T) {
|
|
||||||
before := peerOf
|
|
||||||
peerOf = func(*net.UnixConn) (uint32, int32, error) { return 0, 0, errors.New("no credentials") }
|
|
||||||
t.Cleanup(func() { peerOf = before })
|
|
||||||
asked := false
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
asked = true
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
a, err := cli.Ask(context.Background(), socket, []string{"status"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if asked || a.Refused == "" || a.Exit == 0 {
|
|
||||||
t.Fatalf("a connection with no account was asked (%v) or not refused: %+v", asked, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A request over the bound is refused and never asked.
|
|
||||||
func TestARequestOverTheBoundIsRefused(t *testing.T) {
|
|
||||||
asked := false
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
asked = true
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
a, err := cli.Ask(context.Background(), socket, []string{strings.Repeat("x", cli.MaxRequest)})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if asked || a.Refused == "" {
|
|
||||||
t.Fatalf("a request over the bound was asked (%v): %+v", asked, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A file at the socket's path that is not a socket is never removed.
|
|
||||||
func TestAFileThatIsNotASocketIsKept(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
socket := filepath.Join(dir, "cli.sock")
|
|
||||||
if err := os.WriteFile(socket, []byte("mine"), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
err := Serve(context.Background(), socket, nil, nil)
|
|
||||||
if err == nil || !strings.Contains(err.Error(), "not a socket") {
|
|
||||||
t.Fatalf("a file at the socket's path was not refused: %v", err)
|
|
||||||
}
|
|
||||||
if body, _ := os.ReadFile(socket); string(body) != "mine" {
|
|
||||||
t.Fatal("the file at the socket's path was touched")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The request the controller reads holds these field names; the controller's side holds the same list
|
|
||||||
// (mesh-controller, the mesh-cli request's test).
|
|
||||||
func TestTheRequestToTheControllerKeepsItsFieldNames(t *testing.T) {
|
|
||||||
body, err := json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
body, _ = json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1, Session: "session-3.scope"})
|
|
||||||
if got := keysOf(t, body); got != "account line session uid" {
|
|
||||||
t.Fatalf("the request's fields are %q", got)
|
|
||||||
}
|
|
||||||
body, _ = json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1})
|
|
||||||
if got := keysOf(t, body); got != "account line uid" {
|
|
||||||
t.Fatalf("the request's fields are %q", got)
|
|
||||||
}
|
|
||||||
body, _ = json.Marshal(Asked{Line: []string{"secret"}, Account: "a", UID: 1, Stdin: []byte("x")})
|
|
||||||
if got := keysOf(t, body); got != "account line stdin uid" {
|
|
||||||
t.Fatalf("a request with standard input has the fields %q", got)
|
|
||||||
}
|
|
||||||
body, _ = json.Marshal(Asked{Follow: "call-1", Account: "a", UID: 1})
|
|
||||||
if got := keysOf(t, body); got != "account follow uid" {
|
|
||||||
t.Fatalf("a follow's fields are %q", got)
|
|
||||||
}
|
|
||||||
// What the controller answers as a line's result, and as a running line's: the same names its side holds.
|
|
||||||
body, err = json.Marshal(cli.Answer{Stdout: []byte("o"), Stderr: []byte("e"), Exit: 1, Terminal: true, Why: "w",
|
|
||||||
Refused: "r", Cut: true})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if got := keysOf(t, body); got != "cut exit refused stderr stdout terminal why" {
|
|
||||||
t.Fatalf("the answer's fields are %q", got)
|
|
||||||
}
|
|
||||||
body, _ = json.Marshal(runningResult{Running: true, Call: "call-1"})
|
|
||||||
if got := keysOf(t, body); got != "call running" {
|
|
||||||
t.Fatalf("a running answer's fields are %q", got)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func keysOf(t *testing.T, body []byte) string {
|
|
||||||
t.Helper()
|
|
||||||
var m map[string]any
|
|
||||||
if err := json.Unmarshal(body, &m); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
var keys []string
|
|
||||||
for k := range m {
|
|
||||||
keys = append(keys, k)
|
|
||||||
}
|
|
||||||
sort.Strings(keys)
|
|
||||||
return strings.Join(keys, " ")
|
|
||||||
}
|
|
||||||
|
|
||||||
// holdALine asks one line and returns once the engine is asking it, so it holds its account's slot. A first line
|
|
||||||
// that ends or never arrives fails the test at once, with what came of it, instead of hanging the suite (novox/hq
|
|
||||||
// issue 434).
|
|
||||||
func holdALine(t *testing.T, socket string, started <-chan struct{}) {
|
|
||||||
t.Helper()
|
|
||||||
first := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
_, err := cli.Ask(context.Background(), socket, []string{"push"})
|
|
||||||
first <- err
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case <-started:
|
|
||||||
case err := <-first:
|
|
||||||
t.Fatalf("the first line ended before it was asked: %v", err)
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("the first line was never asked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **One account cannot hold the socket** (review of ADR 0272): its lines beyond its own bound are answered busy at
|
|
||||||
// once, nothing is asked, and the engine goes on accepting.
|
|
||||||
func TestOneAccountsLinesBeyondItsBoundAreAnsweredBusy(t *testing.T) {
|
|
||||||
was := perAccount
|
|
||||||
perAccount = 1
|
|
||||||
t.Cleanup(func() { perAccount = was })
|
|
||||||
release := make(chan struct{})
|
|
||||||
started := make(chan struct{}, 2)
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
// Released before the engine is stopped: cleanups run last first, and the engine waits for its lines.
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
holdALine(t, socket, started)
|
|
||||||
type asked struct {
|
|
||||||
a cli.Answer
|
|
||||||
err error
|
|
||||||
}
|
|
||||||
done := make(chan asked, 1)
|
|
||||||
go func() {
|
|
||||||
a, err := cli.Ask(context.Background(), socket, []string{"status"})
|
|
||||||
done <- asked{a, err}
|
|
||||||
}()
|
|
||||||
select {
|
|
||||||
case got := <-done:
|
|
||||||
// An error is a busy line the client never read (novox/hq issue 434), not an answer.
|
|
||||||
if got.err != nil || !strings.Contains(got.a.Refused, "busy") {
|
|
||||||
t.Fatalf("a second line from the same account reads %+v (%v)", got.a, got.err)
|
|
||||||
}
|
|
||||||
case <-time.After(5 * time.Second):
|
|
||||||
t.Fatal("a second line from the same account waited instead of being answered busy")
|
|
||||||
}
|
|
||||||
if len(started) != 0 {
|
|
||||||
t.Fatal("the busy line was asked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A refusal waits for the request it answers** (novox/hq issue 434): the client writes its line first and reads
|
|
||||||
// the answer after. An engine that answered busy and closed before that write landed broke the client's write, and
|
|
||||||
// the refusal was never read — under the whole suite's load the busy line read as an error. Here the client
|
|
||||||
// connects and holds its line back: no answer may come before the line, and the line, once written, is answered
|
|
||||||
// busy. Ordered by the client's own write, so it holds however slow the machine is.
|
|
||||||
func TestARefusalWaitsForTheRequestItAnswers(t *testing.T) {
|
|
||||||
was, wasWithin := perAccount, refusedWithin
|
|
||||||
perAccount, refusedWithin = 1, time.Minute
|
|
||||||
t.Cleanup(func() { perAccount, refusedWithin = was, wasWithin })
|
|
||||||
release := make(chan struct{})
|
|
||||||
started := make(chan struct{}, 2)
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
holdALine(t, socket, started)
|
|
||||||
conn, err := net.Dial("unix", socket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
// Nothing is written yet: an engine that answers now answers before the request. The wait is far inside
|
|
||||||
// refusedWithin, so an engine that waits is still waiting when it ends.
|
|
||||||
_ = conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
|
|
||||||
var b [1]byte
|
|
||||||
if n, err := conn.Read(b[:]); n > 0 || !errors.Is(err, os.ErrDeadlineExceeded) {
|
|
||||||
t.Fatalf("the engine answered before the request was written (%d bytes, %v)", n, err)
|
|
||||||
}
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
|
||||||
if err := cli.WriteRequest(conn, cli.Request{Line: []string{"status"}}); err != nil {
|
|
||||||
t.Fatalf("the request could not be written: %v", err)
|
|
||||||
}
|
|
||||||
a, err := cli.ReadAnswer(conn)
|
|
||||||
if err != nil || !strings.Contains(a.Refused, "busy") {
|
|
||||||
t.Fatalf("a line written after the engine chose busy reads %+v (%v)", a, err)
|
|
||||||
}
|
|
||||||
if len(started) != 0 {
|
|
||||||
t.Fatal("the busy line was asked")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **Refusals that wait are bounded** (novox/hq issue 434, review of ADR 0272): with refusalsWaiting already full, a
|
|
||||||
// further refused line is answered at once, without waiting for its request.
|
|
||||||
func TestARefusalOverTheWaitingBoundIsAnsweredAtOnce(t *testing.T) {
|
|
||||||
was, wasWithin, wasWaiting := perAccount, refusedWithin, refusalsWaiting
|
|
||||||
perAccount, refusedWithin = 1, time.Minute
|
|
||||||
refusalsWaiting = make(chan struct{}, 1)
|
|
||||||
refusalsWaiting <- struct{}{}
|
|
||||||
t.Cleanup(func() { perAccount, refusedWithin, refusalsWaiting = was, wasWithin, wasWaiting })
|
|
||||||
release := make(chan struct{})
|
|
||||||
started := make(chan struct{}, 2)
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
started <- struct{}{}
|
|
||||||
<-release
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
t.Cleanup(func() { close(release) })
|
|
||||||
holdALine(t, socket, started)
|
|
||||||
conn, err := net.Dial("unix", socket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
// Nothing is written: a refusal that waited would wait refusedWithin, a minute, for it.
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
|
||||||
a, err := cli.ReadAnswer(conn)
|
|
||||||
if err != nil || !strings.Contains(a.Refused, "busy") {
|
|
||||||
t.Fatalf("a refusal over the waiting bound reads %+v (%v)", a, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// countingReader is an endless request with no line end, counting what was read of it.
|
|
||||||
type countingReader struct{ read int }
|
|
||||||
|
|
||||||
func (c *countingReader) Read(p []byte) (int, error) {
|
|
||||||
for i := range p {
|
|
||||||
p[i] = 'x'
|
|
||||||
}
|
|
||||||
c.read += len(p)
|
|
||||||
return len(p), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A refused request is drained, not kept** (novox/hq issue 434): what a refused client writes is read up to
|
|
||||||
// drainCap bytes and no further, however much it sends.
|
|
||||||
func TestARefusedRequestIsDrainedNoFurtherThanItsCap(t *testing.T) {
|
|
||||||
r := &countingReader{}
|
|
||||||
drain(r)
|
|
||||||
if r.read != drainCap {
|
|
||||||
t.Fatalf("drained %d bytes of an endless request; the cap is %d", r.read, drainCap)
|
|
||||||
}
|
|
||||||
after := &countingReader{}
|
|
||||||
drain(io.MultiReader(strings.NewReader("{\"line\":[\"status\"]}\n"), after))
|
|
||||||
if after.read != 0 {
|
|
||||||
t.Fatalf("draining read %d bytes past the end of the request line", after.read)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A process in another namespace is not this machine's account** (review of ADR 0272): a container running with
|
|
||||||
// the operator's user id shares the id and not the machine; refused, and nothing is asked.
|
|
||||||
func TestAPeerInAnotherNamespaceIsRefused(t *testing.T) {
|
|
||||||
before := sameNamespaces
|
|
||||||
sameNamespaces = func(pid int32) error { return errors.New("its mount namespace is not the engine's") }
|
|
||||||
t.Cleanup(func() { sameNamespaces = before })
|
|
||||||
asked := false
|
|
||||||
socket := serving(t, func(context.Context, Asked) cli.Answer {
|
|
||||||
asked = true
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
a, err := cli.Ask(context.Background(), socket, []string{"status"})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if asked || !strings.Contains(a.Refused, "namespace") {
|
|
||||||
t.Fatalf("a peer in another namespace was asked (%v): %+v", asked, a)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The session a line comes from is the kernel's word, and only a login session counts** (review of ADR 0272): a
|
|
||||||
// service running as the operator — the tool runner, a user unit — is not a person at a terminal. The engine reads
|
|
||||||
// the peer's cgroup and sends the login session's scope, or nothing; the controller makes no other terminal.
|
|
||||||
func TestTheLoginSessionAskingIsReadFromTheKernel(t *testing.T) {
|
|
||||||
for _, c := range []struct{ cgroup, session string }{
|
|
||||||
{"0::/user.slice/user-1000.slice/session-3.scope\n", "session-3.scope"},
|
|
||||||
{"0::/user.slice/user-1000.slice/session-c1.scope\n", "session-c1.scope"},
|
|
||||||
{"0::/user.slice/user-1000.slice/user@1000.service/app.slice/tmux.scope\n", ""},
|
|
||||||
{"0::/system.slice/node-tools.service\n", ""},
|
|
||||||
{"0::/user.slice/user-1001.slice/session-3.scope\n", ""}, // another account's session
|
|
||||||
{"12:pids:/user.slice\n0::/user.slice/user-1000.slice/session-9.scope/sub\n", ""},
|
|
||||||
// Below a session scope, in a scope of its own: only the check that the scope is the account slice's direct
|
|
||||||
// child refuses it.
|
|
||||||
{"0::/user.slice/user-1000.slice/session-9.scope/x.scope\n", ""},
|
|
||||||
{"", ""},
|
|
||||||
} {
|
|
||||||
if got := loginSessionIn(c.cgroup, 1000); got != c.session {
|
|
||||||
t.Errorf("%q read as %q, want %q", c.cgroup, got, c.session)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// And it is sent: the test process's own, whatever it is, as the kernel says it.
|
|
||||||
var got Asked
|
|
||||||
socket := serving(t, func(_ context.Context, asked Asked) cli.Answer {
|
|
||||||
got = asked
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
if _, err := cli.Ask(context.Background(), socket, []string{"status"}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
raw, _ := os.ReadFile("/proc/self/cgroup")
|
|
||||||
if want := userSessionIn(string(raw), uint32(os.Getuid())); got.Session != want {
|
|
||||||
t.Fatalf("sent session %q, the kernel says %q", got.Session, want)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **A login session is a person's: systemd's own record says so** (review of ADR 0272): the scope the cgroup names
|
|
||||||
// must be a session logind keeps for this very account, of class `user` — not a greeter's, not the user manager's,
|
|
||||||
// not one whose record names another account — and read at accept, with the namespaces, before anything else.
|
|
||||||
func TestALoginSessionIsAUsersByItsOwnRecord(t *testing.T) {
|
|
||||||
dir := t.TempDir()
|
|
||||||
was := sessionsDir
|
|
||||||
sessionsDir = dir
|
|
||||||
t.Cleanup(func() { sessionsDir = was })
|
|
||||||
write := func(id, body string) {
|
|
||||||
if err := os.WriteFile(filepath.Join(dir, id), []byte(body), 0o644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
write("9", "UID=1000\nUSER=op\nCLASS=user\nSCOPE=session-9.scope\nACTIVE=1\n")
|
|
||||||
write("3", "UID=1000\nCLASS=greeter\nSCOPE=session-3.scope\n")
|
|
||||||
write("5", "UID=1000\nCLASS=user\nSCOPE=session-6.scope\n")
|
|
||||||
// The account the record names is not the slice's: only the record's UID check refuses it.
|
|
||||||
write("8", "UID=1001\nCLASS=user\nSCOPE=session-8.scope\n")
|
|
||||||
// A name with a dot, held as a record that would pass: only the check of the id's characters refuses it.
|
|
||||||
write("a.b", "UID=1000\nCLASS=user\nSCOPE=session-a.b.scope\n")
|
|
||||||
// A key said twice: the first is the record's word (systemd writes each once); a later one is not read.
|
|
||||||
write("11", "UID=1001\nCLASS=user\nSCOPE=session-11.scope\nUID=1000\n")
|
|
||||||
write("12", "UID=1000\nCLASS=user\nSCOPE=session-12.scope\nCLASS=greeter\nUID=1001\n")
|
|
||||||
for _, c := range []struct {
|
|
||||||
cgroup string
|
|
||||||
uid uint32
|
|
||||||
session string
|
|
||||||
}{
|
|
||||||
{"0::/user.slice/user-1000.slice/session-9.scope\n", 1000, "session-9.scope"},
|
|
||||||
{"0::/user.slice/user-1000.slice/x.scope\n", 1000, ""},
|
|
||||||
{"0::/user.slice/user-1000.slice/session-9.scope\n", 1001, ""}, // asked as another uid than the slice's
|
|
||||||
{"0::/user.slice/user-1000.slice/session-8.scope\n", 1000, ""}, // the record names uid 1001
|
|
||||||
{"0::/user.slice/user-1000.slice/session-3.scope\n", 1000, ""}, // a greeter's
|
|
||||||
{"0::/user.slice/user-1000.slice/session-5.scope\n", 1000, ""}, // a record naming another scope
|
|
||||||
{"0::/user.slice/user-1000.slice/session-7.scope\n", 1000, ""}, // no record at all
|
|
||||||
{"0::/user.slice/user-1000.slice/session-a.b.scope\n", 1000, ""},
|
|
||||||
{"0::/user.slice/user-1000.slice/session-...scope\n", 1000, ""},
|
|
||||||
{"0::/user.slice/user-1000.slice/session-11.scope\n", 1000, ""}, // first UID is 1001
|
|
||||||
{"0::/user.slice/user-1000.slice/session-12.scope\n", 1000, "session-12.scope"}, // first UID and CLASS stand
|
|
||||||
} {
|
|
||||||
if got := userSessionIn(c.cgroup, c.uid); got != c.session {
|
|
||||||
t.Errorf("%q as uid %d read as %q, want %q", c.cgroup, c.uid, got, c.session)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// **The session is read at accept, with the namespaces, before the line** (review of ADR 0272): the peer is judged as
|
|
||||||
// it was when it connected. The client sends its line only once the session has been read; were the session read
|
|
||||||
// after the line — in the forward, say — the line would never be sent and the test fails.
|
|
||||||
func TestTheSessionIsReadBeforeTheLine(t *testing.T) {
|
|
||||||
var mu sync.Mutex
|
|
||||||
var order []string
|
|
||||||
note := func(s string) { mu.Lock(); order = append(order, s); mu.Unlock() }
|
|
||||||
read := make(chan struct{}, 1)
|
|
||||||
beforeNS, beforeSession := sameNamespaces, sessionOf
|
|
||||||
sameNamespaces = func(int32) error { note("namespaces"); return nil }
|
|
||||||
sessionOf = func(int32, uint32) string { note("session"); read <- struct{}{}; return "session-1.scope" }
|
|
||||||
t.Cleanup(func() { sameNamespaces, sessionOf = beforeNS, beforeSession })
|
|
||||||
var got Asked
|
|
||||||
socket := serving(t, func(_ context.Context, asked Asked) cli.Answer {
|
|
||||||
note("forward")
|
|
||||||
got = asked
|
|
||||||
return cli.Answer{}
|
|
||||||
})
|
|
||||||
conn, err := net.Dial("unix", socket)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
defer conn.Close()
|
|
||||||
select {
|
|
||||||
case <-read:
|
|
||||||
case <-time.After(3 * time.Second):
|
|
||||||
t.Fatal("the session was not read before the line was sent")
|
|
||||||
}
|
|
||||||
note("line sent")
|
|
||||||
if err := cli.WriteRequest(conn, cli.Request{Line: []string{"status"}}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
|
|
||||||
if _, err := cli.ReadAnswer(conn); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if strings.Join(order, ",") != "namespaces,session,line sent,forward" || got.Session != "session-1.scope" {
|
|
||||||
t.Fatalf("read in the order %v, sent %q", order, got.Session)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// mesh-cli's standard input reaches the controller with its line, whole, and only with the line: a follow of the
|
|
||||||
// same call carries none, and the engine says it nowhere (novox/hq ADR 0259 §10, ADR 0272).
|
|
||||||
func TestStandardInputGoesToTheControllerWithTheLineAndNowhereElse(t *testing.T) {
|
|
||||||
secret := []byte("123456789:the-bot-token-given-at-the-terminal\n")
|
|
||||||
var said []string
|
|
||||||
var mu sync.Mutex
|
|
||||||
var asked [][]byte
|
|
||||||
ask := func(_ context.Context, body []byte) ([]byte, error) {
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
asked = append(asked, append([]byte(nil), body...))
|
|
||||||
if len(asked) == 1 {
|
|
||||||
return []byte(`{"result":{"running":true,"call":"call-7"}}`), nil
|
|
||||||
}
|
|
||||||
return []byte(`{"result":{"exit":0,"terminal":true}}`), nil
|
|
||||||
}
|
|
||||||
was := followEvery
|
|
||||||
followEvery = time.Millisecond
|
|
||||||
t.Cleanup(func() { followEvery = was })
|
|
||||||
socket := filepath.Join(t.TempDir(), "run", "cli.sock")
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
|
||||||
done := make(chan error, 1)
|
|
||||||
go func() {
|
|
||||||
done <- Serve(ctx, socket, func(ctx context.Context, a Asked) cli.Answer { return AskTheController(ctx, ask, a) },
|
|
||||||
func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() })
|
|
||||||
}()
|
|
||||||
defer func() { cancel(); <-done }()
|
|
||||||
for i := 0; i < 200; i++ {
|
|
||||||
if _, err := os.Stat(socket); err == nil {
|
|
||||||
break
|
|
||||||
}
|
|
||||||
time.Sleep(5 * time.Millisecond)
|
|
||||||
}
|
|
||||||
a, err := cli.AskWith(context.Background(), socket, cli.Request{Line: []string{"secret", "accept", "anchor",
|
|
||||||
"telegram", "telegram-token", "--from", "-"}, Stdin: secret})
|
|
||||||
if err != nil || a.Exit != 0 {
|
|
||||||
t.Fatalf("%v %+v", err, a)
|
|
||||||
}
|
|
||||||
mu.Lock()
|
|
||||||
defer mu.Unlock()
|
|
||||||
if len(asked) != 2 {
|
|
||||||
t.Fatalf("asked %d times, want the line and one follow", len(asked))
|
|
||||||
}
|
|
||||||
var first, follow Asked
|
|
||||||
_ = json.Unmarshal(asked[0], &first)
|
|
||||||
_ = json.Unmarshal(asked[1], &follow)
|
|
||||||
if string(first.Stdin) != string(secret) {
|
|
||||||
t.Fatalf("the controller was not given the standard input whole: %q", first.Stdin)
|
|
||||||
}
|
|
||||||
if len(follow.Stdin) != 0 || strings.Contains(string(asked[1]), "stdin") {
|
|
||||||
t.Fatalf("a follow carried the standard input again: %s", asked[1])
|
|
||||||
}
|
|
||||||
for _, s := range said {
|
|
||||||
if strings.Contains(s, "bot-token") {
|
|
||||||
t.Fatalf("the engine said the standard input: %s", s)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,19 +0,0 @@
|
|||||||
package meshcli
|
|
||||||
|
|
||||||
import "net"
|
|
||||||
|
|
||||||
// The kernel's word on who is at the other end of a socket, shared with the node-engine's other local socket, the
|
|
||||||
// write of a filled file (internal/filling, novox/hq ADR 0349 §4): one reading of it, so the two never judge a peer
|
|
||||||
// differently.
|
|
||||||
|
|
||||||
// PeerCredentials is the user id and process id of the process at the other end, as the kernel recorded them when
|
|
||||||
// it connected (SO_PEERCRED). Nothing the process says is read.
|
|
||||||
func PeerCredentials(conn *net.UnixConn) (uint32, int32, error) { return peerCredentials(conn) }
|
|
||||||
|
|
||||||
// SameNamespaces refuses a process whose mount or user namespace is not the engine's: a user id is this machine's
|
|
||||||
// account only in this machine's namespaces.
|
|
||||||
func SameNamespaces(pid int32) error { return namespacesOf(pid) }
|
|
||||||
|
|
||||||
// NotOwnDirectory says what is wrong with a socket's directory, or nothing: it must be a directory, not a link,
|
|
||||||
// owned by the account the engine runs as.
|
|
||||||
func NotOwnDirectory(dir string) string { return notOwnDirectory(dir) }
|
|
||||||
@@ -1,319 +0,0 @@
|
|||||||
// Package ownresync hands the vault the value of this machine's own secret that runs, where the vault holds another
|
|
||||||
// (novox/hq issue 1126, cause C4).
|
|
||||||
//
|
|
||||||
// The vault's `move-in` never replaces a value it holds, so its first import of an own secret stays fixed; where the
|
|
||||||
// controller later made that secret anew (a rotation at start, a re-issue), the new value runs and the vault keeps the
|
|
||||||
// old one. The node's shadow composer sees it: composing from the vault's value, it opens both seals with this
|
|
||||||
// machine's key and its answer's shown body says delivery.MaskedAnotherValue. After each shadow answer this step asks the
|
|
||||||
// vault's `move-in` with replace_running: true and what runs, for each own secret so masked, once per running value, and
|
|
||||||
// says what it asked and what the vault answered. It is move-in's, the verb this node may ask already, so no seat and no
|
|
||||||
// bus grant changes for it.
|
|
||||||
//
|
|
||||||
// **Only while this node applies the controller's compositions**, and only for a module whose composition that runs is
|
|
||||||
// the one its want names: then the controller's value is what runs, and what runs is the truth the vault lacks. Once the
|
|
||||||
// node composes its own, the vault's value is the source and nothing is handed. **The switch** is the vault's
|
|
||||||
// (`own_take_switch`, a person's act): while it is off the vault refuses every ask, and this step says so once per value.
|
|
||||||
// The vault also refuses any value it changed in the last 15 minutes, so a re-issue the node has not applied yet is
|
|
||||||
// never put back.
|
|
||||||
package ownresync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"sort"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/catalogue"
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/selfcompose"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Verb is the vault's verb this step asks, with ReplaceRunning true in both steps.
|
|
||||||
const Verb = "move-in"
|
|
||||||
|
|
||||||
// ReplaceRunning is move-in's argument that makes it take the running value over the one the vault holds.
|
|
||||||
const ReplaceRunning = "replace_running"
|
|
||||||
|
|
||||||
// Env is what the step reads and asks.
|
|
||||||
type Env struct {
|
|
||||||
Node string
|
|
||||||
// Composes is whose compositions the node applies now: delivery.ComposesController, delivery.ComposesOwn, or ""
|
|
||||||
// while its agent has not decided. The step hands nothing but for ComposesController.
|
|
||||||
Composes func() string
|
|
||||||
// Running is what the node runs now (agent.Everything).
|
|
||||||
Running func() []delivery.Composed
|
|
||||||
// Get reads one key's latest value; nil and no error when there is none.
|
|
||||||
Get func(ctx context.Context, bucket, key string) ([]byte, error)
|
|
||||||
// Unseal opens a value sealed to this machine; Seal seals a value to a raw X25519 key in base64 (identity.Seal).
|
|
||||||
Unseal func(sealed string) ([]byte, error)
|
|
||||||
Seal func(to string, value []byte) (string, error)
|
|
||||||
// Ask asks one of the vault's verbs in this engine's own name (link.Session.AskVault).
|
|
||||||
Ask func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error)
|
|
||||||
Say func(string)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Outcome is what the step did for one own secret: asked or not, and what was said.
|
|
||||||
type Outcome struct {
|
|
||||||
Module string
|
|
||||||
Name string
|
|
||||||
Asked bool
|
|
||||||
Said string
|
|
||||||
// Retry says a read or an ask the bus did not answer stopped it: no answer, kept as none, and the step is to be
|
|
||||||
// taken again after a bound (issue 1126, 2026-10-28: "want.shanks.brother could not be read ... no responders").
|
|
||||||
Retry bool
|
|
||||||
}
|
|
||||||
|
|
||||||
// Resync is the step of one node agent's run.
|
|
||||||
type Resync struct {
|
|
||||||
env Env
|
|
||||||
|
|
||||||
mu sync.Mutex
|
|
||||||
// done is, by module/name, the running value's fingerprint the vault was asked with or why it was not: each is
|
|
||||||
// asked, and each reason said, once.
|
|
||||||
done map[string]string
|
|
||||||
}
|
|
||||||
|
|
||||||
// New is the step for one node.
|
|
||||||
func New(env Env) *Resync {
|
|
||||||
if env.Say == nil {
|
|
||||||
env.Say = func(string) {}
|
|
||||||
}
|
|
||||||
return &Resync{env: env, done: map[string]string{}}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Take reads the node's shadow answers and hands the vault what runs of each own secret they say another value of.
|
|
||||||
func (r *Resync) Take(ctx context.Context, answers map[string]delivery.CandidateAnswer) []Outcome {
|
|
||||||
seen := map[delivery.OwnValueDiffers]bool{}
|
|
||||||
var drift []delivery.OwnValueDiffers
|
|
||||||
proposals := make([]string, 0, len(answers))
|
|
||||||
for g := range answers {
|
|
||||||
proposals = append(proposals, g)
|
|
||||||
}
|
|
||||||
sort.Strings(proposals)
|
|
||||||
for _, g := range proposals {
|
|
||||||
for _, o := range delivery.OwnValuesDiffer(answers[g]) {
|
|
||||||
if !seen[o] {
|
|
||||||
seen[o] = true
|
|
||||||
drift = append(drift, o)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if len(drift) == 0 {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
composes := ""
|
|
||||||
if r.env.Composes != nil {
|
|
||||||
composes = r.env.Composes()
|
|
||||||
}
|
|
||||||
var out []Outcome
|
|
||||||
for _, o := range drift {
|
|
||||||
res := r.one(ctx, composes, o)
|
|
||||||
if res.Said != "" || res.Retry {
|
|
||||||
out = append(out, res)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return out
|
|
||||||
}
|
|
||||||
|
|
||||||
// one is the step for one own secret: Said is empty when it was done before for the same value and reason.
|
|
||||||
func (r *Resync) one(ctx context.Context, composes string, o delivery.OwnValueDiffers) Outcome {
|
|
||||||
res := Outcome{Module: o.Module, Name: o.Name}
|
|
||||||
key := o.Module + "/" + o.Name
|
|
||||||
if composes == "" {
|
|
||||||
return res // the agent has not decided whose compositions it applies: taken again once it has
|
|
||||||
}
|
|
||||||
if composes != delivery.ComposesController {
|
|
||||||
return r.said(key, "not:composes", res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
|
||||||
"nothing is handed: this node composes its own (%q), so the vault's value is the source", key, composes))
|
|
||||||
}
|
|
||||||
sealed, why, failed := r.running(ctx, o)
|
|
||||||
if failed != nil {
|
|
||||||
// The bus did not answer: no reason to keep, and taken again (Retry); said once per reason.
|
|
||||||
res.Retry = true
|
|
||||||
return r.said(key, "retry:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
|
||||||
"nothing is handed yet: %s; asked again", key, why))
|
|
||||||
}
|
|
||||||
if why != "" {
|
|
||||||
return r.said(key, "not:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
|
|
||||||
"nothing is handed: %s", key, why))
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256([]byte(sealed))
|
|
||||||
fp := hex.EncodeToString(sum[:])
|
|
||||||
r.mu.Lock()
|
|
||||||
asked := r.done[key] == fp
|
|
||||||
r.mu.Unlock()
|
|
||||||
if asked {
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
value, err := r.env.Unseal(sealed)
|
|
||||||
if err != nil {
|
|
||||||
return r.said(key, "not:unseal:"+fp, res, fmt.Sprintf("%s: what runs does not open with this machine's key, so "+
|
|
||||||
"nothing is handed: %v", key, err))
|
|
||||||
}
|
|
||||||
defer wipe(value)
|
|
||||||
said, err := r.hand(ctx, o, value)
|
|
||||||
if err != nil {
|
|
||||||
if ctx.Err() != nil {
|
|
||||||
return res // the run ended: asked again in the next
|
|
||||||
}
|
|
||||||
if selfcompose.BusFailed(err) {
|
|
||||||
// The vault did not answer on the bus: no word of the vault's, so not kept as asked (Retry).
|
|
||||||
res.Retry = true
|
|
||||||
return r.said(key, "retry:ask", res, fmt.Sprintf("%s: the vault's %s with replace_running did not answer, "+
|
|
||||||
"and is asked again: %v", key, Verb, err))
|
|
||||||
}
|
|
||||||
said = fmt.Sprintf("%s: the vault's %s with replace_running refused: %v", key, Verb, err)
|
|
||||||
}
|
|
||||||
// Once per running value, whatever the vault answered: a refusal is the vault's word, said once, and a new value
|
|
||||||
// that runs asks again.
|
|
||||||
r.mu.Lock()
|
|
||||||
r.done[key] = fp
|
|
||||||
r.mu.Unlock()
|
|
||||||
res.Asked, res.Said = true, said
|
|
||||||
r.env.Say(said)
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
// said says why once per reason, and keeps the reason as done.
|
|
||||||
func (r *Resync) said(key, reason string, res Outcome, why string) Outcome {
|
|
||||||
r.mu.Lock()
|
|
||||||
again := r.done[key] == reason
|
|
||||||
r.done[key] = reason
|
|
||||||
r.mu.Unlock()
|
|
||||||
if again {
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
res.Said = why
|
|
||||||
r.env.Say(why)
|
|
||||||
return res
|
|
||||||
}
|
|
||||||
|
|
||||||
// running is the sealed value of o in the composition that runs, when that composition is the one the module's want
|
|
||||||
// names; or why it is not handed, with the error of a read the bus did not answer, which is no answer.
|
|
||||||
func (r *Resync) running(ctx context.Context, o delivery.OwnValueDiffers) (sealed, why string, failed error) {
|
|
||||||
var comp *delivery.Composed
|
|
||||||
if r.env.Running != nil {
|
|
||||||
for _, c := range r.env.Running() {
|
|
||||||
if c.Module == o.Module {
|
|
||||||
c := c
|
|
||||||
comp = &c
|
|
||||||
break
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if comp == nil {
|
|
||||||
return "", o.Module + " runs no composition here", nil
|
|
||||||
}
|
|
||||||
wantKey, err := delivery.WantKey(r.env.Node, o.Module)
|
|
||||||
if err != nil {
|
|
||||||
return "", err.Error(), nil
|
|
||||||
}
|
|
||||||
raw, err := r.env.Get(ctx, delivery.AssignerWantsBucket, wantKey)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("its want %s could not be read (%v)", wantKey, err), err
|
|
||||||
}
|
|
||||||
if raw == nil {
|
|
||||||
return "", fmt.Sprintf("it has no want %s", wantKey), nil
|
|
||||||
}
|
|
||||||
want, err := delivery.DecodeWant(raw)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("its want %s is not one: %v", wantKey, err), nil
|
|
||||||
}
|
|
||||||
if want.Self || want.Removed || want.Composed == "" {
|
|
||||||
return "", fmt.Sprintf("its want %s names no composition of the controller's", wantKey), nil
|
|
||||||
}
|
|
||||||
// What runs is the composition the want names, as the module's have says: applied and running.
|
|
||||||
haveKey, err := delivery.HaveKey(r.env.Node, o.Module)
|
|
||||||
if err != nil {
|
|
||||||
return "", err.Error(), nil
|
|
||||||
}
|
|
||||||
raw, err = r.env.Get(ctx, delivery.HaveBucket, haveKey)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("its have %s could not be read (%v)", haveKey, err), err
|
|
||||||
}
|
|
||||||
if raw == nil {
|
|
||||||
return "", fmt.Sprintf("it has no have %s", haveKey), nil
|
|
||||||
}
|
|
||||||
have, err := delivery.DecodeHave(raw)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Sprintf("its have %s is not one: %v", haveKey, err), nil
|
|
||||||
}
|
|
||||||
if have.Composed != want.Composed || have.Phase != delivery.PhaseRunning {
|
|
||||||
return "", fmt.Sprintf("what runs (%s, %s) is not yet the composition its want names (%s)", have.Composed,
|
|
||||||
have.Phase, want.Composed), nil
|
|
||||||
}
|
|
||||||
// On a node the resource's id carries its module's name (`only-office.needs-jwt`); delivery.OwnNeedName reads it.
|
|
||||||
id := o.Module + "." + catalogue.NeedID(o.Name)
|
|
||||||
for _, res := range comp.Resources {
|
|
||||||
var body struct {
|
|
||||||
ID string `json:"id"`
|
|
||||||
Sealed any `json:"sealed"`
|
|
||||||
}
|
|
||||||
if json.Unmarshal(res, &body) != nil {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if name, own := delivery.OwnNeedName(o.Module, body.ID); !own || name != o.Name {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if s, ok := body.Sealed.(string); ok && s != "" {
|
|
||||||
return s, "", nil
|
|
||||||
}
|
|
||||||
return "", fmt.Sprintf("its resource %s carries no value sealed to this machine", id), nil
|
|
||||||
}
|
|
||||||
return "", fmt.Sprintf("what runs holds no resource %s", id), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// hand asks the vault's two steps with value, and checks what the vault holds after by its fingerprint.
|
|
||||||
func (r *Resync) hand(ctx context.Context, o delivery.OwnValueDiffers, value []byte) (string, error) {
|
|
||||||
raw, err := r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "module": o.Module, "name": o.Name,
|
|
||||||
"why": "this node's shadow answer says the vault holds another value than runs here"})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
var opened struct {
|
|
||||||
Call string `json:"call"`
|
|
||||||
SealTo string `json:"seal_to"`
|
|
||||||
Secret string `json:"secret"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
|
|
||||||
return "", fmt.Errorf("the vault answered no call and no key to seal to")
|
|
||||||
}
|
|
||||||
sealed, err := r.env.Seal(opened.SealTo, value)
|
|
||||||
if err != nil {
|
|
||||||
return "", fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
|
|
||||||
}
|
|
||||||
raw, err = r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "call": opened.Call, "sealed": sealed})
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
var kept struct {
|
|
||||||
Secret string `json:"secret"`
|
|
||||||
From int `json:"from_generation"`
|
|
||||||
To int `json:"to_generation"`
|
|
||||||
Unchanged bool `json:"unchanged"`
|
|
||||||
Fingerprint string `json:"fingerprint"`
|
|
||||||
Said string `json:"said"`
|
|
||||||
}
|
|
||||||
if err := json.Unmarshal(raw, &kept); err != nil || kept.Secret == "" {
|
|
||||||
return "", fmt.Errorf("the vault answered %s with no secret", opened.Secret)
|
|
||||||
}
|
|
||||||
sum := sha256.Sum256(value)
|
|
||||||
if kept.Fingerprint != "sha256:"+hex.EncodeToString(sum[:]) {
|
|
||||||
return "", fmt.Errorf("the vault says it holds %s at generation %d, but not the value handed in", kept.Secret, kept.To)
|
|
||||||
}
|
|
||||||
if kept.Unchanged {
|
|
||||||
return fmt.Sprintf("handed the vault what runs of %s: it held that value already (generation %d)", kept.Secret, kept.To), nil
|
|
||||||
}
|
|
||||||
return fmt.Sprintf("handed the vault what runs of %s: generation %d replaced by %d, the value that runs", kept.Secret,
|
|
||||||
kept.From, kept.To), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func wipe(b []byte) {
|
|
||||||
for i := range b {
|
|
||||||
b[i] = 0
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,117 +0,0 @@
|
|||||||
package ownresync
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/sha256"
|
|
||||||
"encoding/hex"
|
|
||||||
"encoding/json"
|
|
||||||
"errors"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"git.novox.be/novox/mesh-sdk/go/delivery"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Issue 1126, cause C4: after a shadow answer saying the vault holds another value of an own secret than runs, the node
|
|
||||||
// hands the vault what runs, in the verb's two steps, once per running value; and only while it applies the
|
|
||||||
// controller's compositions and runs the one its want names.
|
|
||||||
func TestTheNodeHandsTheVaultWhatRunsOfADriftedOwnSecret(t *testing.T) {
|
|
||||||
const running = "the-value-that-runs"
|
|
||||||
digest := "sha256:" + strings.Repeat("a", 64)
|
|
||||||
fp := "src2:" + strings.Repeat("b", 64)
|
|
||||||
sum := sha256.Sum256([]byte(running))
|
|
||||||
answers := map[string]delivery.CandidateAnswer{"g": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateDiffers,
|
|
||||||
Differs: []delivery.ModuleDiffers{{Module: "only-office", Why: "the resource differs", Bodies: []delivery.ResourceBodies{
|
|
||||||
{ID: "only-office.needs-jwt", Composed: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`, Running: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`},
|
|
||||||
{ID: "only-office.needs-salt", Composed: `{"sealed":"<masked: another seal of the same value>"}`}}}}}}
|
|
||||||
for name, row := range map[string]struct {
|
|
||||||
composes string
|
|
||||||
runs string // the have's composition digest
|
|
||||||
refuse string // the vault's refusal
|
|
||||||
asks int // the vault's verb asked, both passes
|
|
||||||
says string
|
|
||||||
// Issue 1126, 2026-10-28: a read or an ask the bus did not answer is no answer: taken again (Retry), never kept.
|
|
||||||
wantRead string // the want's read: "" answers it, "fails" the bus does not answer, "absent" no key
|
|
||||||
retry bool
|
|
||||||
}{
|
|
||||||
"applies the controller's, runs its want": {delivery.ComposesController, digest, "", 2, "generation 1 replaced by 2", "", false},
|
|
||||||
"composes its own": {delivery.ComposesOwn, digest, "", 0, "composes its own", "", false},
|
|
||||||
"not decided yet": {"", digest, "", 0, "", "", false},
|
|
||||||
"runs not yet what its want names": {delivery.ComposesController, "sha256:" + strings.Repeat("c", 64), "", 0, "not yet the composition", "", false},
|
|
||||||
"the vault refuses": {delivery.ComposesController, digest, "switched off", 1, "switched off", "", false},
|
|
||||||
"the vault does not answer": {delivery.ComposesController, digest, "nats: no responders available for request", 2, "asked again", "", true},
|
|
||||||
"the want's read does not answer": {delivery.ComposesController, digest, "", 0, "asked again", "fails", true},
|
|
||||||
"it has no want": {delivery.ComposesController, digest, "", 0, "it has no want", "absent", false},
|
|
||||||
} {
|
|
||||||
var asked []map[string]any
|
|
||||||
var said []string
|
|
||||||
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest})
|
|
||||||
have, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: row.runs, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
|
|
||||||
r := New(Env{Node: "novox", Composes: func() string { return row.composes },
|
|
||||||
Running: func() []delivery.Composed {
|
|
||||||
return []delivery.Composed{{Node: "novox", Module: "only-office", FP: fp, Resources: []json.RawMessage{
|
|
||||||
// As novox runs it (its shadow answer, 2026-10-28): the id carries the module's name.
|
|
||||||
json.RawMessage(`{"id":"only-office.needs-jwt","module":"only-office","type":"file",` +
|
|
||||||
`"path":"/var/lib/only-office/jwt.secret","sealed":"box-of-the-running-value"}`)}}}
|
|
||||||
},
|
|
||||||
Get: func(_ context.Context, bucket, key string) ([]byte, error) {
|
|
||||||
switch {
|
|
||||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "fails":
|
|
||||||
return nil, errors.New("the bucket mesh-delivery-assigner_wants: nats: no responders available for request")
|
|
||||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "absent":
|
|
||||||
return nil, nil
|
|
||||||
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office":
|
|
||||||
return want, nil
|
|
||||||
case bucket == delivery.HaveBucket && key == "novox.only-office":
|
|
||||||
return have, nil
|
|
||||||
}
|
|
||||||
return nil, nil
|
|
||||||
},
|
|
||||||
Unseal: func(sealed string) ([]byte, error) {
|
|
||||||
if sealed != "box-of-the-running-value" {
|
|
||||||
return nil, errors.New("not this machine's")
|
|
||||||
}
|
|
||||||
return []byte(running), nil
|
|
||||||
},
|
|
||||||
Seal: func(to string, value []byte) (string, error) { return to + "(" + string(value) + ")", nil },
|
|
||||||
Ask: func(_ context.Context, verb string, args map[string]any) (json.RawMessage, error) {
|
|
||||||
asked = append(asked, args)
|
|
||||||
if verb != "move-in" || args["replace_running"] != true {
|
|
||||||
t.Errorf("%s: asked %s with %v, not move-in with replace_running", name, verb, args)
|
|
||||||
}
|
|
||||||
if row.refuse != "" {
|
|
||||||
return nil, errors.New(row.refuse)
|
|
||||||
}
|
|
||||||
if args["call"] == nil {
|
|
||||||
if args["module"] != "only-office" || args["name"] != "jwt" || args["node"] != nil {
|
|
||||||
t.Errorf("%s: the first step names %v", name, args)
|
|
||||||
}
|
|
||||||
return json.RawMessage(`{"call":"c1","seal_to":"key","secret":"own/novox/only-office/jwt"}`), nil
|
|
||||||
}
|
|
||||||
if args["sealed"] != "key("+running+")" {
|
|
||||||
t.Errorf("%s: handed %v, not what runs sealed to the call's key", name, args["sealed"])
|
|
||||||
}
|
|
||||||
return json.RawMessage(`{"secret":"own/novox/only-office/jwt","from_generation":1,"to_generation":2,` +
|
|
||||||
`"fingerprint":"sha256:` + hex.EncodeToString(sum[:]) + `"}`), nil
|
|
||||||
},
|
|
||||||
Say: func(s string) { said = append(said, s) }})
|
|
||||||
first := r.Take(context.Background(), answers)
|
|
||||||
again := r.Take(context.Background(), answers)
|
|
||||||
if len(asked) != row.asks {
|
|
||||||
t.Errorf("%s: the vault asked %d times, want %d: %v", name, len(asked), row.asks, asked)
|
|
||||||
}
|
|
||||||
if retry := len(first) == 1 && first[0].Retry; retry != row.retry {
|
|
||||||
t.Errorf("%s: taken again %v, want %v: %+v", name, retry, row.retry, first)
|
|
||||||
}
|
|
||||||
if row.retry && (len(again) != 1 || !again[0].Retry || again[0].Said != "") || !row.retry && len(again) != 0 {
|
|
||||||
t.Errorf("%s: the same answer did or said something again: %+v", name, again)
|
|
||||||
}
|
|
||||||
all := strings.Join(said, "\n")
|
|
||||||
if row.says == "" && (len(first) != 0 || all != "") || row.says != "" && (len(first) != 1 || !strings.Contains(all, row.says)) {
|
|
||||||
t.Errorf("%s: did %+v and said %q, want %q", name, first, all, row.says)
|
|
||||||
}
|
|
||||||
if strings.Contains(all, running) {
|
|
||||||
t.Errorf("%s: the value is in what the node said", name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -15,10 +15,10 @@ import (
|
|||||||
// **The packages this node's compositions declare, kept by its engine** (novox/hq ADR 0565 §6, §15; issue 1953). The
|
// **The packages this node's compositions declare, kept by its engine** (novox/hq ADR 0565 §6, §15; issue 1953). The
|
||||||
// composer keeps the package resources of each composition it handed, and after a pass that handed something new (or
|
// composer keeps the package resources of each composition it handed, and after a pass that handed something new (or
|
||||||
// the run's first) it works out the node's key of the engine's state packages from them (delivery.DeclaredPackagesOf,
|
// the run's first) it works out the node's key of the engine's state packages from them (delivery.DeclaredPackagesOf,
|
||||||
// the controller mirror's layout and order) and writes it only when the value differs from what the key holds (ADR 0289
|
// in the layout and order the controller's mirror had) and writes it only when the value differs from what the key holds (ADR 0289
|
||||||
// §1): no timer, and no composing of another node. A module held keeps the packages of what was handed of it, as it keeps
|
// §1): no timer, and no composing of another node. A module held keeps the packages of what was handed of it, as it keeps
|
||||||
// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it beside the
|
// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it, and with
|
||||||
// controller's mirror until the lease write, and with write writes it again (ADR 0565 §15).
|
// write writes it again (ADR 0565 §15).
|
||||||
|
|
||||||
// PackagesStore is this node's key of the engine's state packages (link.EnginePackages).
|
// PackagesStore is this node's key of the engine's state packages (link.EnginePackages).
|
||||||
type PackagesStore interface {
|
type PackagesStore interface {
|
||||||
|
|||||||
@@ -1057,7 +1057,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
go func() { sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond }); close(kept) }()
|
go func() { sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond }); close(kept) }()
|
||||||
time.Sleep(30 * time.Millisecond)
|
time.Sleep(30 * time.Millisecond)
|
||||||
sb.missing(delivery.PlacementCandidatesBucket, false)
|
sb.missing(delivery.PlacementCandidatesBucket, false)
|
||||||
sb.made(7)
|
sb.made(6)
|
||||||
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
|
||||||
tp, _ := delivery.EncodePlacement(trunk)
|
tp, _ := delivery.EncodePlacement(trunk)
|
||||||
sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
||||||
@@ -1321,7 +1321,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
nctx, ncancel := context.WithCancel(ctx)
|
nctx, ncancel := context.WithCancel(ctx)
|
||||||
defer ncancel()
|
defer ncancel()
|
||||||
go func() { _ = nsh.Run(nctx) }()
|
go func() { _ = nsh.Run(nctx) }()
|
||||||
nb.made(7)
|
nb.made(6)
|
||||||
nb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
nb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
|
||||||
nb.put(delivery.HaveBucket, "n1.node", rec, 1)
|
nb.put(delivery.HaveBucket, "n1.node", rec, 1)
|
||||||
for name, m := range mods {
|
for name, m := range mods {
|
||||||
@@ -1335,19 +1335,19 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk})
|
oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk})
|
||||||
oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
|
oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
|
||||||
nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
|
nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
|
||||||
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} {
|
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, delivery.BuilderBuildsBucket} {
|
||||||
nb.send(bucket, agent.Update{Caught: true})
|
nb.send(bucket, agent.Update{Caught: true})
|
||||||
}
|
}
|
||||||
select {
|
select {
|
||||||
case a := <-nanswered:
|
case a := <-nanswered:
|
||||||
if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, delivery.AssignerWantsBucket) || g.Proposal != "same" {
|
if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, ResolvedBucket) || g.Proposal != "same" {
|
||||||
t.Errorf("a shadow waiting for its wants answered %+v", g)
|
t.Errorf("a shadow waiting for its values answered %+v", g)
|
||||||
}
|
}
|
||||||
case <-time.After(5 * time.Second):
|
case <-time.After(5 * time.Second):
|
||||||
t.Fatal("a shadow not ready after its wait answered nothing")
|
t.Fatal("a shadow not ready after its wait answered nothing")
|
||||||
}
|
}
|
||||||
nmu.Lock()
|
nmu.Lock()
|
||||||
if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+delivery.AssignerWantsBucket) {
|
if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+ResolvedBucket) {
|
||||||
t.Errorf("a shadow not ready said %q", nsaid)
|
t.Errorf("a shadow not ready said %q", nsaid)
|
||||||
}
|
}
|
||||||
nmu.Unlock()
|
nmu.Unlock()
|
||||||
@@ -1355,7 +1355,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
|
|||||||
scancel()
|
scancel()
|
||||||
<-kept
|
<-kept
|
||||||
setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time
|
setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time
|
||||||
nb.send(delivery.AssignerWantsBucket, agent.Update{Caught: true})
|
nb.send(ResolvedBucket, agent.Update{Caught: true})
|
||||||
select {
|
select {
|
||||||
case a := <-nanswered:
|
case a := <-nanswered:
|
||||||
if g := a["same"]; g.Verdict != delivery.CandidateSame || !strings.Contains(g.Note, "this answer took over") {
|
if g := a["same"]; g.Verdict != delivery.CandidateSame || !strings.Contains(g.Note, "this answer took over") {
|
||||||
|
|||||||
@@ -32,8 +32,6 @@ type Shadow struct {
|
|||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
candidates map[string]map[string]candidateKey // by proposal, by node
|
candidates map[string]map[string]candidateKey // by proposal, by node
|
||||||
answers map[string]delivery.CandidateAnswer
|
answers map[string]delivery.CandidateAnswer
|
||||||
// wants are the node's wants by module (agent.NodeModule for its own); a change of one answers again.
|
|
||||||
wants map[string]delivery.Want
|
|
||||||
// ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy
|
// ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy
|
||||||
// from the declaration kept before the switch (issue 1126, probe #532 on shanks).
|
// from the declaration kept before the switch (issue 1126, probe #532 on shanks).
|
||||||
ran map[string]string
|
ran map[string]string
|
||||||
@@ -64,7 +62,7 @@ type answeredAt struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it.
|
// inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it.
|
||||||
var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves", "wants"}
|
var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves"}
|
||||||
|
|
||||||
// wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at):
|
// wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at):
|
||||||
// its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words.
|
// its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words.
|
||||||
@@ -99,7 +97,7 @@ type candidateKey struct {
|
|||||||
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
|
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
|
||||||
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
|
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
|
||||||
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
|
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
|
||||||
answers: map[string]delivery.CandidateAnswer{}, wants: map[string]delivery.Want{}, ran: map[string]string{}}
|
answers: map[string]delivery.CandidateAnswer{}, ran: map[string]string{}}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
|
// Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
|
||||||
@@ -141,7 +139,6 @@ func (s *Shadow) Run(ctx context.Context) error {
|
|||||||
defer cancel()
|
defer cancel()
|
||||||
s.mu.Lock()
|
s.mu.Lock()
|
||||||
s.candidates = map[string]map[string]candidateKey{}
|
s.candidates = map[string]map[string]candidateKey{}
|
||||||
s.wants = map[string]delivery.Want{}
|
|
||||||
s.ran = map[string]string{}
|
s.ran = map[string]string{}
|
||||||
s.answered = map[string]answeredAt{}
|
s.answered = map[string]answeredAt{}
|
||||||
s.moved = [len(inputWords)]uint64{}
|
s.moved = [len(inputWords)]uint64{}
|
||||||
@@ -159,8 +156,6 @@ func (s *Shadow) Run(ctx context.Context) error {
|
|||||||
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
|
||||||
// What this node runs moved: an apply ends in its module's have.
|
// What this node runs moved: an apply ends in its module's have.
|
||||||
{delivery.HaveBucket, node + ".>", nil, nil},
|
{delivery.HaveBucket, node + ".>", nil, nil},
|
||||||
// Its wants.
|
|
||||||
{delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil},
|
|
||||||
}
|
}
|
||||||
type tagged struct {
|
type tagged struct {
|
||||||
in int
|
in int
|
||||||
@@ -225,8 +220,6 @@ func (s *Shadow) Run(ctx context.Context) error {
|
|||||||
case t.in == 5:
|
case t.in == 5:
|
||||||
folded := c.takeModuleHave(t.u)
|
folded := c.takeModuleHave(t.u)
|
||||||
changed = s.takeRan(t.u) || folded
|
changed = s.takeRan(t.u) || folded
|
||||||
case t.in == 6:
|
|
||||||
changed = s.takeWant(t.u)
|
|
||||||
default:
|
default:
|
||||||
changed = c.takeChanged(ins[t.in], t.u)
|
changed = c.takeChanged(ins[t.in], t.u)
|
||||||
}
|
}
|
||||||
@@ -685,33 +678,6 @@ func sameAnswer(a, b delivery.CandidateAnswer) bool {
|
|||||||
return errA == nil && errB == nil && string(x) == string(y)
|
return errA == nil && errB == nil && string(x) == string(y)
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeWant takes one of the node's wants, and says whether it changed.
|
|
||||||
func (s *Shadow) takeWant(u agent.Update) bool {
|
|
||||||
module := ""
|
|
||||||
var w delivery.Want
|
|
||||||
if _, err := delivery.ParseWantNodeKey(u.Key); err == nil {
|
|
||||||
module = agent.NodeModule
|
|
||||||
} else if node, m, err := delivery.ParseWantKey(u.Key); err == nil && node == s.c.env.Node {
|
|
||||||
module = m
|
|
||||||
if !u.Deleted {
|
|
||||||
if w, err = delivery.DecodeWant(u.Value); err != nil || w.Removed || w.FP == "" {
|
|
||||||
u.Deleted = true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
s.mu.Lock()
|
|
||||||
defer s.mu.Unlock()
|
|
||||||
old, had := s.wants[module]
|
|
||||||
if u.Deleted {
|
|
||||||
delete(s.wants, module)
|
|
||||||
return had
|
|
||||||
}
|
|
||||||
s.wants[module] = w
|
|
||||||
return !had || old.FP != w.FP
|
|
||||||
}
|
|
||||||
|
|
||||||
// sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not
|
// sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not
|
||||||
// open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work).
|
// open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work).
|
||||||
func (c *Composer) sameValue(a, b string) (same, known bool) {
|
func (c *Composer) sameValue(a, b string) (same, known bool) {
|
||||||
|
|||||||
@@ -13,23 +13,26 @@
|
|||||||
// or is deleted is said, the key last heard stands, and the list loaded stays; with no key ever heard nothing is
|
// or is deleted is said, the key last heard stands, and the list loaded stays; with no key ever heard nothing is
|
||||||
// written. The list a composition carries is never written.
|
// written. The list a composition carries is never written.
|
||||||
//
|
//
|
||||||
// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition in the node's
|
// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition as this node runs
|
||||||
// composition bucket, verified as the agent verified it before anything of it is read: what tells the bus node it is
|
// it (Env.Running: the agent's own record of what runs, its own composition or the declaration kept before the switch),
|
||||||
// one, and the file's place. The have of that module says what was written, which list it follows and this node's key,
|
// never one read from a composition bucket: the controller that wrote `composed-<node>` ended (issue 1126), and the bus
|
||||||
// which the grants part seals to.
|
// module on the bus node runs from what the agent holds. That is what tells the bus node it is one, and the file's
|
||||||
|
// place. The have of that module says what was written, which list it follows and this node's key, which the grants
|
||||||
|
// part seals to.
|
||||||
//
|
//
|
||||||
// **It watches, it never polls**, and looks again after every apply of the bus module and when asked (Again). It reads no
|
// **It watches, it never polls**: the grants part's key is watched, and what runs is looked at again when the agent says
|
||||||
// want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list.
|
// it changed, after every apply of the bus module and when asked (Again). It reads no want and writes none, moves no
|
||||||
|
// fingerprint, and touches nothing of the bus module but its user list.
|
||||||
package userlist
|
package userlist
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
|
||||||
"crypto/sha256"
|
"crypto/sha256"
|
||||||
"encoding/hex"
|
"encoding/hex"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -49,10 +52,10 @@ type Written = delivery.UserListWritten
|
|||||||
|
|
||||||
// Env is everything the follower needs of the node.
|
// Env is everything the follower needs of the node.
|
||||||
type Env struct {
|
type Env struct {
|
||||||
Node string
|
Node string
|
||||||
Composed agent.Watcher
|
// Running is the compositions this node runs now, as its agent holds them (agent.Agent.Everything): where the user
|
||||||
// Trusted is the keys a composition may be signed by: one any other key signed declares no file.
|
// list's file is is read from these alone (issue 1126). Nil runs nothing.
|
||||||
Trusted func() []ed25519.PublicKey
|
Running func() []delivery.Composed
|
||||||
// OnDisk is what the resource's file holds on the machine now; nil when there is none.
|
// OnDisk is what the resource's file holds on the machine now; nil when there is none.
|
||||||
OnDisk func(resource json.RawMessage) ([]byte, error)
|
OnDisk func(resource json.RawMessage) ([]byte, error)
|
||||||
// Write writes the user list's file resource of a module's composition alone, with body as its content, when it
|
// Write writes the user list's file resource of a module's composition alone, with body as its content, when it
|
||||||
@@ -79,19 +82,18 @@ type Env struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type source struct {
|
type source struct {
|
||||||
key string
|
|
||||||
module string
|
module string
|
||||||
rev uint64
|
|
||||||
resource json.RawMessage
|
resource json.RawMessage
|
||||||
wanted string
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Follower follows the node's compositions for the bus's user list.
|
// Follower follows the grants part's user list into the file the bus module this node runs declares.
|
||||||
type Follower struct {
|
type Follower struct {
|
||||||
env Env
|
env Env
|
||||||
kick chan struct{}
|
kick chan struct{}
|
||||||
mu sync.Mutex
|
// looks asks a look at what runs (RunsChanged): written only when the file it names moved.
|
||||||
srcs map[string]source // by composition key
|
looks chan struct{}
|
||||||
|
mu sync.Mutex
|
||||||
|
srcs map[string]source // by module, of what runs now
|
||||||
// cur is the source last taken, and written whether its write went through.
|
// cur is the source last taken, and written whether its write went through.
|
||||||
cur source
|
cur source
|
||||||
written bool
|
written bool
|
||||||
@@ -116,7 +118,8 @@ func New(env Env) *Follower {
|
|||||||
if env.Backoff == nil {
|
if env.Backoff == nil {
|
||||||
env.Backoff = func(int) time.Duration { return time.Minute }
|
env.Backoff = func(int) time.Duration { return time.Minute }
|
||||||
}
|
}
|
||||||
f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
|
f := &Follower{env: env, kick: make(chan struct{}, 1), looks: make(chan struct{}, 1), srcs: map[string]source{},
|
||||||
|
said: map[string]bool{}}
|
||||||
if env.Grants != nil && env.Kept != nil {
|
if env.Grants != nil && env.Kept != nil {
|
||||||
if raw, rev := env.Kept(); raw != nil {
|
if raw, rev := env.Kept(); raw != nil {
|
||||||
if g, err := delivery.DecodeGrantsUserList(raw); err == nil {
|
if g, err := delivery.DecodeGrantsUserList(raw); err == nil {
|
||||||
@@ -127,8 +130,8 @@ func New(env Env) *Follower {
|
|||||||
return f
|
return f
|
||||||
}
|
}
|
||||||
|
|
||||||
// Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module,
|
// Again asks the follower to look at what runs and the file again: when what runs changed, after an apply of the bus
|
||||||
// and when a person asks. It never blocks.
|
// module, and when a person asks. It never blocks.
|
||||||
func (f *Follower) Again() {
|
func (f *Follower) Again() {
|
||||||
select {
|
select {
|
||||||
case f.kick <- struct{}{}:
|
case f.kick <- struct{}{}:
|
||||||
@@ -136,21 +139,25 @@ func (f *Follower) Again() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// State is the module whose list the follower writes ("" when no composition of this node carries one) and what it
|
// RunsChanged asks the follower to look at what this node runs again: the list is written again only when the file the
|
||||||
// last did.
|
// bus module declares moved. It never blocks.
|
||||||
|
func (f *Follower) RunsChanged() {
|
||||||
|
select {
|
||||||
|
case f.looks <- struct{}{}:
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// State is the module whose list the follower writes ("" when nothing this node runs carries one) and what it last did.
|
||||||
func (f *Follower) State() (string, Written) {
|
func (f *Follower) State() (string, Written) {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
return f.mod, f.state
|
return f.mod, f.state
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run follows the compositions until ctx ends.
|
// Run follows the grants part's list until ctx ends.
|
||||||
func (f *Follower) Run(ctx context.Context) error {
|
func (f *Follower) Run(ctx context.Context) error {
|
||||||
updates, err := f.env.Composed.Watch(ctx, f.env.Node+".>")
|
f.RunsChanged() // the first look at what runs, at once
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("the user list cannot be followed: %w", err)
|
|
||||||
}
|
|
||||||
caught := false
|
|
||||||
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
|
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
|
||||||
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
|
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
|
||||||
// never writes a list older than the key; a watch refused or ended is tried again on the backoff, and meanwhile the
|
// never writes a list older than the key; a watch refused or ended is tried again on the backoff, and meanwhile the
|
||||||
@@ -185,17 +192,11 @@ func (f *Follower) Run(ctx context.Context) error {
|
|||||||
return nil
|
return nil
|
||||||
case <-f.kick:
|
case <-f.kick:
|
||||||
again = true
|
again = true
|
||||||
|
f.look()
|
||||||
|
case <-f.looks:
|
||||||
|
f.look()
|
||||||
case <-retry:
|
case <-retry:
|
||||||
retry = nil
|
retry = nil
|
||||||
case u, ok := <-updates:
|
|
||||||
if !ok {
|
|
||||||
return fmt.Errorf("the watch of the compositions ended")
|
|
||||||
}
|
|
||||||
if u.Caught {
|
|
||||||
caught = true
|
|
||||||
} else {
|
|
||||||
f.take(u)
|
|
||||||
}
|
|
||||||
case u, ok := <-grants:
|
case u, ok := <-grants:
|
||||||
if !ok {
|
if !ok {
|
||||||
failed("its watch ended")
|
failed("its watch ended")
|
||||||
@@ -210,9 +211,6 @@ func (f *Follower) Run(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
f.takeGrants(u)
|
f.takeGrants(u)
|
||||||
}
|
}
|
||||||
if !caught {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
open()
|
open()
|
||||||
if opened && !grantsCaught && grants != nil {
|
if opened && !grantsCaught && grants != nil {
|
||||||
continue // the switch is not heard yet: nothing is written before it is
|
continue // the switch is not heard yet: nothing is written before it is
|
||||||
@@ -226,11 +224,37 @@ func (f *Follower) Run(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// busNode says some composition of this node carries the bus's user list.
|
// busNode says something this node runs carries the bus's user list, or did.
|
||||||
func (f *Follower) busNode() bool {
|
func (f *Follower) busNode() bool {
|
||||||
f.mu.Lock()
|
f.mu.Lock()
|
||||||
defer f.mu.Unlock()
|
defer f.mu.Unlock()
|
||||||
return f.cur.key != "" || len(f.srcs) > 0
|
return f.cur.module != "" || len(f.srcs) > 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// look reads what this node runs now for the bus's user list: each module whose composition carries it.
|
||||||
|
func (f *Follower) look() {
|
||||||
|
var running []delivery.Composed
|
||||||
|
if f.env.Running != nil {
|
||||||
|
running = f.env.Running()
|
||||||
|
}
|
||||||
|
srcs := map[string]source{}
|
||||||
|
for _, c := range running {
|
||||||
|
if c.Node != "" && c.Node != f.env.Node {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
raw, _, _, ok := userList(c.Resources)
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
module := c.Module
|
||||||
|
if module == "" {
|
||||||
|
module = agent.NodeModule
|
||||||
|
}
|
||||||
|
srcs[module] = source{module: module, resource: raw}
|
||||||
|
}
|
||||||
|
f.mu.Lock()
|
||||||
|
f.srcs = srcs
|
||||||
|
f.mu.Unlock()
|
||||||
}
|
}
|
||||||
|
|
||||||
// takeGrants keeps the grants part's key as heard. A key deleted, or a value it cannot read, is said and changes nothing:
|
// takeGrants keeps the grants part's key as heard. A key deleted, or a value it cannot read, is said and changes nothing:
|
||||||
@@ -272,51 +296,6 @@ func (f *Follower) key() string {
|
|||||||
return f.env.Key()
|
return f.env.Key()
|
||||||
}
|
}
|
||||||
|
|
||||||
// take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted.
|
|
||||||
func (f *Follower) take(u agent.Update) {
|
|
||||||
node, module, _, isNode, err := delivery.ParseComposedKey(u.Key)
|
|
||||||
if err != nil || node != f.env.Node || isNode {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
f.mu.Lock()
|
|
||||||
defer f.mu.Unlock()
|
|
||||||
if u.Deleted {
|
|
||||||
delete(f.srcs, u.Key)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
c, err := delivery.DecodeComposed(u.Value)
|
|
||||||
if err != nil {
|
|
||||||
delete(f.srcs, u.Key)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
var trusted []ed25519.PublicKey
|
|
||||||
if f.env.Trusted != nil {
|
|
||||||
trusted = f.env.Trusted()
|
|
||||||
}
|
|
||||||
if _, err := delivery.VerifyComposed(c, trusted); err != nil {
|
|
||||||
// The agent says it is refused; a refused composition is never a source of the list.
|
|
||||||
delete(f.srcs, u.Key)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.Node != f.env.Node || c.Module != module {
|
|
||||||
// Signed, and for another node or module than its key says: a re-put of another composition is no source.
|
|
||||||
delete(f.srcs, u.Key)
|
|
||||||
f.sayOnceLocked(fmt.Sprintf("the composition under %s is %s's of %s, and is no source of the bus's user list",
|
|
||||||
u.Key, c.Node, c.Module))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
raw, content, digest, ok := userList(c.Resources)
|
|
||||||
if !ok {
|
|
||||||
delete(f.srcs, u.Key)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
wanted := digest
|
|
||||||
if wanted == "" {
|
|
||||||
wanted = Digest([]byte(content))
|
|
||||||
}
|
|
||||||
f.srcs[u.Key] = source{key: u.Key, module: module, rev: u.Revision, resource: raw, wanted: wanted}
|
|
||||||
}
|
|
||||||
|
|
||||||
// settle writes the grants part's list: when its key moved, or when asked.
|
// settle writes the grants part's list: when its key moved, or when asked.
|
||||||
func (f *Follower) settle(ctx context.Context, again bool) { f.settleGrants(ctx, again) }
|
func (f *Follower) settle(ctx context.Context, again bool) { f.settleGrants(ctx, again) }
|
||||||
|
|
||||||
@@ -337,18 +316,26 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) {
|
|||||||
g = *f.grants
|
g = *f.grants
|
||||||
}
|
}
|
||||||
rev := f.grantsRev
|
rev := f.grantsRev
|
||||||
|
// What runs now: the module written before while it still carries the list, else the first by name; what was
|
||||||
|
// written before when nothing runs that carries it now (a look between two applies).
|
||||||
file := f.cur
|
file := f.cur
|
||||||
for _, s := range f.srcs {
|
if s, ok := f.srcs[f.cur.module]; ok {
|
||||||
if s.rev > file.rev {
|
file = s
|
||||||
file = s
|
} else if len(f.srcs) > 0 {
|
||||||
|
names := make([]string, 0, len(f.srcs))
|
||||||
|
for m := range f.srcs {
|
||||||
|
names = append(names, m)
|
||||||
}
|
}
|
||||||
|
sort.Strings(names)
|
||||||
|
file = f.srcs[names[0]]
|
||||||
}
|
}
|
||||||
if file.key == "" {
|
if file.module == "" {
|
||||||
f.mu.Unlock()
|
f.mu.Unlock()
|
||||||
return // no composition of this node declares the file: not the bus node
|
return // nothing this node runs declares the file: not the bus node
|
||||||
}
|
}
|
||||||
|
moved := file.module != f.cur.module || pathOf(file.resource) != pathOf(f.cur.resource)
|
||||||
f.cur = file
|
f.cur = file
|
||||||
if !again && f.state.Source != "" && f.state.Revision == rev && f.written {
|
if !again && !moved && f.state.Source != "" && f.state.Revision == rev && f.written {
|
||||||
f.mu.Unlock()
|
f.mu.Unlock()
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -522,6 +509,15 @@ func userList(resources []json.RawMessage) (json.RawMessage, string, string, boo
|
|||||||
return nil, "", "", false
|
return nil, "", "", false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// pathOf is where a file resource says its file is; "" when it says none.
|
||||||
|
func pathOf(raw json.RawMessage) string {
|
||||||
|
var head struct {
|
||||||
|
Path string `json:"path"`
|
||||||
|
}
|
||||||
|
_ = json.Unmarshal(raw, &head)
|
||||||
|
return head.Path
|
||||||
|
}
|
||||||
|
|
||||||
// Digest is content's digest as the have says it, `sha256:<hex>`.
|
// Digest is content's digest as the have says it, `sha256:<hex>`.
|
||||||
func Digest(content []byte) string {
|
func Digest(content []byte) string {
|
||||||
sum := sha256.Sum256(content)
|
sum := sha256.Sum256(content)
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ package userlist
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"crypto/ed25519"
|
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"strings"
|
"strings"
|
||||||
@@ -17,17 +16,6 @@ import (
|
|||||||
|
|
||||||
const node = "novox"
|
const node = "novox"
|
||||||
|
|
||||||
func key() (ed25519.PublicKey, ed25519.PrivateKey) {
|
|
||||||
pub, k, err := ed25519.GenerateKey(nil)
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
return pub, k
|
|
||||||
}
|
|
||||||
|
|
||||||
var controllerPub, controllerKey = key()
|
|
||||||
var _, strangerKey = key()
|
|
||||||
|
|
||||||
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
|
||||||
|
|
||||||
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
|
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
|
||||||
@@ -69,19 +57,14 @@ func list(users ...string) string {
|
|||||||
return b.String()
|
return b.String()
|
||||||
}
|
}
|
||||||
|
|
||||||
// comp is a composition of module at fp holding the user list content (none when empty), signed by signer; by digest
|
// comp is a composition of module at fp as this node runs it, holding the user list content (none when empty); by
|
||||||
// when byDigest, as the controller sends a file over 64 KiB; for another node or module than its key when asked.
|
// digest when byDigest, as a file over 64 KiB is given; for another node than this one when asked.
|
||||||
func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p put) (string, []byte) {
|
func comp(module, fp, content string, p put) delivery.Composed {
|
||||||
t.Helper()
|
server, _ := json.Marshal(map[string]any{"id": module + ".server-conf", "type": "file", "module": module,
|
||||||
owner := module
|
|
||||||
if p.asModule != "" {
|
|
||||||
owner = p.asModule
|
|
||||||
}
|
|
||||||
server, _ := json.Marshal(map[string]any{"id": owner + ".server-conf", "type": "file", "module": owner,
|
|
||||||
"path": "/var/lib/nats-module/conf/nats.conf", "content": "include accounts.conf\n"})
|
"path": "/var/lib/nats-module/conf/nats.conf", "content": "include accounts.conf\n"})
|
||||||
rs := []json.RawMessage{server}
|
rs := []json.RawMessage{server}
|
||||||
if content != "" {
|
if content != "" {
|
||||||
r := map[string]any{"id": owner + ".bus-users", "type": "file", "module": owner,
|
r := map[string]any{"id": module + ".bus-users", "type": "file", "module": module,
|
||||||
"path": "/var/lib/nats-module/conf/accounts.conf", "content": content, "mode": "0600"}
|
"path": "/var/lib/nats-module/conf/accounts.conf", "content": content, "mode": "0600"}
|
||||||
if p.byDigest {
|
if p.byDigest {
|
||||||
delete(r, "content")
|
delete(r, "content")
|
||||||
@@ -94,22 +77,7 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p
|
|||||||
if p.asNode != "" {
|
if p.asNode != "" {
|
||||||
c.Node = p.asNode
|
c.Node = p.asNode
|
||||||
}
|
}
|
||||||
if p.asModule != "" {
|
return c
|
||||||
c.Module = p.asModule
|
|
||||||
}
|
|
||||||
k, err := delivery.ComposedKey(node, module, fp)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
signed, err := delivery.SignComposed(c, signer)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
raw, err := delivery.EncodeComposed(signed)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
return k, raw
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
|
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
|
||||||
@@ -125,12 +93,12 @@ type gput struct {
|
|||||||
|
|
||||||
type put struct {
|
type put struct {
|
||||||
module, fp, content string
|
module, fp, content string
|
||||||
stranger, deleted bool
|
|
||||||
byDigest bool
|
byDigest bool
|
||||||
asNode, asModule string
|
asNode string
|
||||||
}
|
}
|
||||||
|
|
||||||
// Defends novox/hq ADR 0547 §8: the bus node writes the newest signed list alone, only when it changed, and reports it.
|
// Defends novox/hq ADR 0547 §8: the bus node writes the grants part's list alone, only when it changed, and reports it,
|
||||||
|
// into the file the bus module it runs declares (issue 1126, B17: what runs, never a composition bucket).
|
||||||
func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
||||||
one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c")
|
one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c")
|
||||||
five := list("controller", "a", "b", "c", "d")
|
five := list("controller", "a", "b", "c", "d")
|
||||||
@@ -152,13 +120,14 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
// wantSource and wantFollows are what the last have says it took the list from and follows.
|
// wantSource and wantFollows are what the last have says it took the list from and follows.
|
||||||
wantSource, wantFollows string
|
wantSource, wantFollows string
|
||||||
}{
|
}{
|
||||||
// Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module's
|
// Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module
|
||||||
// composition declares; the list a composition carries is never written, whatever becomes of the grants key.
|
// this node runs declares; the list a composition carries is never written, whatever becomes of the grants key.
|
||||||
{name: "the grants part's list is written into the file a composition declares",
|
// B17: the file's place is read from what runs, as the agent holds it, never from a composition bucket.
|
||||||
|
{name: "the grants part's list is written into the file the bus module it runs declares",
|
||||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
||||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||||
{name: "a key that says follow the controller still writes the grants part's list",
|
{name: "a bus module it runs by digest still names the file",
|
||||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
|
before: []put{{module: "nats", fp: fpOf("a"), content: two, byDigest: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
|
||||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||||
{name: "a newer composition's list is not written",
|
{name: "a newer composition's list is not written",
|
||||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
|
||||||
@@ -167,8 +136,8 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
|
||||||
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}},
|
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}},
|
||||||
grants: []gput{{follow: delivery.FollowGrants, content: three}}},
|
grants: []gput{{follow: delivery.FollowGrants, content: three}}},
|
||||||
{name: "a composition signed by a stranger declares no file",
|
{name: "another node's composition declares no file",
|
||||||
before: []put{{module: "nats", fp: fpOf("a"), content: two, stranger: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}},
|
before: []put{{module: "nats", fp: fpOf("a"), content: two, asNode: "ace"}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}},
|
||||||
{name: "an unreadable grants key with none heard before writes nothing, never the composition's list",
|
{name: "an unreadable grants key with none heard before writes nothing, never the composition's list",
|
||||||
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{unreadable: true}},
|
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{unreadable: true}},
|
||||||
onDisk: two, wantErr: "key is not heard", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
onDisk: two, wantErr: "key is not heard", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
||||||
@@ -201,9 +170,8 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
|
||||||
} {
|
} {
|
||||||
t.Run(c.name, func(t *testing.T) {
|
t.Run(c.name, func(t *testing.T) {
|
||||||
f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})}
|
|
||||||
objects := map[string]string{}
|
|
||||||
var mu sync.Mutex
|
var mu sync.Mutex
|
||||||
|
var running []delivery.Composed
|
||||||
var writes []string
|
var writes []string
|
||||||
var reports []Written
|
var reports []Written
|
||||||
reported := make(chan struct{}, 64)
|
reported := make(chan struct{}, 64)
|
||||||
@@ -257,7 +225,12 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
gf.first = append(gf.first, u)
|
gf.first = append(gf.first, u)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fl := New(Env{Node: node, Composed: f, Grants: gf,
|
fl := New(Env{Node: node, Grants: gf,
|
||||||
|
Running: func() []delivery.Composed {
|
||||||
|
mu.Lock()
|
||||||
|
defer mu.Unlock()
|
||||||
|
return append([]delivery.Composed(nil), running...)
|
||||||
|
},
|
||||||
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
|
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
|
||||||
Keep: func([]byte, uint64) {},
|
Keep: func([]byte, uint64) {},
|
||||||
Backoff: func(int) time.Duration { return time.Hour },
|
Backoff: func(int) time.Duration { return time.Hour },
|
||||||
@@ -271,8 +244,7 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
out, err := seal.Open(b, nodeKey.PrivateKey)
|
out, err := seal.Open(b, nodeKey.PrivateKey)
|
||||||
return []byte(out), err
|
return []byte(out), err
|
||||||
},
|
},
|
||||||
Key: func() string { return nodeKey.PublicKey },
|
Key: func() string { return nodeKey.PublicKey },
|
||||||
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
|
|
||||||
OnDisk: func(json.RawMessage) ([]byte, error) {
|
OnDisk: func(json.RawMessage) ([]byte, error) {
|
||||||
if c.onDisk == "" {
|
if c.onDisk == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -296,30 +268,14 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
},
|
},
|
||||||
Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) },
|
Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) },
|
||||||
})
|
})
|
||||||
update := func(p put) agent.Update {
|
|
||||||
f.rev++
|
|
||||||
signer := controllerKey
|
|
||||||
if p.stranger {
|
|
||||||
signer = strangerKey
|
|
||||||
}
|
|
||||||
k, raw := comp(t, p.module, p.fp, p.content, signer, p)
|
|
||||||
if p.byDigest {
|
|
||||||
objects[Digest([]byte(p.content))] = p.content
|
|
||||||
}
|
|
||||||
if p.deleted {
|
|
||||||
return agent.Update{Key: k, Revision: f.rev, Deleted: true}
|
|
||||||
}
|
|
||||||
return agent.Update{Key: k, Value: raw, Revision: f.rev}
|
|
||||||
}
|
|
||||||
for _, p := range c.before {
|
for _, p := range c.before {
|
||||||
f.first = append(f.first, update(p))
|
running = append(running, comp(p.module, p.fp, p.content, p))
|
||||||
}
|
}
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
done := make(chan error, 1)
|
done := make(chan error, 1)
|
||||||
go func() { done <- fl.Run(ctx) }()
|
go func() { done <- fl.Run(ctx) }()
|
||||||
<-f.watching
|
// What runs changes after the first list is written: the first write waits for the grants part's key.
|
||||||
// A later put after the first list is written: the first write waits for the grants part's key to catch up.
|
|
||||||
if len(c.after) > 0 {
|
if len(c.after) > 0 {
|
||||||
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
|
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
|
||||||
mu.Lock()
|
mu.Lock()
|
||||||
@@ -331,7 +287,10 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, p := range c.after {
|
for _, p := range c.after {
|
||||||
f.ch <- update(p)
|
mu.Lock()
|
||||||
|
running = []delivery.Composed{comp(p.module, p.fp, p.content, p)}
|
||||||
|
mu.Unlock()
|
||||||
|
fl.RunsChanged()
|
||||||
}
|
}
|
||||||
if len(grantsAfter) > 0 {
|
if len(grantsAfter) > 0 {
|
||||||
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
|
||||||
@@ -384,23 +343,18 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
|
|||||||
if c.wantUnwatched != (st.Unwatched != "") {
|
if c.wantUnwatched != (st.Unwatched != "") {
|
||||||
t.Fatalf("the have says unwatched %q", st.Unwatched)
|
t.Fatalf("the have says unwatched %q", st.Unwatched)
|
||||||
}
|
}
|
||||||
// While the grants part's list is followed, an apply of the controller's composition writes not its list.
|
// While the grants part's list is followed, an apply of the bus module's composition writes not its list.
|
||||||
if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" {
|
if cmp := comp("nats", fpOf("a"), two, put{}); c.wantFollows != "" {
|
||||||
cmp, err := delivery.DecodeComposed(raw)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
held, err := fl.Hold(cmp)
|
held, err := fl.Hold(cmp)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, _, _, hasList := userList(held.Resources)
|
if _, _, _, hasList := userList(held.Resources); hasList || len(held.Resources) < 1 {
|
||||||
if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 {
|
t.Fatalf("an apply while following %s holds the composition's list: %v", c.wantFollows, hasList)
|
||||||
t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList)
|
|
||||||
}
|
}
|
||||||
// A list on the machine that cannot be read stops the apply; it is never left out to be removed.
|
// A list on the machine that cannot be read stops the apply; it is never left out to be removed.
|
||||||
if c.wantFollows == delivery.FollowGrants {
|
if c.wantFollows == delivery.FollowGrants {
|
||||||
readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) {
|
readErr := New(Env{Node: node, OnDisk: func(json.RawMessage) ([]byte, error) {
|
||||||
return nil, errors.New("permission denied")
|
return nil, errors.New("permission denied")
|
||||||
}})
|
}})
|
||||||
readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants}
|
readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants}
|
||||||
|
|||||||
+12
-113
@@ -1,155 +1,54 @@
|
|||||||
// Package witness is the node-engine watching a core build it placed — the controller on the control
|
// Package witness is the node-engine watching a core build it placed — the node tools on every machine — and
|
||||||
// node, the node tools on every machine — and restoring the build before it when the new one is not
|
// restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the
|
||||||
// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the
|
// component being replaced is never the only witness of its successor).
|
||||||
// only witness of its successor).
|
|
||||||
//
|
//
|
||||||
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what
|
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what
|
||||||
// shape, within which bound — said once, here, and held by contract_test.go. The controller's side
|
// shape, within which bound — said once, here, and held by contract_test.go.
|
||||||
// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a
|
|
||||||
// change to this file and its test.
|
|
||||||
package witness
|
package witness
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The two witnesses, as a process declares which watches it (`witness`). A process that says none is
|
// The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one
|
||||||
// placed as ever; one that says nothing is judged by its name's default (Default).
|
// that says nothing is judged by its name's default (Default).
|
||||||
const (
|
const (
|
||||||
// ByLease: healthy when the controller this machine started holds the controller's lease.
|
|
||||||
ByLease = "lease"
|
|
||||||
// ByPing: healthy when this machine's runtime answers the services protocol's PING.
|
// ByPing: healthy when this machine's runtime answers the services protocol's PING.
|
||||||
ByPing = "ping"
|
ByPing = "ping"
|
||||||
// ByNone: not judged.
|
// ByNone: not judged.
|
||||||
ByNone = "none"
|
ByNone = "none"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket
|
|
||||||
// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five.
|
|
||||||
const (
|
|
||||||
LeaseBucket = "mesh-controller_lease"
|
|
||||||
LeaseKey = "holder"
|
|
||||||
// LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is
|
|
||||||
// not holding it, whatever the key still says.
|
|
||||||
LeaseAge = 15 * time.Second
|
|
||||||
// Skew is how far the controller's clock and this host's may disagree about when it took the
|
|
||||||
// lease. One machine, one clock, in practice: a margin, not a tolerance.
|
|
||||||
Skew = 2 * time.Second
|
|
||||||
)
|
|
||||||
|
|
||||||
// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
|
// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
|
||||||
const (
|
const (
|
||||||
// ControllerWithin: the controller holds the lease within sixty seconds of starting.
|
|
||||||
ControllerWithin = 60 * time.Second
|
|
||||||
// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
|
// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
|
||||||
// each PING answered within PingWithin.
|
// each PING answered within PingWithin.
|
||||||
NodeToolsWithin = 60 * time.Second
|
NodeToolsWithin = 60 * time.Second
|
||||||
PingWithin = 5 * time.Second
|
PingWithin = 5 * time.Second
|
||||||
Every = 5 * time.Second
|
Every = 5 * time.Second
|
||||||
// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
|
// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
|
||||||
// unwitnessed: the grant missing, the bucket missing, the bus away the whole time.
|
// unwitnessed: the grant missing, the bus away the whole time.
|
||||||
GiveUp = 30 * time.Minute
|
GiveUp = 30 * time.Minute
|
||||||
)
|
)
|
||||||
|
|
||||||
// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease
|
|
||||||
// Holder, field for field by its JSON names. Only what the witness reads is required; a field the
|
|
||||||
// controller adds later is ignored here.
|
|
||||||
type ControllerLease struct {
|
|
||||||
// Instance names one controller process: "controller@<machine> pid <pid> since <RFC 3339>".
|
|
||||||
Instance string `json:"instance"`
|
|
||||||
// Host is the machine it runs on, as its own hostname says.
|
|
||||||
Host string `json:"host,omitempty"`
|
|
||||||
// Build is the controller's build as it knows it; not read here — the toolchain stamps no
|
|
||||||
// version, so it says "development build" — and a bundle's identity is its digest, which the
|
|
||||||
// host already knows.
|
|
||||||
Build string `json:"build,omitempty"`
|
|
||||||
Epoch uint64 `json:"epoch,omitempty"`
|
|
||||||
// Taken is when this instance took the key, Renewed when it last renewed it, by its own clock.
|
|
||||||
Taken time.Time `json:"taken"`
|
|
||||||
Renewed time.Time `json:"renewed"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// ParseLease reads the key's value.
|
|
||||||
func ParseLease(value []byte) (ControllerLease, error) {
|
|
||||||
var l ControllerLease
|
|
||||||
if err := json.Unmarshal(value, &l); err != nil {
|
|
||||||
return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err)
|
|
||||||
}
|
|
||||||
return l, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or
|
|
||||||
// after `since` — the controller this machine started then, and not the one before it — and is
|
|
||||||
// held now. Why says what it saw when it is not.
|
|
||||||
func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) {
|
|
||||||
switch {
|
|
||||||
case l.Instance == "":
|
|
||||||
return false, "the lease names no holder"
|
|
||||||
case host != "" && !sameMachine(l.Host, host):
|
|
||||||
return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host)
|
|
||||||
case l.Taken.Before(since.Add(-Skew)):
|
|
||||||
return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s",
|
|
||||||
l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339))
|
|
||||||
case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge:
|
|
||||||
return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s",
|
|
||||||
l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge)
|
|
||||||
}
|
|
||||||
return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch)
|
|
||||||
}
|
|
||||||
|
|
||||||
// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
|
|
||||||
func sameMachine(a, b string) bool {
|
|
||||||
short := func(s string) string {
|
|
||||||
s = strings.ToLower(strings.TrimSpace(s))
|
|
||||||
if i := strings.IndexByte(s, '.'); i > 0 {
|
|
||||||
s = s[:i]
|
|
||||||
}
|
|
||||||
return s
|
|
||||||
}
|
|
||||||
return short(a) == short(b)
|
|
||||||
}
|
|
||||||
|
|
||||||
func latest(a, b time.Time) time.Time {
|
|
||||||
if b.After(a) {
|
|
||||||
return b
|
|
||||||
}
|
|
||||||
return a
|
|
||||||
}
|
|
||||||
|
|
||||||
// NodeToolsService is the runtime's name on the services protocol, and its instance is this
|
// NodeToolsService is the runtime's name on the services protocol, and its instance is this
|
||||||
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
|
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
|
||||||
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
|
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
|
||||||
const NodeToolsService = "node-tools"
|
const NodeToolsService = "node-tools"
|
||||||
|
|
||||||
// Default is the witness a process is judged by when it names none: the two core processes the mesh
|
// Default is the witness a process is judged by when it names none: the core process the mesh composes on every
|
||||||
// composes, by the names it composes them under, and nothing else.
|
// machine, by the name it composes it under, and nothing else.
|
||||||
func Default(process string) string {
|
func Default(process string) string {
|
||||||
switch process {
|
if process == NodeToolsService {
|
||||||
case "mesh-controller":
|
|
||||||
return ByLease
|
|
||||||
case NodeToolsService:
|
|
||||||
return ByPing
|
return ByPing
|
||||||
}
|
}
|
||||||
return ByNone
|
return ByNone
|
||||||
}
|
}
|
||||||
|
|
||||||
// Within is the bound for a witness.
|
// Within is the bound for a witness.
|
||||||
func Within(by string) time.Duration {
|
func Within(string) time.Duration { return NodeToolsWithin }
|
||||||
if by == ByLease {
|
|
||||||
return ControllerWithin
|
|
||||||
}
|
|
||||||
return NodeToolsWithin
|
|
||||||
}
|
|
||||||
|
|
||||||
// Component names what a witness judges, as a rollback says it.
|
// Component names what a witness judges, as a rollback says it.
|
||||||
func Component(by string) string {
|
func Component(string) string { return link.ComponentNodeTools }
|
||||||
if by == ByLease {
|
|
||||||
return link.ComponentController
|
|
||||||
}
|
|
||||||
return link.ComponentNodeTools
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -9,82 +9,7 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// The contract, held. Each of these is a line the controller's side relies on or writes; a change on
|
// The contract, held (novox/hq to-be 45 §8).
|
||||||
// either side changes this test (novox/hq to-be 45 §8).
|
|
||||||
|
|
||||||
// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names.
|
|
||||||
func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) {
|
|
||||||
if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" {
|
|
||||||
t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder",
|
|
||||||
LeaseBucket, LeaseKey)
|
|
||||||
}
|
|
||||||
if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
|
|
||||||
t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got)
|
|
||||||
}
|
|
||||||
if LeaseAge != 15*time.Second {
|
|
||||||
t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is
|
|
||||||
// read. The value below is the shape that Holder marshals to.
|
|
||||||
func TestTheLeaseValueIsTheControllersHolder(t *testing.T) {
|
|
||||||
written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` +
|
|
||||||
`"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}`
|
|
||||||
l, err := ParseLease([]byte(written))
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() {
|
|
||||||
t.Fatalf("the lease was not read whole: %+v", l)
|
|
||||||
}
|
|
||||||
// And a field the controller adds later does not stop it being read.
|
|
||||||
if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil {
|
|
||||||
t.Fatalf("a lease value with a field this host does not know was refused: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Healthy is: held now, on this machine, by an instance that took it after the new build started.
|
|
||||||
func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) {
|
|
||||||
started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
||||||
now := started.Add(30 * time.Second)
|
|
||||||
held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example",
|
|
||||||
Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58}
|
|
||||||
for _, c := range []struct {
|
|
||||||
name string
|
|
||||||
l ControllerLease
|
|
||||||
want bool
|
|
||||||
says string
|
|
||||||
}{
|
|
||||||
{"held by the new instance", held, true, "holds the lease"},
|
|
||||||
{"nobody", ControllerLease{}, false, "no holder"},
|
|
||||||
{"the old instance, taken before the restart", func() ControllerLease {
|
|
||||||
l := held
|
|
||||||
l.Taken = started.Add(-time.Hour)
|
|
||||||
return l
|
|
||||||
}(), false, "before the new build started"},
|
|
||||||
{"another machine's controller", func() ControllerLease {
|
|
||||||
l := held
|
|
||||||
l.Host = "home-server"
|
|
||||||
return l
|
|
||||||
}(), false, "not this machine"},
|
|
||||||
{"taken and no longer renewed", func() ControllerLease {
|
|
||||||
l := held
|
|
||||||
l.Renewed = now.Add(-20 * time.Second)
|
|
||||||
return l
|
|
||||||
}(), false, "past its"},
|
|
||||||
{"taken within the clocks' skew of the start", func() ControllerLease {
|
|
||||||
l := held
|
|
||||||
l.Taken = started.Add(-time.Second)
|
|
||||||
return l
|
|
||||||
}(), true, "holds the lease"},
|
|
||||||
} {
|
|
||||||
got, why := c.l.HeldBySince("anchor", started, now)
|
|
||||||
if got != c.want || !strings.Contains(why, c.says) {
|
|
||||||
t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
|
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
|
||||||
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
|
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
|
||||||
@@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// What is judged by default is the mesh's two core processes, by the names the controller composes
|
// What is judged by default is the mesh's core process on every machine, by the name it is composed under, and
|
||||||
// them under, and nothing else.
|
// nothing else: the controller ended (issue 1126), so its name is judged as any other.
|
||||||
func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) {
|
func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) {
|
||||||
for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} {
|
for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} {
|
||||||
if got := Default(name); got != want {
|
if got := Default(name); got != want {
|
||||||
t.Errorf("%s is judged by %q, want %q", name, got, want)
|
t.Errorf("%s is judged by %q, want %q", name, got, want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
|
if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
|
||||||
t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five")
|
t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A verdict on the wire names its component as the controller's condition does.
|
// A verdict on the wire names its component as the controller's condition does.
|
||||||
func TestAVerdictNamesItsComponent(t *testing.T) {
|
func TestAVerdictNamesItsComponent(t *testing.T) {
|
||||||
raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack})
|
raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack})
|
||||||
if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" {
|
if !strings.Contains(string(raw), `"component":"node-tools"`) {
|
||||||
t.Fatalf("a verdict names its component as %s", raw)
|
t.Fatalf("a verdict names its component as %s", raw)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string {
|
|||||||
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
|
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
|
||||||
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
|
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
name := "mesh-controller"
|
name := "node-tools"
|
||||||
start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
|
start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
|
||||||
|
|
||||||
// Build A placed and running.
|
// Build A placed and running.
|
||||||
p, err := Place(root, name, ByLease, buildA, "", "", start)
|
p, err := Place(root, name, ByPing, buildA, "", "", start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
@@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Build B placed while A still runs: A is moved aside, and reachable at its new path.
|
// Build B placed while A still runs: A is moved aside, and reachable at its new path.
|
||||||
place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute))
|
place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute))
|
||||||
for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
|
for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
|
||||||
info, err := os.Stat(dir)
|
info, err := os.Stat(dir)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -14,15 +14,14 @@ import (
|
|||||||
// concluded (to-be 45 §8).
|
// concluded (to-be 45 §8).
|
||||||
//
|
//
|
||||||
// **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer
|
// **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer
|
||||||
// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a
|
// from the bus — the PING delivered — so a machine whose own link is down, or a
|
||||||
// bus that refuses the question, never rolls a build back for the host's own trouble. A look that
|
// bus that refuses the question, never rolls a build back for the host's own trouble. A look that
|
||||||
// could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed:
|
// could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed:
|
||||||
// neither proved nor rolled back, and said.
|
// neither proved nor rolled back, and said.
|
||||||
type Watcher struct {
|
type Watcher struct {
|
||||||
Root string
|
Root string
|
||||||
// Node is this machine's node name, the instance its runtime answers PING as; Host its hostname,
|
// Node is this machine's node name, the instance its runtime answers PING as.
|
||||||
// as the controller's lease names its machine.
|
Node string
|
||||||
Node, Host string
|
|
||||||
// Asker is the link open now, or nil.
|
// Asker is the link open now, or nil.
|
||||||
Asker func() link.Asker
|
Asker func() link.Asker
|
||||||
Run Runner
|
Run Runner
|
||||||
@@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
|
|||||||
every = Every
|
every = Every
|
||||||
}
|
}
|
||||||
now := w.now()
|
now := w.now()
|
||||||
healthy, why, err := w.judge(ctx, s, now)
|
healthy, why, err := w.judge(ctx, s)
|
||||||
|
|
||||||
w.hold(func() {
|
w.hold(func() {
|
||||||
// Read again in turn: an apply may have placed another build meanwhile, which starts its own
|
// Read again in turn: an apply may have placed another build meanwhile, which starts its own
|
||||||
@@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not.
|
// judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not.
|
||||||
func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) {
|
func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) {
|
||||||
var asker link.Asker
|
var asker link.Asker
|
||||||
if w.Asker != nil {
|
if w.Asker != nil {
|
||||||
asker = w.Asker()
|
asker = w.Asker()
|
||||||
@@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri
|
|||||||
asking, cancel := context.WithTimeout(ctx, PingWithin)
|
asking, cancel := context.WithTimeout(ctx, PingWithin)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
switch s.Witness {
|
switch s.Witness {
|
||||||
case ByLease:
|
|
||||||
value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey)
|
|
||||||
if err != nil {
|
|
||||||
return false, "", err
|
|
||||||
}
|
|
||||||
if !found {
|
|
||||||
return false, "nobody holds the controller's lease", nil
|
|
||||||
}
|
|
||||||
lease, err := ParseLease(value)
|
|
||||||
if err != nil {
|
|
||||||
return false, err.Error(), nil
|
|
||||||
}
|
|
||||||
held, why := lease.HeldBySince(w.Host, s.Started, now)
|
|
||||||
return held, why, nil
|
|
||||||
case ByPing:
|
case ByPing:
|
||||||
err := asker.Ping(asking, s.Process, w.Node)
|
err := asker.Ping(asking, s.Process, w.Node)
|
||||||
switch {
|
switch {
|
||||||
|
|||||||
@@ -13,8 +13,8 @@ import (
|
|||||||
"github.com/novox/mesh-host/internal/link"
|
"github.com/novox/mesh-host/internal/link"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Each rollback path, induced against a real directory: a controller that never takes the lease, a
|
// Each rollback path, induced against a real directory: a runtime that never answers PING, a build
|
||||||
// runtime that never answers PING, a build declared not reversible; and a healthy update, which
|
// declared not reversible; and a healthy update, which
|
||||||
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
|
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
|
||||||
|
|
||||||
const (
|
const (
|
||||||
@@ -62,18 +62,10 @@ func kept(root, name string) bool {
|
|||||||
// asker answers as the build it is told is running would.
|
// asker answers as the build it is told is running would.
|
||||||
type asker struct {
|
type asker struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
lease func() ([]byte, bool, error)
|
|
||||||
ping func() error
|
ping func() error
|
||||||
asked int
|
asked int
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) {
|
|
||||||
a.mu.Lock()
|
|
||||||
defer a.mu.Unlock()
|
|
||||||
a.asked++
|
|
||||||
return a.lease()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (a *asker) Ping(context.Context, string, string) error {
|
func (a *asker) Ping(context.Context, string, string) error {
|
||||||
a.mu.Lock()
|
a.mu.Lock()
|
||||||
defer a.mu.Unlock()
|
defer a.mu.Unlock()
|
||||||
@@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
|
|||||||
}
|
}
|
||||||
|
|
||||||
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
|
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
|
||||||
return &Watcher{Root: root, Node: "anchor", Host: "anchor",
|
return &Watcher{Root: root, Node: "anchor",
|
||||||
Asker: func() link.Asker {
|
Asker: func() link.Asker {
|
||||||
if a == nil {
|
if a == nil {
|
||||||
return nil
|
return nil
|
||||||
@@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) {
|
// A runtime that starts and never answers PING: restored to the build before, once, said once.
|
||||||
return func() ([]byte, bool, error) {
|
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
|
||||||
return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`,
|
|
||||||
instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A controller that starts and never takes the lease: restored to the build before, once, said once.
|
|
||||||
func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||||
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||||
if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") {
|
if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") {
|
||||||
t.Fatal("the new controller is not running with the build before it kept beside it")
|
t.Fatal("the new runtime is not running with the build before it kept beside it")
|
||||||
}
|
}
|
||||||
|
|
||||||
// The old instance's lease, taken an hour ago, is all the bus ever shows.
|
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
|
||||||
a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)}
|
|
||||||
m := &machine{}
|
m := &machine{}
|
||||||
var verdicts []link.Rollback
|
var verdicts []link.Rollback
|
||||||
w := watcher(root, a, m, &clock, &verdicts)
|
w := watcher(root, a, m, &clock, &verdicts)
|
||||||
|
|
||||||
look(w, &clock, int(ControllerWithin/Every)-1)
|
look(w, &clock, int(NodeToolsWithin/Every)-1)
|
||||||
if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB {
|
if len(verdicts) != 0 || running(t, root, "node-tools") != buildB {
|
||||||
t.Fatalf("rolled back before its bound: %+v", verdicts)
|
t.Fatalf("rolled back before its bound: %+v", verdicts)
|
||||||
}
|
}
|
||||||
look(w, &clock, 1)
|
look(w, &clock, 1)
|
||||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
|
||||||
verdicts[0].Component != link.ComponentController {
|
verdicts[0].Component != link.ComponentNodeTools {
|
||||||
t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts)
|
t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts)
|
||||||
}
|
}
|
||||||
if running(t, root, "mesh-controller") != buildA {
|
if running(t, root, "node-tools") != buildA {
|
||||||
t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller"))
|
t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools"))
|
||||||
}
|
}
|
||||||
if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" {
|
if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" {
|
||||||
t.Fatalf("the machine was asked %v", m.commands)
|
t.Fatalf("the machine was asked %v", m.commands)
|
||||||
}
|
}
|
||||||
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") {
|
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") {
|
||||||
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
|
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
|
|||||||
t.Fatalf("what every report says is %+v", s)
|
t.Fatalf("what every report says is %+v", s)
|
||||||
}
|
}
|
||||||
// The mesh still declares B: refused, A kept running, the verdict still standing.
|
// The mesh still declares B: refused, A kept running, the verdict still standing.
|
||||||
p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock)
|
p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock)
|
||||||
if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") {
|
if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") {
|
||||||
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller"))
|
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools"))
|
||||||
}
|
}
|
||||||
if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
|
if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
|
||||||
t.Fatal("refusing the rolled-back build changed what stands or started a trial")
|
t.Fatal("refusing the rolled-back build changed what stands or started a trial")
|
||||||
}
|
}
|
||||||
// A newer build is a new trial, and ends what was concluded once it is proved.
|
// A newer build is a new trial, and ends what was concluded once it is proved.
|
||||||
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
|
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
|
||||||
if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 {
|
if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 {
|
||||||
t.Fatal("a newer build after a rollback was not placed on trial")
|
t.Fatal("a newer build after a rollback was not placed on trial")
|
||||||
}
|
}
|
||||||
if err := Proved(root, "mesh-controller"); err != nil {
|
if err := Proved(root, "node-tools"); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if len(Standing(root)) != 0 || kept(root, "mesh-controller") {
|
if len(Standing(root)) != 0 || kept(root, "node-tools") {
|
||||||
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
|
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// A runtime that never answers PING: restored, once.
|
|
||||||
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
|
||||||
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
|
||||||
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
|
||||||
|
|
||||||
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
|
|
||||||
m := &machine{}
|
|
||||||
var verdicts []link.Rollback
|
|
||||||
w := watcher(root, a, m, &clock, &verdicts)
|
|
||||||
look(w, &clock, int(NodeToolsWithin/Every)+20)
|
|
||||||
|
|
||||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools {
|
|
||||||
t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts)
|
|
||||||
}
|
|
||||||
if running(t, root, "node-tools") != buildA || kept(root, "node-tools") {
|
|
||||||
t.Fatal("the runtime running is not the build before, or a copy of it is still kept")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
|
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
|
||||||
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
|
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||||
// The controller placed by a host from before any witness: a record, and no state.
|
// The runtime placed by a host from before any witness: a record, and no state.
|
||||||
if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil {
|
if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||||
started := clock
|
|
||||||
|
|
||||||
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
|
a := &asker{ping: func() error { return link.ErrNoAnswer }}
|
||||||
m := &machine{}
|
m := &machine{}
|
||||||
var verdicts []link.Rollback
|
var verdicts []link.Rollback
|
||||||
w := watcher(root, a, m, &clock, &verdicts)
|
w := watcher(root, a, m, &clock, &verdicts)
|
||||||
look(w, &clock, 5)
|
look(w, &clock, 5)
|
||||||
if !kept(root, "mesh-controller") {
|
if !kept(root, "node-tools") {
|
||||||
t.Fatal("the build before was deleted while the new one was still on trial")
|
t.Fatal("the build before was deleted while the new one was still on trial")
|
||||||
}
|
}
|
||||||
// A third build while the second is on trial: the one kept is still the one seen healthy.
|
// A third build while the second is on trial: the one kept is still the one seen healthy.
|
||||||
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
|
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
|
||||||
if s, _ := Load(root, "mesh-controller"); s.Previous != buildA {
|
if s, _ := Load(root, "node-tools"); s.Previous != buildA {
|
||||||
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
|
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
|
||||||
}
|
}
|
||||||
started = clock
|
a.ping = func() error { return nil }
|
||||||
a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock)
|
|
||||||
look(w, &clock, 2)
|
look(w, &clock, 2)
|
||||||
if len(verdicts) != 0 || len(m.commands) != 0 {
|
if len(verdicts) != 0 || len(m.commands) != 0 {
|
||||||
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
|
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
|
||||||
}
|
}
|
||||||
if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC {
|
if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC {
|
||||||
t.Fatal("the build before was not deleted once the new one was proved")
|
t.Fatal("the build before was not deleted once the new one was proved")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
|
|||||||
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||||
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock)
|
place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock)
|
||||||
|
|
||||||
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
|
a := &asker{ping: func() error { return link.ErrNoAnswer }}
|
||||||
m := &machine{}
|
m := &machine{}
|
||||||
var verdicts []link.Rollback
|
var verdicts []link.Rollback
|
||||||
w := watcher(root, a, m, &clock, &verdicts)
|
w := watcher(root, a, m, &clock, &verdicts)
|
||||||
look(w, &clock, int(ControllerWithin/Every)+20)
|
look(w, &clock, int(NodeToolsWithin/Every)+20)
|
||||||
|
|
||||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
|
||||||
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
|
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
|
||||||
@@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
|||||||
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
|
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
|
||||||
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
|
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
|
||||||
}
|
}
|
||||||
if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
|
if len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
|
||||||
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
|
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
|
||||||
running(t, root, "mesh-controller"))
|
running(t, root, "node-tools"))
|
||||||
}
|
}
|
||||||
if len(Standing(root)) != 1 {
|
if len(Standing(root)) != 1 {
|
||||||
t.Fatal("the not-reversible verdict does not stand")
|
t.Fatal("the not-reversible verdict does not stand")
|
||||||
@@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
|
|||||||
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
|
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
|
||||||
root := t.TempDir()
|
root := t.TempDir()
|
||||||
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
|
||||||
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
|
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
|
||||||
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
|
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
|
||||||
|
|
||||||
a := &asker{lease: func() ([]byte, bool, error) {
|
a := &asker{ping: func() error {
|
||||||
return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk)
|
return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk)
|
||||||
}}
|
}}
|
||||||
m := &machine{}
|
m := &machine{}
|
||||||
var verdicts []link.Rollback
|
var verdicts []link.Rollback
|
||||||
w := watcher(root, a, m, &clock, &verdicts)
|
w := watcher(root, a, m, &clock, &verdicts)
|
||||||
look(w, &clock, int(GiveUp/Every)-1)
|
look(w, &clock, int(GiveUp/Every)-1)
|
||||||
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
|
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
|
||||||
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
|
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
|
||||||
}
|
}
|
||||||
look(w, &clock, 1)
|
look(w, &clock, 1)
|
||||||
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
|
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
|
||||||
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
|
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
|
||||||
}
|
}
|
||||||
if !kept(root, "mesh-controller") {
|
if !kept(root, "node-tools") {
|
||||||
t.Fatal("the build before was deleted though the new one was never seen healthy")
|
t.Fatal("the build before was deleted though the new one was never seen healthy")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user