novox/mesh-host
18
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge pull request 'The node-engine without the controller's stream and lease (issue 1126 group B)' (#197) from feat/1126-removal into main

Browse Source
This commit was merged in pull request #197.
This commit is contained in:
mesh-tools
2026-11-02 13:41:28 +00:00
parent 84737f4cb2 049c3b8c11
commit 98fbc490a4
64 changed files with 386 additions and 3943 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
-20
View File
@@ -171,26 +171,6 @@ carries. The claim is not. The foundation is four container services, and:
So `foundation.lock` here is a real bundle with a placeholder's content. Saying that plainly
beats shipping a host that claims a foundation it has never raised.
## mesh-cli's socket (novox/hq ADR 0272)
The engine serves `/run/nox-mesh-host/cli.sock` for `mesh-cli`, the operator's command (`internal/meshcli`). It
reads the asking account from the kernel (`SO_PEERCRED`) and asks the controller on this node's own subject; the
controller decides what the line may do, and the engine runs nothing it receives. One line per connection, each
connection's deadline set the moment it is accepted, two lines at once per account and eight for the machine,
"busy" beyond that.
**Only a login session can be the terminal.** The engine reads the asking process's cgroup and sends its login
session's scope (`session-<id>.scope` under the account's own `user-<uid>.slice`, which logind's own record in
`/run/systemd/sessions/<id>` must say is this account's, of class `user`), or nothing, read the moment the
connection is accepted; the controller makes
nothing else the terminal. A service running as the operator — the tool runner, a unit of the account's user
manager — is at most an ordinary call.
**A process outside the machine's own namespaces is refused**, whatever its user id: the peer's mount and user
namespace must be the engine's. So `mesh-cli` run inside a flatpak, a toolbox or distrobox container, or any other
container — even one running as the operator's own id — is refused with the reason; run it from the machine's own
shell.
## A capability is detected, never assumed
The reason this is the first thing built rather than a detail of it.
cmd/mesh-host/authority_test.go
+2 -1
View File
@@ -9,6 +9,7 @@ import (
"encoding/pem"
"os"
"path/filepath"
"strings"
"testing"
"time"
@@ -73,7 +74,7 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) {
if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil {
t.Error("a controller-signed message is accepted once the vault's key is read")
}
if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
if got := keys().Trusts; strings.Contains(strings.Join(got, " "), ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) ||
trust.VaultAuthority() != vaultCA || told != 2 {
t.Errorf("after the signers are read the record trusts %v (told %d)", got, told)
}
cmd/mesh-host/engineseat.go
+5 -10
View File
@@ -333,8 +333,8 @@ func composeFrom(current *agentNow, composer *atomic.Pointer[selfcompose.Compose
// packagesReadFor bounds the verb packages' reads and its write.
const packagesReadFor = 20 * time.Second
// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages
// beside the controller's mirror's key for it and their difference; with write, the key written again from what the
// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages;
// with write, the key written again from what the
// composer that runs now handed, refused in words while the node composes none of its own.
func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[selfcompose.Composer]) func(write bool) (any, error) {
return func(write bool) (any, error) {
@@ -344,8 +344,8 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s
if write {
c := composer.Load()
if c == nil {
return nil, errors.New("this node composes none of its own modules now (it applies the controller's " +
"compositions, or its node agent is starting), so it has no packages of its own to write")
return nil, errors.New("this node composes none of its own modules now (its node agent is starting), " +
"so it has no packages of its own to write")
}
w, err := c.WritePackages(ctx)
if err != nil {
@@ -354,12 +354,7 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s
wrote = &w
}
key, keyErr := link.EnginePackages{Session: session}.Get(ctx, node)
var mirror []byte
h, mirrorErr := session.KVHistory(ctx, link.MirrorPackagesBucket, node)
if mirrorErr == nil && len(h) > 0 && !h[len(h)-1].Deleted {
mirror = h[len(h)-1].Value
}
v := link.PackagesAnswer(node, key, mirror, keyErr, mirrorErr)
v := link.PackagesAnswer(node, key, keyErr)
v.Wrote = wrote
return v, nil
}
cmd/mesh-host/enrol_seat.go
+8 -15
View File
@@ -16,17 +16,17 @@ import (
// enrolThroughSeat joins this machine through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): it redeems the token
// with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps
// its membership and its login, dials as the login node-logins names and confirms. handled is false when the bus does not serve
// the seat to this token (the controller still holds its lease, so its enrolment subject is the way in): nothing was sent or
// kept then, and the caller enrols as before.
// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this
// token (link.ErrNoSeat) is said as the error: nothing was sent or kept.
func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity,
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) (handled bool, err error) {
sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error {
req := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public,
SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported}
if found != nil {
var err error
if req.Tunnel, err = json.Marshal(found); err != nil {
return true, err
return err
}
}
req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(delivery.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey)))
@@ -61,20 +61,13 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n
},
}
answer, err := j.Run(ctx)
switch {
case errors.Is(err, link.ErrNoSeat):
// The path before the lease write: the controller's enrolment subject is granted to the token only while the
// controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said.
fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+
"subject, the path before the lease write, which the bus refuses once the lease is written\n", err)
return false, nil
case err != nil:
return true, err
if err != nil {
return err
}
fmt.Printf("\nenrolled as %s through the seat mesh-enrolment, dialled as %s and confirmed\n", answer.Node, answer.Login)
fmt.Printf(" identity %s\n", identityPath)
fmt.Println(" nothing is applied until the placement of this node names a module")
return true, nil
return nil
}
// keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is
cmd/mesh-host/healthowed_test.go
-1
View File
@@ -21,7 +21,6 @@ type owedBus struct {
rv map[string]uint64
}
func (b *owedBus) Report(context.Context, string, []byte) error { return nil }
func (b *owedBus) Health(context.Context, string, []byte) error { return nil }
func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil }
cmd/mesh-host/main.go
+12 -167
View File
@@ -100,7 +100,7 @@ func versionAt(self, stamped string) string {
const usage = `nox-mesh-host — the node-engine
A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The
operator's command is nox (mesh-cli), on any node, from their own account (novox/hq ADR 0272);
operator's command is nox (mesh-cli), on any node, from their own account, through the seats' verbs;
the subcommands here are for installing a machine, joining it, and rescuing one no mesh can
reach, run as root at the machine.
@@ -110,8 +110,7 @@ reach, run as root at the machine.
key make this machine's tunnel key, or read the one it made, and print the public
half: what its join token is issued for (novox/hq ADR 0169)
enrol --token T join the mesh — through the tunnel when the token was issued for a key
overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this
node's overlay key and the mesh is told, signed; --tunnel <iface> when several are up
overlay take refused: the controller that took a rekey ended and no seat takes one now
owned what this host has applied and still owns
hand-over <dir> hand a directory used as found to the mesh: the next apply gives it its declared
owner and mode. The operator's line is nox node hand-over <node> <dir> on the
@@ -809,167 +808,21 @@ func enrol(ctx context.Context, opts options) error {
detected := profile.Detect(ctx, profile.Default(nil), opts.timeout)
reported := profileAsReported(detected)
// Signed with the identity just generated, so the mesh can tell this machine from anyone else
// who knows its public key (novox/hq issue 083).
proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public,
sealing.Public, serving.Public))
// The token says where to go and which certificate that address must present. It says nothing
// about which bus is there, and does not need to: there is one, and this host knows which
// (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone).
//
// **It used to leave this empty** and mean "whatever the mesh runs today", which was true
// while two buses existed and became a refusal the moment one did: an empty transport is not
// the bus's name, so every enrolment ended at "this token is for the \"\" bus"
// (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus
// changed.
// Through the seat mesh-enrolment when the bus serves it to this token (novox/hq ADR 0587 §6): the controller's enrolment
// subject below stays for as long as the controller holds its lease.
if handled, err := enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath); handled || err != nil {
return err
}
reply, err := link.Enrol(ctx,
link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS},
*name, token.Secret,
mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found,
opts.timeout)
if err != nil {
return err
}
// The mesh's name for this node wins over what the machine called itself: the token was
// issued for a node record, and that record is what the identity binds to.
mine.Node = reply.Node
mine.Membership = identity.Membership{
Broker: firstNonEmpty(reply.Broker, token.Broker),
Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint),
Signer: firstNonEmpty2(reply.Signer, token.Signer),
Password: reply.Password,
// **Which bus this membership is for, said rather than left empty** (novox/hq
// 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name
// is not this one's — so a node enrolled without it came up and reconnected for ever
// against its own record: "this membership is for \"\", and the mesh's bus is nats". The
// reply does not carry it because there is one bus and the host knows which (ADR 0131);
// what was missing was writing that down where the link reads it.
Transport: link.OnNATS,
}
if mine.Membership.Password == "" {
// The mesh did not replace the token's secret, so it is still this node's broker
// password. Said rather than silently kept: a one-time secret living on as a credential
// is worth knowing about.
mine.Membership.Password = token.Secret
fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " +
"remains this node's credential")
}
// Saved only now, and only once the mesh has said it knows this node. A node holding an
// identity the mesh has never recorded would believe it had joined and be believed by
// nobody — worse than not having joined, because nothing would look wrong.
if err := identity.Save(identityPath, mine); err != nil {
return fmt.Errorf(
"the mesh accepted this node as %q and its identity could not be saved: %w\n"+
"That token is spent, so getting back needs a new one", reply.Node, err)
}
if !mine.Membership.Joined() {
return fmt.Errorf(
"the mesh accepted this node as %q but did not say how to reach it again, so this "+
"identity could not be used after a restart. Nothing was saved", reply.Node)
}
// Written before the identity, so a node that dies between the two has a key file with no
// identity — which enrols again cleanly — rather than an identity naming a key that is not
// there, which looks joined and cannot come up.
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
[]byte(mine.Overlay.Private+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot write this node's overlay key: %w", err)
}
if err := os.WriteFile(identity.SealingKeyPath(opts.state),
[]byte(sealing.Private+"\n"), 0o600); err != nil {
return fmt.Errorf("cannot write this node's sealing key: %w", err)
}
if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil {
return fmt.Errorf("cannot write this node's serving key: %w", err)
}
fmt.Printf("\nenrolled as %s\n", reply.Node)
fmt.Printf(" identity %s\n", identityPath)
fmt.Printf(" queue %s\n", reply.Queue)
return nil
// Through the seat mesh-enrolment (novox/hq ADR 0587 §6), the one way in: the controller's enrolment subject ended
// with the controller (issue 1126).
return enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath)
}
// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over
// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a
// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them
// every credential the mesh sealed to it.
func overlayCommand(ctx context.Context, opts options) error {
// overlayCommand is `mesh-host overlay take`: a node taking the tunnel found on its machine over after it enrolled
// (novox/hq ADR 0105). Refused, and nothing is changed: the rekey it sent was taken by the controller alone, which
// ended by its unassignment (issue 1126), and no seat takes one now; a key written here that no part of the mesh knows
// would raise the mesh's interface on a key its hub does not peer.
func overlayCommand(_ context.Context, opts options) error {
if opts.file != "take" {
return errors.New("overlay take [--tunnel <iface>] — the one thing `overlay` does here")
}
identityPath := identity.Path(opts.state)
mine, err := identity.Load(identityPath)
if err != nil {
return err
}
found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel)
if err != nil {
return fmt.Errorf("%w. Nothing was changed", err)
}
taken, rekey, err := rekeyOnto(mine, found)
if err != nil {
return err
}
fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public)
fmt.Printf(" identity, sealing and serving keys are untouched\n")
// Told first, then written: a mesh told and a machine not yet written is put right by running
// this again (the mesh refuses the stale second rekey and changes nothing; the files are
// rewritten the same). A machine written and a mesh not told would raise the mesh's interface
// on a key the mesh does not know at the next restart.
if err := link.Publish(ctx, link.Membership{
Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint,
Password: mine.Membership.Password, Signer: mine.Membership.Signer,
Transport: mine.Membership.Transport,
}, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil {
return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err)
}
fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node)
if err := os.WriteFile(identity.OverlayKeyPath(opts.state),
[]byte(taken.Overlay.Private+"\n"), 0o600); err != nil {
return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err)
}
if err := identity.Save(identityPath, taken); err != nil {
return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err)
}
fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n",
identity.OverlayKeyPath(opts.state))
fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint <host>:%d …`, `plan %s --json`, then push\n",
mine.Node, found.Port, mine.Node)
return nil
}
// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey
// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same
// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names
// is the one before the take, so the mesh can tell a repeat from a replay.
func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) {
overlay, err := identity.OverlayKeyFrom(found.PrivateKey())
if err != nil {
return identity.Identity{}, link.Rekey{}, err
}
previous := mine.Overlay.Public
if previous == overlay.Public && mine.OverlayBefore != "" {
previous = mine.OverlayBefore
}
taken := mine
taken.Overlay = overlay
if previous != overlay.Public {
taken.OverlayBefore = previous
}
presented := carried(found)
rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented}
rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented))
return taken, rekey, nil
return errors.New("overlay take is refused: the controller that took a rekey ended (issue 1126) and no seat " +
"takes one now. Nothing was changed")
}
// carried is a found tunnel as it is presented to the mesh: everything but its private key.
@@ -1120,9 +973,6 @@ func runLink(ctx context.Context, opts options) error {
Membership: membership,
// And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report.
Keys: recordKeys(opts.state, trust.Trusts),
// And no controller's key: every ask the controller would sign is refused, never verified with the membership's
// signer (novox/hq ADR 0563 §7; issue 1126, the controller ended by its unassignment).
ControllerKey: noControllerKey,
// And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8).
HaveWritten: proof.haveWritten,
// And the operator's hand-over of a directory used as found, carried by the controller from the
@@ -1213,9 +1063,6 @@ func runLink(ctx context.Context, opts options) error {
}, BusyAskEvery)
// And the core builds this host placed are judged, whichever host placed them (to-be 45 §8).
go watchWhatThisHostPlaced(aside, mine.Node, queue, say)
// And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller
// (novox/hq ADR 0272).
go serveMeshCLI(aside, queue, say)
// And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR
// 0349 §4).
go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say)
@@ -1236,8 +1083,6 @@ func runLink(ctx context.Context, opts options) error {
// be a control surface on every machine, reachable by anything that can reach the machine, in
// exchange for saving twenty seconds — and the whole security argument rests on there not being
// one. A signal is delivered by the service manager to a process it already supervises.
// (mesh-cli's socket is no such surface: it has no network address and nothing sent to it acts on
// the machine — it is asked of the controller, novox/hq ADR 0272 §3.)
//
// SIGHUP, because that is the signal a long-running program conventionally reads as *look again*,
// and nothing here is being reloaded from a file that a different signal would suit better.
cmd/mesh-host/meshcli.go
-25
View File
@@ -1,25 +0,0 @@
package main
import (
"context"
"git.novox.be/novox/mesh-sdk/go/cli"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/meshcli"
)
// serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account
// from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served
// is said once, and this engine goes on applying; mesh-cli then says the engine does not answer.
func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) {
if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil {
say("mesh-cli is not answered on this machine: " + err.Error())
}
}
// forwardOver asks the controller over the queue's link, and follows the line to what came of it.
func forwardOver(queue *link.Session) meshcli.Forward {
return func(ctx context.Context, asked meshcli.Asked) cli.Answer {
return meshcli.AskTheController(ctx, queue.AskController, asked)
}
}
cmd/mesh-host/nodeagent.go
+11 -47
View File
@@ -3,7 +3,6 @@ package main
import (
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -375,42 +374,6 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource {
return out
}
// operatorKeysFile is the file beside the node's state that holds the operator's public keys, one ed25519 key in base64 a
// line (a line that starts with # is a note). The switch's last declaration writes it as a file resource, beside the
// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption.
const operatorKeysFile = "operator-keys"
// trustedKeys are the keys a composition in the node's composition bucket may be signed by: the one given (none for the
// agent; the enrolment's signer for the user-list follower, which reads only where the bus's user-list file is from
// such a composition, never its list) and the operator's, read from operatorKeysFile each time it is asked, so a key
// that arrives while the agent runs is trusted at the next composition. A file that cannot be read trusts the one given
// alone, and says so.
func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey {
return func() []ed25519.PublicKey {
keys := []ed25519.PublicKey{}
if controller := controllerNow(); len(controller) == ed25519.PublicKeySize {
keys = append(keys, controller)
}
raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile))
if err != nil {
return keys
}
for _, line := range strings.Split(string(raw), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
key, err := base64.StdEncoding.DecodeString(line)
if err != nil || len(key) != ed25519.PublicKeySize {
fmt.Fprintf(os.Stderr, "mesh-host: a line of %s is not an ed25519 public key in base64, and is not trusted\n", operatorKeysFile)
continue
}
keys = append(keys, ed25519.PublicKey(key))
}
return keys
}
}
// keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say
// none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read.
func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) {
@@ -511,10 +474,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
}
}
// The bus's user list, the grants part's alone (ADR 0547 §8, clause 11; issue 1126), written into the file the
// bus module's composition in the node's bucket declares: on every node, and a node whose compositions carry no
// list writes nothing. Those compositions were signed with the enrolment's signer and nobody writes them now; they
// are read for the file's place alone, never for a list, until the place is read elsewhere (milestone B17).
users := userListFollower(opts, session, composedBucket, trustedKeys(opts, func() ed25519.PublicKey { return signer }), say, func() {
// bus module this node runs declares, as its agent holds what runs (its own composition, or the declaration kept
// before the switch): never read from the node's composition bucket, which the controller wrote. On every node;
// a node that runs no list writes nothing. Looked at again whenever what runs changed (RunsChanged, Settled).
users := userListFollower(opts, session, func() []delivery.Composed {
if now := current.get(); now != nil {
return now.Everything()
}
return nil
}, say, func() {
// This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few
// seconds: what failed in them is tried again once that is over, not at its next wait.
time.AfterFunc(reloadSettles, func() {
@@ -558,12 +526,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
Settled: func(module string) {
knockFilling(module+" settled", say)
judgeWhatRunsNow(opts.state, signer)
users.RunsChanged()
},
// And once the haves are read at the start, and whenever a module is taken as running without an apply, so
// a refusal from the engine's start does not wait for some module to settle (issue 924).
RunsChanged: func(why string) {
knockFilling(why, say)
judgeWhatRunsNow(opts.state, signer)
users.RunsChanged()
},
Say: say,
// A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min.
@@ -648,14 +618,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
shadowHooks := hooks
shadowHooks.refused, shadowHooks.certify = nil, nil
shadowEnv := composerEnv(session, node, say, shadowHooks)
// And, after each answer, what runs of an own secret the answer says the vault holds another value of: handed to
// the vault only while a node applied the controller's compositions (issue 1126, cause C4; ownresync), so nothing now.
composes := &composesNow{}
resync := ownResync(session, node, say, shadowEnv, composes)
shadow := selfcompose.NewShadow(shadowEnv,
func(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
session.NoteCandidates(ctx, delivery.WithValuesDiffer(answers))
go resync.take(ctx, answers)
})
go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again
// Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's
@@ -664,7 +629,6 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus
composer := selfComposer(session, node, hooks, say)
machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774)
composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone
composes.set(delivery.ComposesOwn)
env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy
env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does
session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere")
cmd/mesh-host/nodeagent_integration_test.go
+3 -17
View File
@@ -5,7 +5,6 @@ package main
import (
"context"
"crypto/ed25519"
"encoding/base64"
"encoding/json"
"os"
"path/filepath"
@@ -45,7 +44,6 @@ type itBed struct {
opts options
session *link.Session
controll ed25519.PrivateKey
operator ed25519.PrivateKey
said []string
// refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met.
refusals []string
@@ -137,20 +135,12 @@ func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed {
t.Fatal(err)
}
_, b.controll, _ = ed25519.GenerateKey(nil)
_, b.operator, _ = ed25519.GenerateKey(nil)
return b
}
// start runs the node agent under a session on the bus, trusting the controller's key, and the operator's when asked.
func (b *itBed) start(trustOperator bool) (stop func()) {
// start runs the node agent under a session on the bus.
func (b *itBed) start() (stop func()) {
b.t.Helper()
if trustOperator {
pub := b.operator.Public().(ed25519.PublicKey)
if err := os.WriteFile(filepath.Join(filepath.Dir(b.opts.state), operatorKeysFile),
[]byte("# the operator\n"+base64.StdEncoding.EncodeToString(pub)+"\n"), 0o600); err != nil {
b.t.Fatal(err)
}
}
b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second,
Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }}
b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode})
@@ -440,10 +430,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
builds := kvOf(delivery.BuilderBuildsBucket)
candidates := kvOf(delivery.PlacementCandidatesBucket)
packages := kvOf(link.PackagesBucket)
// The controller's stream the node's facts go into, so the node's key carries the facts it says.
if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil {
t.Fatal(err)
}
path := filepath.Join(b.dir, "web.conf")
man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` +
`"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` +
@@ -497,7 +483,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) {
t.Fatal(err)
}
stop := b.start(false)
stop := b.start()
// Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then
// places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine.
time.Sleep(300 * time.Millisecond)
cmd/mesh-host/nodekey.go
+4 -23
View File
@@ -26,12 +26,10 @@ const (
)
// heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of
// the user list on the link open now, told when the key is first heard or changes.
// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is.
type heardKey struct {
mu sync.Mutex
pem string
kick func()
mu sync.Mutex
pem string
}
// vaultKey is the vault's key this engine heard.
@@ -43,8 +41,7 @@ func (h *heardKey) get() string {
return h.pem
}
// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer
// with no key (a vault before this one) changes nothing.
// heard keeps the key a node-key answer gave. An answer with no key (a vault before this one) changes nothing.
func (h *heardKey) heard(answer json.RawMessage) {
var a struct {
VaultKey string `json:"vault_key"`
@@ -53,24 +50,8 @@ func (h *heardKey) heard(answer json.RawMessage) {
return
}
h.mu.Lock()
changed := a.VaultKey != h.pem
h.pem = a.VaultKey
kick := h.kick
h.mu.Unlock()
if changed && kick != nil {
kick()
}
}
// following names the follower told when the key changes, and tells it at once when a key is known.
func (h *heardKey) following(kick func()) {
h.mu.Lock()
h.kick = kick
known := h.pem != ""
h.mu.Unlock()
if known && kick != nil {
kick()
}
}
// nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the
cmd/mesh-host/nodekey_test.go
+6 -13
View File
@@ -162,29 +162,22 @@ func asksAgainUntilItHearsTheVaultsKey(t *testing.T) {
}
// The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its
// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one
// (an older vault) or with no public key changes nothing (ADR 0547 clause 9).
// answer gives is kept; an answer without one (an older vault) or with no public key changes nothing (ADR 0547 clause
// 9).
func TestTheNodeKeyContractIsThirteen(t *testing.T) {
if link.NodeKeyContract != 13 {
t.Errorf("NodeKeyContract %d", link.NodeKeyContract)
}
h := &heardKey{}
kicks := 0
h.following(func() { kicks++ })
h.heard(json.RawMessage(`{"recorded":true}`))
h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`))
if h.get() != "" || kicks != 0 {
t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks)
if h.get() != "" {
t.Fatalf("an answer with no key was kept: %q", h.get())
}
pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n"
answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem})
h.heard(answer)
h.heard(answer)
if h.get() != pem || kicks != 1 {
t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks)
}
h.following(func() { kicks += 10 })
if kicks != 11 {
t.Fatalf("a follower named after the key was heard was not told: %d", kicks)
if h.get() != pem {
t.Fatalf("key %q, want the key", h.get())
}
}
cmd/mesh-host/ownresync.go
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"context"
"encoding/json"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/ownresync"
"github.com/novox/mesh-host/internal/selfcompose"
)
// composesNow is whose compositions this run of the agent applies, once decided ("" before).
type composesNow struct {
mu sync.Mutex
now string
}
func (c *composesNow) set(s string) { c.mu.Lock(); c.now = s; c.mu.Unlock() }
func (c *composesNow) get() string { c.mu.Lock(); defer c.mu.Unlock(); return c.now }
// resyncStep is the step that hands the vault what runs of an own secret a shadow answer says another value of (issue
// 1126, cause C4; internal/ownresync): one pass at a time, and the last answers taken again once the agent has decided
// whose compositions it applies.
type resyncStep struct {
r *ownresync.Resync
one sync.Mutex
mu sync.Mutex
last map[string]delivery.CandidateAnswer
// tries and pending are the step taken again after a bound when a read or an ask the bus did not answer stopped it
// (Outcome.Retry, issue 1126, 2026-10-28), one wait at a time.
tries int
pending bool
}
func ownResync(session *link.Session, node string, say func(string), env selfcompose.Env, composes *composesNow) *resyncStep {
return &resyncStep{r: ownresync.New(ownresync.Env{Node: node, Composes: composes.get, Running: env.Running,
Get: env.Get, Unseal: env.Unseal, Seal: identity.Seal, Say: say,
Ask: func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error) {
return session.AskVault(ctx, verb, args)
}})}
}
// take keeps answers as the last and takes them.
func (s *resyncStep) take(ctx context.Context, answers map[string]delivery.CandidateAnswer) {
s.mu.Lock()
s.last = answers
s.mu.Unlock()
s.again(ctx)
}
// again takes the last answers, and once more after a bound (selfcompose.LaterWait) while the bus did not answer for one.
func (s *resyncStep) again(ctx context.Context) {
s.one.Lock()
defer s.one.Unlock()
s.mu.Lock()
last := s.last
s.mu.Unlock()
if last == nil {
return
}
retry := false
for _, o := range s.r.Take(ctx, last) {
retry = retry || o.Retry
}
s.mu.Lock()
defer s.mu.Unlock()
if !retry {
s.tries = 0
return
}
if s.pending || ctx.Err() != nil {
return
}
wait := selfcompose.LaterWait(s.tries)
s.tries++
s.pending = true
go func() {
select {
case <-ctx.Done():
case <-time.After(wait):
}
s.mu.Lock()
s.pending = false
s.mu.Unlock()
if ctx.Err() == nil {
s.again(ctx)
}
}()
}
cmd/mesh-host/rekey_test.go
-94
View File
@@ -1,94 +0,0 @@
package main
import (
"crypto/ed25519"
"strings"
"testing"
"github.com/novox/mesh-host/internal/identity"
"github.com/novox/mesh-host/internal/link"
"github.com/novox/mesh-host/internal/tunnel"
)
// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and
// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a
// proof signed by the identity key, over the previous key, the new one and the tunnel.
func anEnrolledNode(t *testing.T) identity.Identity {
t.Helper()
mine, err := identity.Generate("anchor")
if err != nil {
t.Fatal(err)
}
mine.Overlay, err = identity.GenerateOverlayKey()
if err != nil {
t.Fatal(err)
}
mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa",
Signer: make([]byte, ed25519.PublicKeySize), Password: "p"}
return mine
}
func aFoundTunnel(t *testing.T) tunnel.Found {
t.Helper()
private, err := identity.GenerateOverlayKey()
if err != nil {
t.Fatal(err)
}
found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" +
"Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n"))
if err != nil {
t.Fatal(err)
}
found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf"
return found
}
func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) {
mine := anEnrolledNode(t)
found := aFoundTunnel(t)
before := mine.Overlay.Public
taken, rekey, err := rekeyOnto(mine, found)
if err != nil {
t.Fatal(err)
}
if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() {
t.Fatal("the overlay key is not the tunnel's")
}
if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) ||
taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password ||
taken.Membership.Broker != mine.Membership.Broker {
t.Fatal("something other than the overlay key moved")
}
if taken.OverlayBefore != before {
t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore)
}
if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil ||
rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 {
t.Fatalf("the rekey does not say what moved: %+v", rekey)
}
if !ed25519.Verify(ed25519.PublicKey(mine.Public),
link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
t.Fatal("the rekey is not signed by this node's identity key over what it says")
}
if ed25519.Verify(ed25519.PublicKey(mine.Public),
link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) {
t.Fatal("the proof is not bound to the node")
}
for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} {
if strings.Contains(said, found.PrivateKey()) {
t.Fatal("the private key travels")
}
}
// Run again after the take — the mesh not yet told, or told and refused — the previous key it
// names is still the one before the take, so the mesh can tell a repeat from a replay.
again, second, err := rekeyOnto(taken, found)
if err != nil {
t.Fatal(err)
}
if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey {
t.Fatalf("a take run again does not name the key before the first: %+v", second)
}
}
cmd/mesh-host/trust.go
-4
View File
@@ -36,10 +36,6 @@ type meshTrust struct {
// newMeshTrust trusts nothing until the vault's key is read.
func newMeshTrust() *meshTrust { return &meshTrust{} }
// noControllerKey is the controller's key as this engine trusts it: none. A link told it refuses every ask signed as
// the controller's, where a link told nothing would verify it with the membership's signer (link.HandOverCheck).
func noControllerKey() ed25519.PublicKey { return nil }
// onChange names who is told when what it trusts changed.
func (t *meshTrust) onChange(f func()) {
t.mu.Lock()
cmd/mesh-host/userlist.go
+4 -6
View File
@@ -2,7 +2,6 @@ package main
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -26,13 +25,12 @@ import (
// userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the
// agent trusts, the write alone, and the have.
func userListFollower(opts options, session *link.Session, composedBucket string, trusted func() []ed25519.PublicKey,
func userListFollower(opts options, session *link.Session, running func() []delivery.Composed,
say link.Announce, wrote func()) *userlist.Follower {
return userlist.New(userlist.Env{
Node: session.Membership.Node,
Composed: bucketWatch{session, composedBucket},
Trusted: trusted,
OnDisk: userListOnDisk,
Node: session.Membership.Node,
Running: running,
OnDisk: userListOnDisk,
Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) {
digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say)
if changed && wrote != nil {
cmd/mesh-host/witnessing.go
+1 -2
View File
@@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool {
// watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8).
func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) {
host, _ := os.Hostname()
w := &witness.Watcher{
Root: apply.DaemonRoot(), Node: node, Host: host,
Root: apply.DaemonRoot(), Node: node,
Asker: func() link.Asker { return queue.Asker() },
Run: witness.Runner(apply.ExecRunner),
Hold: func(f func()) {
go.mod
+1 -1
View File
@@ -3,7 +3,7 @@ module github.com/novox/mesh-host
go 1.26.0
require (
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791
github.com/nats-io/nats-server/v2 v2.11.17
github.com/nats-io/nats.go v1.54.0
golang.org/x/crypto v0.57.0
go.sum
+2 -6
View File
@@ -1,9 +1,5 @@
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs=
git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
internal/apply/process_witness_test.go
+11 -11
View File
@@ -72,36 +72,36 @@ func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) {
t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles })
run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil }
oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"})
newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"})
controller := func(body []byte, digest, env string) string {
return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) +
`","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}`
oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"})
newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"})
runtime := func(body []byte, digest, env string) string {
return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) +
`","digest":"` + digest + `","run":["./node-tools"],"env":{"X":"` + env + `"}}`
}
_, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{},
_, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest, "1")), store.State{},
store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
_, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state,
_, state, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "1")), state,
store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run),
if _, err := witness.Restore(context.Background(), bundles, "node-tools", "never answered", witness.Runner(run),
time.Now()); err != nil {
t.Fatal(err)
}
// The mesh still declares the new build, and its unit changes: the process is re-placed.
report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state,
report, _, err := Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "2")), state,
store.OriginDeclared, run, nil, nil)
if err != nil {
t.Fatal(err)
}
if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" {
if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "old\n" {
t.Fatalf("the rolled-back build was placed again: %q", got)
}
if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") {
if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "rolled back on this machine") {
t.Fatalf("the apply does not say why it kept the build before: %+v", o)
}
}
internal/filling/filling.go
+5 -6
View File
@@ -35,10 +35,9 @@ import (
"time"
"git.novox.be/novox/mesh-sdk/go/declaration"
"github.com/novox/mesh-host/internal/meshcli"
)
// Socket is where the node-engine accepts a write of a filled file, beside mesh-cli's socket.
// Socket is where the node-engine accepts a write of a filled file.
const Socket = "/run/nox-mesh-host/fill.sock"
// MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive.
@@ -293,11 +292,11 @@ func UIDOf(account string) (uint32, error) {
return uint32(id), nil
}
// peerOf and sameNamespaces are the kernel's word on the asking process, as mesh-cli's socket reads it. Variables so
// peerOf and sameNamespaces are the kernel's word on the asking process (peer_linux.go). Variables so
// a test can say.
var (
peerOf = meshcli.PeerCredentials
sameNamespaces = meshcli.SameNamespaces
peerOf = peerCredentials
sameNamespaces = namespacesOf
)
// within bounds how long an accepted connection may take to say its request, and refusedWithin how long a busy one
@@ -352,7 +351,7 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("the filled files' socket directory: %w", err)
}
if problem := meshcli.NotOwnDirectory(dir); problem != "" {
if problem := notOwnDirectory(dir); problem != "" {
return fmt.Errorf("filled files are not written on request on this machine: %s", problem)
}
if err := os.Chmod(dir, 0o755); err != nil {
internal/filling/filling_test.go
+1 -1
View File
@@ -532,7 +532,7 @@ func TestTheSocketGoesOnAfterAnAcceptFails(t *testing.T) {
}
}
// The socket's directory is given the mode the socket needs, as mesh-cli's is.
// The socket's directory is given the mode the socket needs.
func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) {
dir, err := os.MkdirTemp("", "fill")
if err != nil {
internal/meshcli/peer_linux.go → internal/filling/peer_linux.go
+1 -1
View File
@@ -1,6 +1,6 @@
//go:build linux
package meshcli
package filling
import (
"errors"
internal/meshcli/peer_other.go → internal/filling/peer_other.go
+1 -1
View File
@@ -1,6 +1,6 @@
//go:build !linux
package meshcli
package filling
import (
"errors"
internal/link/asking.go
-53
View File
@@ -1,53 +0,0 @@
package link
import (
"context"
"fmt"
"time"
)
// The enrolment conversation, as the host's own words for it.
//
// **Its own seam rather than part of Link**, because almost nothing about it is the same. The
// credential is a one-time secret rather than this node's own; there is no declaration to hear; the
// whole exchange is a single question asked and possibly asked again. And the stakes differ: a node
// that fails here is not in the mesh at all, where a node that fails in Link has merely lost touch
// with one it belongs to.
// Approach is how a node reaches a mesh it does not yet belong to.
//
// The three things a token carries about where to go, and nothing about who is asking: the address,
// the certificate that address must present, and which bus is at the other end — the mesh's own
// (hearing.go), and a token naming any other is refused before anything is sent.
type Approach struct {
Address string
Fingerprint string
Transport string
}
// Asking is one open enrolment conversation.
type Asking interface {
// Ask puts the request to the mesh and waits for one answer, or says why none came.
//
// Called again, with the same bytes, while the mesh says "try again": the keys this node
// generated are the ones it keeps, so the same request is the same enrolment and the mesh holds
// the token for it (novox/hq issue 083).
Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error)
// Close lets go of the connection made with the token.
Close()
}
// Present opens an enrolment conversation with the mesh.
//
// The connection is made before anything is sent, and the certificate is checked while it is being
// made — so a node pointed at the wrong bus finds out before its token has left the machine (ADR
// 0004).
func Present(ctx context.Context, to Approach, node, secret string,
timeout time.Duration) (Asking, error) {
if to.Transport != OnNATS {
return nil, fmt.Errorf("this token is for the %q bus, and the mesh's bus is %s", to.Transport, OnNATS)
}
return presentNats(ctx, to, node, secret, timeout)
}
internal/link/asking_nats.go
-186
View File
@@ -1,186 +0,0 @@
package link
import (
"context"
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"time"
"github.com/nats-io/nats.go"
)
// The enrolment conversation on the bus being built.
//
// **The reply address is the whole of what changes**, and it changes for a reason the transport
// forces rather than a preference. Core NATS request/reply puts the caller's inbox in the message's
// reply field and a plain responder answers it — but this request goes into a stream, and a message
// a JetStream consumer delivers has had that field claimed for the consumer's own ack address. So by
// the time the controller reads the request, the transport's reply field names where the
// *controller* must acknowledge. Verified against a running server (design 25 §2).
//
// The address therefore travels as a field of the request, and this subscribes it before publishing:
// a node that published first could miss an answer to a question nobody was listening for.
// enrolInbox is where a node enrolling waits.
//
// Under `_INBOX.enrol.<node>.`, which is exactly what its enrolment user may subscribe and no
// wider — so an answer sealed to one machine cannot be read by another enrolling beside it. The
// random tail is this attempt's own: a reply left over from an attempt that timed out is not the
// answer to this question, which is what the correlation id does on the other transport.
func enrolInbox(node string) (string, error) {
tail := make([]byte, 8)
if _, err := rand.Read(tail); err != nil {
return "", fmt.Errorf("cannot make a reply address: %w", err)
}
return "_INBOX.enrol." + node + "." + hex.EncodeToString(tail), nil
}
type natsAsking struct {
conn *nats.Conn
js nats.JetStreamContext
inbox string
answers *nats.Subscription
lost chan error
}
func presentNats(_ context.Context, to Approach, node, secret string,
timeout time.Duration) (Asking, error) {
config, err := PinnedConfig(to.Fingerprint)
if err != nil {
return nil, err
}
inbox, err := enrolInbox(node)
if err != nil {
return nil, err
}
lost := make(chan error, 1)
// The user is this token's own — `enrol.<node>`, which may publish the enrolment subject and
// subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same
// string the request claims, so the server proves somebody holds the token and the request
// proves the same thing to the controller without it having to ask the server who connected.
// **Its own inbox space, because that is the only one it may listen in** (novox/hq
// 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the
// client picks, and the client's default is `_INBOX.<random>` — which this user may not
// subscribe to, so the enrolment failed with a permissions violation on a subject nobody had
// chosen. The permission is `_INBOX.enrol.<node>.>` (design 25 §6), so the client is told to
// pick its inboxes there; the reply address below is in the same space for the same reason.
conn, err := nats.Connect(natsURL(to.Address),
nats.Secure(config),
nats.CustomInboxPrefix("_INBOX.enrol."+node),
nats.UserInfo("enrol."+node, secret),
nats.Name("mesh-host/enrol/"+node),
nats.Timeout(timeout),
nats.NoReconnect(),
nats.DisconnectErrHandler(func(_ *nats.Conn, err error) {
select {
case lost <- fmt.Errorf("the bus closed the connection: %w", err):
default:
}
}),
)
if err != nil {
if errors.Is(err, ErrWrongCertificate) {
return nil, err
}
// Not quoted back with the credential: the secret is one-time and still a secret.
return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", to.Address, node, err)
}
js, err := conn.JetStream()
if err != nil {
conn.Close()
return nil, fmt.Errorf("the bus at %s has no JetStream: %w", to.Address, err)
}
// Subscribed before anything is published, so an answer cannot arrive before there is anywhere
// for it to land.
answers, err := conn.SubscribeSync(inbox)
if err != nil {
conn.Close()
return nil, fmt.Errorf("this node cannot listen for the mesh's answer: %w", err)
}
if err := conn.Flush(); err != nil {
conn.Close()
return nil, fmt.Errorf("this node's reply address did not reach the bus: %w", err)
}
return &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: lost}, nil
}
func (a *natsAsking) Close() {
if a.answers != nil {
_ = a.answers.Unsubscribe()
}
if a.conn != nil {
a.conn.Close()
}
}
// Ask publishes the request with this attempt's reply address written into it, and waits there.
func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) {
addressed, err := withReplyTo(request, a.inbox)
if err != nil {
return nil, err
}
publish, cancel := context.WithTimeout(ctx, wait)
defer cancel()
// Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be
// assumed, because the node has nothing else to go on.
//
// **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when
// an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second
// credential, which replaced the first, which is the one the node had already been given. The
// machine then reconnected for ever as a user whose password the mesh had rotated out from under
// it, and the controller's log said "enrolled anchor" twice in the same second.
//
// The id is the message: the same bytes carry the same id, so the stream discards the client's
// own retry, and a genuine second attempt — which carries a new reply address — is a different
// message and is let through.
sum := sha256.Sum256(addressed)
if _, err := a.js.Publish(EnrolSubject, addressed,
nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil {
return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err)
}
// Waited for rather than assumed. A published message that nothing answers means the controller
// is not running, and a node that carried on regardless would believe it had joined a mesh that
// has never heard of it.
//
// **The wait may legitimately be several store-window cycles long**: the controller naks the
// request with a delay while its store is restarting, and the node is waiting on the other side
// of that — which is exactly the combination that would have delivered the answer to a caller
// who had given up, had the address travelled in the transport's field.
answered, cancelAnswer := context.WithTimeout(ctx, wait)
defer cancelAnswer()
for {
msg, err := a.answers.NextMsgWithContext(answered)
switch {
case err == nil:
return msg.Data, nil
case errors.Is(err, context.DeadlineExceeded):
select {
case reason := <-a.lost:
return nil, reason
default:
}
return nil, fmt.Errorf(
"the bus accepted this node's connection and nothing answered within %s. The mesh's "+
"bus is running and its controller is not", wait)
case errors.Is(err, context.Canceled):
return nil, ctx.Err()
default:
select {
case reason := <-a.lost:
return nil, reason
default:
}
return nil, fmt.Errorf("waiting for the mesh's answer: %w", err)
}
}
}
internal/link/asking_nats_test.go
-192
View File
@@ -1,192 +0,0 @@
package link
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"time"
"github.com/nats-io/nats.go"
)
// The enrolment round trip against a real server.
//
// **This is the test that keeps a reason from becoming folklore.** The reply address travels in the
// request's payload because a JetStream consumer's delivery has had the transport's reply field
// claimed for its own ack address — which is a fact about a server, not a rule anybody can check by
// reading. Both halves are asserted here: that the field really is eaten, and that the answer
// reaches the node anyway.
//
// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js
// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNatsAnEnrolment
// asking is a conversation on a bus with no TLS. Built directly rather than through Present because
// the pin is what Present adds and PinnedConfig's own tests cover it; what is under test here is the
// address the answer comes back on.
func asking(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, node string) *natsAsking {
t.Helper()
inbox, err := enrolInbox(node)
if err != nil {
t.Fatal(err)
}
answers, err := conn.SubscribeSync(inbox)
if err != nil {
t.Fatal(err)
}
if err := conn.Flush(); err != nil {
t.Fatal(err)
}
a := &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: make(chan error, 1)}
// **Not a.Close.** Close closes the connection, and this connection is the test's own, shared with
// the stand-in. Cleanups run last-in first-out and the stand-in was made first, so its cleanup
// would run on a closed connection: Unsubscribe returns ErrConnectionClosed without deleting the
// consumer it made, and every request after that fails. The consumer then outlives the test on a
// workqueue stream, which allows one consumer per filter, and the next test cannot subscribe
// (issue 604). The connection is closed by aBus, last of all.
t.Cleanup(func() { _ = answers.Unsubscribe() })
return a
}
// theMeshAnswers stands in for the controller: it consumes the enrolment off the stream, reads the
// reply address out of the payload — never from the transport field — and answers there. It reports
// what the transport field actually held, which is the claim design 25 §2 rests on.
func theMeshAnswers(t *testing.T, conn *nats.Conn, js nats.JetStreamContext,
reply EnrolReply) <-chan string {
t.Helper()
sawReplyField := make(chan string, 1)
// **A consumer of this test's own, gone before it starts and gone when it ends.** The stand-in
// used to be one durable name shared by every test, released by Unsubscribe alone. Unsubscribe
// only asks the server to let go: it deletes the consumer only when this subscription made it,
// and the server drops the binding on its own time. A test that started in that window (or one
// that found the consumer left by an earlier run on a long-lived bus) was refused with "consumer
// is already bound to a subscription". Per-test names remove the sharing; the delete-and-wait
// before and after removes the dependence on the server's timing.
durable := standinName(t)
stream, err := js.StreamNameBySubject(EnrolSubject)
if err != nil {
t.Fatal(err)
}
standinGone(t, js, stream, durable)
sub, err := js.Subscribe(EnrolSubject, func(msg *nats.Msg) {
select {
case sawReplyField <- msg.Reply:
default:
}
var addressed struct {
ReplyTo string `json:"reply_to"`
}
if err := json.Unmarshal(msg.Data, &addressed); err != nil || addressed.ReplyTo == "" {
_ = msg.Ack()
return
}
body, _ := json.Marshal(reply)
// Published explicitly to the address the payload named, never msg.Respond — which would
// send it to whatever the transport's reply field holds, and that is the point.
_ = conn.Publish(addressed.ReplyTo, body)
_ = msg.Ack()
}, nats.Durable(durable), nats.ManualAck())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_ = sub.Unsubscribe()
standinGone(t, js, stream, durable)
})
return sawReplyField
}
// standinName is the stand-in's durable name for one test: the test's name with every character a
// consumer name may not hold replaced.
func standinName(t *testing.T) string {
t.Helper()
return "controller-standin-" + strings.Map(func(r rune) rune {
if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' {
return r
}
return '_'
}, t.Name())
}
// standinGone deletes the durable consumer if it exists and waits until the server no longer reports
// it, so that nothing after this call can meet its binding.
func standinGone(t *testing.T, js nats.JetStreamContext, stream, durable string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for {
_, err := js.ConsumerInfo(stream, durable)
if errors.Is(err, nats.ErrConsumerNotFound) {
return
}
if errors.Is(err, nats.ErrConnectionClosed) {
t.Fatalf("the connection closed before consumer %s was deleted: %v", durable, err)
}
_ = js.DeleteConsumer(stream, durable)
if time.Now().After(deadline) {
t.Fatalf("consumer %s on %s still exists 5s after it was deleted", durable, stream)
}
time.Sleep(20 * time.Millisecond)
}
}
// An enrolment is answered on the address the request carried, and the transport's own reply field
// held something else entirely.
func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) {
conn, js := aBus(t)
const node = "joining"
sawReplyField := theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node, Password: "p"})
a := asking(t, conn, js, node)
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
answer, err := a.Ask(context.Background(), request, 8*time.Second)
if err != nil {
t.Fatalf("no answer reached the node: %v", err)
}
var reply EnrolReply
if err := json.Unmarshal(answer, &reply); err != nil {
t.Fatal(err)
}
if !reply.Accepted || reply.Node != node {
t.Fatalf("the answer was not the mesh's: %+v", reply)
}
// And the field the answer would have gone to, had it used the transport's: the consumer's own
// ack address. If a future server stopped doing this, the payload-borne address would still
// work and this line is what would say the reason had changed.
select {
case field := <-sawReplyField:
if field == a.inbox {
t.Fatalf("the transport's reply field held this node's inbox (%s), so the payload "+
"address is no longer load-bearing — check design 25 §2 before relying on it", field)
}
if field == "" {
t.Fatal("the transport's reply field was empty rather than claimed, which is a third " +
"behaviour from the two design 25 §2 describes")
}
case <-time.After(2 * time.Second):
t.Fatal("the stand-in never saw the request")
}
}
// A request that names no reply address is not answered, and the node says so as a mesh that is not
// running rather than hanging. The controller has nowhere to send an answer, which is the failure
// the payload field exists to make impossible — asserted so that a request built without it fails
// loudly here rather than quietly on a machine.
func TestNatsAnEnrolmentWithNoReplyAddressIsNotAnswered(t *testing.T) {
conn, js := aBus(t)
const node = "silent"
theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node})
a := asking(t, conn, js, node)
// Published without going through Ask, so the reply address is genuinely absent.
request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"})
if _, err := js.Publish(EnrolSubject, request); err != nil {
t.Fatal(err)
}
if _, err := a.Ask(context.Background(), []byte(`{"node":"`+node+`"}`), 0); err == nil {
t.Fatal("a node with no answer coming was told it had one")
}
}
internal/link/bus.go
+5 -40
View File
@@ -10,28 +10,14 @@ import (
"github.com/nats-io/nats.go"
)
// Bus is what a host needs of the mesh's bus, in the mesh's own words.
//
// A host says exactly two things unprompted: what it applied, and that it is here. They are not
// the same kind of statement and the difference is the whole of this interface — one must arrive
// and one must not be insisted on.
//
// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own
// interface over its own client libraries, not a contract shared with the controller. The two
// agree because a conformance fixture holds them to one envelope, which is the only kind of
// agreement that survives being in different repositories.
type Bus interface {
// Report says what this node applied. **It must arrive.** A report that fails leaves the
// mesh believing the node never answered while the node believes it did, and the two go on
// disagreeing with nothing anywhere saying so — the shape of fault this project keeps
// finding. Returns false when it could not be delivered, so the caller can say so.
Report(ctx context.Context, node string, body []byte) error
}
// Bus is the link open now as the session holds it. What it can do is asked of it by what it implements (HaveBus,
// HealthBus, ToolBus, FilesBus): the report it once published into the controller's CONTROL stream went with the
// controller (issue 1126), and what the node applied is said in its haves.
type Bus interface{}
// --- The bus the mesh runs on today -----------------------------------------------------------
// OverNATS is the bus as a connection. A report goes through JetStream because it must survive
// the controller's store restarting; a heartbeat does not, because it must not.
// OverNATS is the bus as a connection.
type OverNATS struct {
Conn *nats.Conn
JS nats.JetStreamContext
@@ -39,11 +25,6 @@ type OverNATS struct {
Node string
}
// ReportSubject is this node's own, and no other node's: a host's account may
// publish `mesh.control.<its own node>.>` and nothing wider, so the subject is the authority on
// which node a report is about.
func ReportSubject(node string) string { return "mesh.control." + node + ".report" }
// HealthSubject is where this node says its long-running resources' health between reports (novox/hq
// ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a
// statement lost is said again within a minute while anything is not healthy.
@@ -65,11 +46,6 @@ func (b OverNATS) Health(ctx context.Context, node string, body []byte) error {
return b.Conn.FlushWithContext(flush)
}
// CLISubject is where this node's engine asks the controller a line `mesh-cli` was given on this machine
// (novox/hq ADR 0272 §3): its own node's, inside the grant every engine already has (`mesh.control.<node>.>`), so the
// bus server is the authority on which node asked. Core NATS, never the CONTROL stream, which names its own subjects.
func CLISubject(node string) string { return "mesh.control." + node + ".cli" }
// ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared
// tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other.
func ToolSubject(module, tool, node string) string {
@@ -90,17 +66,6 @@ func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte,
return msg.Data, nil
}
func (b OverNATS) Report(ctx context.Context, node string, body []byte) error {
// Into the CONTROL stream and awaited: this is the message the store-window guarantee is
// about (novox/hq ADR 0083). The controller naks with a delay while its store is away and
// the message is redelivered; a publish the bus never accepted must fail here rather than
// be assumed.
if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil {
return fmt.Errorf("reporting: %w", err)
}
return nil
}
// HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes.
// **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket
// is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every
internal/link/enginepackages.go
+13 -35
View File
@@ -17,10 +17,6 @@ import (
// mesh-host.packages (delivery.EnginePackagesRead).
const PackagesBucket = "mesh-host_" + delivery.EnginePackagesState
// MirrorPackagesBucket is the controller's declared-packages mirror (ADR 0507 §5), which the engine reads its own node's
// key of until the lease write, to compare (the node seat's verb packages).
const MirrorPackagesBucket = "mesh-controller_declared-packages"
// packagesStore binds PackagesBucket over the link open now: bound, never created.
func (l *natsLink) packagesStore() (nats.KeyValue, error) {
kv, err := l.js.KeyValue(PackagesBucket)
@@ -83,51 +79,33 @@ func (p EnginePackages) Put(_ context.Context, key string, value []byte) error {
return err
}
// PackagesView is the node seat's verb packages' answer (ADR 0565 §6): this node's key of the engine's state packages,
// and until the lease write the controller's mirror's key for the node and every package they hold differently.
// PackagesView is the node seat's verb packages' answer (ADR 0565 §6): this node's key of the engine's state packages.
// The controller's mirror it was once compared with ended with the controller (issue 1126).
type PackagesView struct {
Node string `json:"node"`
// Key is the node's key as the bus holds it (Source); null when the engine has written none.
Source string `json:"source"`
Key *delivery.DeclaredPackages `json:"key"`
KeyError string `json:"key_error,omitempty"`
// Mirror is the controller's mirror's key for the node (MirrorSource); null when it holds none or cannot be read.
MirrorSource string `json:"mirror_source"`
Mirror *delivery.DeclaredPackages `json:"mirror"`
MirrorError string `json:"mirror_error,omitempty"`
// Differences is every package the two hold differently, read only when both are; Agree when there is none.
Differences []delivery.PackageDifference `json:"differences"`
Agree bool `json:"agree"`
// Wrote says, for write, whether the key was written again (false: it already held what the engine composed).
Wrote *bool `json:"wrote,omitempty"`
Note string `json:"note,omitempty"`
}
// PackagesAnswer is the verb's answer from the two keys as read (nil: no key), each with its read's error.
func PackagesAnswer(node string, key, mirror []byte, keyErr, mirrorErr error) PackagesView {
v := PackagesView{Node: node, Source: delivery.EnginePackagesRead, MirrorSource: "mesh-controller.declared-packages",
Differences: []delivery.PackageDifference{}}
read := func(raw []byte, err error, into **delivery.DeclaredPackages, said *string) {
if err == nil && raw != nil {
var d delivery.DeclaredPackages
if err = json.Unmarshal(raw, &d); err == nil {
*into = &d
}
}
if err != nil {
*said = err.Error()
// PackagesAnswer is the verb's answer from the key as read (nil: no key), with its read's error.
func PackagesAnswer(node string, key []byte, keyErr error) PackagesView {
v := PackagesView{Node: node, Source: delivery.EnginePackagesRead}
if keyErr == nil && key != nil {
var d delivery.DeclaredPackages
if keyErr = json.Unmarshal(key, &d); keyErr == nil {
v.Key = &d
}
}
read(key, keyErr, &v.Key, &v.KeyError)
read(mirror, mirrorErr, &v.Mirror, &v.MirrorError)
switch {
case v.Key != nil && v.Mirror != nil:
v.Differences = delivery.DiffDeclaredPackages(*v.Key, *v.Mirror)
v.Agree = len(v.Differences) == 0
case v.Key == nil && v.KeyError == "":
v.Note = "the engine has written no key for this node yet: pacman reads the controller's mirror for it"
case v.Mirror == nil && v.MirrorError == "":
v.Note = "the controller's mirror holds no key for this node: nothing to compare"
case keyErr != nil:
v.KeyError = keyErr.Error()
case v.Key == nil:
v.Note = "the engine has written no key for this node yet"
}
return v
}
internal/link/enrol.go
+10 -180
View File
@@ -1,70 +1,21 @@
package link
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"time"
)
// The wire format shared with the control plane, which defines it separately because this binary
// requires nothing present and does not import it. A test on each side asserts the field names.
const (
Exchange = "mesh"
KeyEnrol = "enrol"
)
// What a joining machine keeps of the enrolment's words: the tunnel it found, as it travels, and how long it asks
// while the seat mesh-enrolment says to ask again (Redeem). The enrolment through the controller's subject on the
// CONTROL stream ended with the controller (issue 1126).
// QueueFor is the queue this node consumes from — the only one its account may read.
func QueueFor(node string) string { return "node." + node }
// EnrolRequest is what this node says when joining.
type EnrolRequest struct {
Node string `json:"node"`
Secret string `json:"secret"`
PublicKey []byte `json:"public_key"`
// OverlayKey is the public half of this node's key on the private network — a different key
// from PublicKey above, generated at the same moment and for a different purpose.
//
// Sent with enrolment because the overlay is the first declaration a node receives, and the
// mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled
// and unreachable for a round trip, which is the state everything else here works to avoid.
OverlayKey string `json:"overlay_key,omitempty"`
// SealingKey is the public half of the key secrets are sealed to. A third key, and the
// reasoning is the same one twice over: the mesh must be able to send this node something
// nothing else can read, and it must never be able to read it either.
SealingKey string `json:"sealing_key,omitempty"`
// ServingKey is the public half of the key this node serves TLS with on its internal name.
// The mesh signs a certificate binding it; the private half never leaves the machine, so
// there is nothing to seal and nothing that could be stolen from the mesh's copy.
ServingKey string `json:"serving_key,omitempty"`
Profile map[string]any `json:"profile,omitempty"`
// Proof is this node's identity key signing EnrolProof over this request: that the presenter
// holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish
// on a token this key already spent (novox/hq issue 083).
Proof []byte `json:"proof,omitempty"`
// ReplyTo is where the mesh's answer goes, as a field of the request rather than the
// transport's own reply address (design 25 §2). Written by the transport that needs it —
// withReplyTo, once per attempt — because a request going into a stream has had the transport's
// reply field claimed for the consumer's ack address before the controller ever reads it.
//
// Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the
// field means what it has always meant. Named here so both sides of the wire hold the same
// field name, which is what the shape test on each side is for.
ReplyTo string `json:"reply_to,omitempty"`
// Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR
// 0105): everything about it but that key. Sent with the keys because it is one of them —
// OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the
// hub's address, the range and the carried peers from it before the first declaration.
Tunnel *Tunnel `json:"tunnel,omitempty"`
// Approach is how a node reaches a mesh it does not yet belong to: the address, the certificate that address must
// present, and which bus is at the other end, the mesh's own (hearing.go); a token naming any other is refused before
// anything is sent.
type Approach struct {
Address string
Fingerprint string
Transport string
}
// Tunnel is a found tunnel as it travels: no private key.
@@ -86,36 +37,6 @@ type TunnelPeer struct {
Address string `json:"address"`
}
// EnrolReply is what the mesh says back.
type EnrolReply struct {
Accepted bool `json:"accepted"`
// TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the
// token is held for a moment by another enrolment — and that nothing was spent. The same
// request is asked again (novox/hq issue 083).
TryAgain bool `json:"try_again,omitempty"`
Node string `json:"node,omitempty"`
Queue string `json:"queue,omitempty"`
// What this node keeps so it can come back on its own. Without these a restart would need a
// person with a new token, which would make disconnection a crisis rather than the ordinary
// situation novox/hq ADR 0004 says it is.
Password string `json:"password,omitempty"`
Broker string `json:"broker,omitempty"`
Fingerprint string `json:"fingerprint,omitempty"`
Signer []byte `json:"signer,omitempty"`
Refusal string `json:"refusal,omitempty"`
}
// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it
// presents, so a proof cannot be moved to another request. The mesh builds the same bytes.
func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte {
return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) +
"\x00" + overlay + "\x00" + sealing + "\x00" + serving)
}
// ErrRefused is what a node gets when the mesh will not have it.
var ErrRefused = errors.New("the mesh refused this enrolment")
@@ -126,94 +47,3 @@ const EnrolPatience = 2 * time.Minute
// AskAgainAfter is the pause between asks while the mesh says "try again".
const AskAgainAfter = 3 * time.Second
// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking.
var ErrNotNow = errors.New("the mesh could not answer this enrolment")
// answered decides what one reply means: done, ask again, or stop with an error. Separate from the
// broker so it can be held to that by a test.
func answered(reply EnrolReply, asking time.Duration) (again bool, err error) {
switch {
case reply.Accepted:
return false, nil
case reply.TryAgain && asking < EnrolPatience:
return true, nil
case reply.TryAgain:
// Said with what to do. The mesh holds the token for this attempt's keys for as long as
// this node kept asking, so a new attempt — with keys of its own — waits that out first.
return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+
"enrol again with it; if it is then refused, issue a new one",
ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience)
default:
return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal)
}
}
// Enrol presents this node's key and its one-time secret, and waits to be told it is known.
//
// The broker has already authenticated this connection: the account was created when the token
// was issued and the secret is its password. So this is not how the node gets in — it is what it
// says once it is in, and the secret travels again because the control plane must not have to ask
// the broker who connected.
func Enrol(ctx context.Context, to Approach, node, secret string, public []byte,
overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte,
tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) {
asking, err := Present(ctx, to, node, secret, timeout)
if err != nil {
return EnrolReply{}, err
}
defer asking.Close()
request := EnrolRequest{Node: node, Secret: secret, PublicKey: public,
OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile,
Proof: proof, Tunnel: tunnel}
body, err := json.Marshal(request)
if err != nil {
return EnrolReply{}, err
}
// Asked, and asked again with the same request while the mesh says "try again": the keys this
// node generated are the ones it keeps, so the same request is the same enrolment, and the mesh
// holds the token for it (novox/hq issue 083).
began := time.Now()
for {
answer, err := asking.Ask(ctx, body, timeout)
if err != nil {
return EnrolReply{}, err
}
var reply EnrolReply
if err := json.Unmarshal(answer, &reply); err != nil {
return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err)
}
again, err := answered(reply, time.Since(began))
if err != nil {
return reply, err
}
if !again {
return reply, nil
}
select {
case <-ctx.Done():
return EnrolReply{}, ctx.Err()
case <-time.After(AskAgainAfter):
}
}
}
// withReplyTo writes this attempt's reply address into the request, as a field of its own.
//
// **Written into the bytes rather than carried beside them**, because the whole point is that the
// address survives a stream: a JetStream consumer's delivery has had the transport's reply field
// claimed for its own ack address, so a reply address that is not in the payload is one the
// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the
// field before marshalling, so one request can be asked again with a fresh address each time without
// the caller knowing that is what happens.
func withReplyTo(request []byte, inbox string) ([]byte, error) {
var fields map[string]any
if err := json.Unmarshal(request, &fields); err != nil {
return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err)
}
fields["reply_to"] = inbox
return json.Marshal(fields)
}
internal/link/enrol_again_test.go
-34
View File
@@ -1,34 +0,0 @@
package link
import (
"errors"
"strings"
"testing"
"time"
)
// What one reply means to a node enrolling (novox/hq issue 083): accepted is done; "try again" is
// asked again, until the node's patience runs out and it says the token was not spent; anything
// else is a refusal.
func TestAnEnrollingNodeAsksAgainWhileTheMeshSaysNotNow(t *testing.T) {
if again, err := answered(EnrolReply{Accepted: true}, 0); again || err != nil {
t.Fatalf("an accepted enrolment was not done: again=%v err=%v", again, err)
}
if again, err := answered(EnrolReply{TryAgain: true}, time.Second); !again || err != nil {
t.Fatalf("a mesh saying not now was not asked again: again=%v err=%v", again, err)
}
again, err := answered(EnrolReply{TryAgain: true, Refusal: "restarting"}, EnrolPatience)
if again || !errors.Is(err, ErrNotNow) {
t.Fatalf("a mesh saying not now past the node's patience did not stop it: again=%v err=%v", again, err)
}
if !errorsMention(err, "not spent") {
t.Errorf("giving up did not say the token can be used again: %v", err)
}
if again, err := answered(EnrolReply{Refusal: "that token cannot be used"}, 0); again || !errors.Is(err, ErrRefused) {
t.Fatalf("a refusal was not a refusal: again=%v err=%v", again, err)
}
}
func errorsMention(err error, what string) bool {
return err != nil && strings.Contains(err.Error(), what)
}
internal/link/enrol_proof_test.go
-13
View File
@@ -1,13 +0,0 @@
package link
import "testing"
// The bytes a node signs when it enrols. The mesh builds the same bytes to check the signature, in
// another repository; this known answer is repeated in its test, so the two cannot drift apart
// without one of them failing (novox/hq issue 083).
func TestWhatAnEnrollingNodeSignsIsFixed(t *testing.T) {
got := string(EnrolProof("s", []byte{1, 2, 3}, "o", "e", "v"))
if want := "novox-mesh-enrol\x00s\x00AQID\x00o\x00e\x00v"; got != want {
t.Fatalf("the enrolment proof's message changed: %q, want %q", got, want)
}
}
internal/link/enrol_seat.go
+6 -6
View File
@@ -18,8 +18,8 @@ import (
// Joining a mesh through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): the node-engine dials the bus as enrol.<node>
// with the certificate's fingerprint pinned, calls `redeem` with the public halves of the keys it made where it runs and a
// proof signed with its identity key, refuses an answer not signed by the key its token carries, keeps its membership, dials
// again as the login node-logins names for it, and calls `confirm`. The controller's enrolment subject (Enrol) stays for as long
// as the controller holds its lease.
// again as the login node-logins names for it, and calls `confirm`. It is the one way in: the controller's enrolment subject
// ended with the controller (issue 1126).
// EnrolmentSeat is the seat that adopts a new machine, and its two verbs a node-engine calls.
const (
@@ -33,8 +33,8 @@ func CalledSeatSubject(seat, verb, user string) string {
return "mesh.seat." + seat + ".call." + verb + "." + strings.ReplaceAll(user, ".", "~")
}
// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call (the
// controller still holds its lease) or no holder answers it.
// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call, or
// no holder answers it.
var ErrNoSeat = errors.New("the seat mesh-enrolment is not served to this machine's token")
// VerifyRedeemAnswer holds an answer to what it must be before anything is kept (ADR 0587 §6): it names the node asked for,
@@ -80,8 +80,8 @@ func seatAnswer(reply []byte) (json.RawMessage, error) {
}
// Redeem calls the seat's redeem as enrol.<node>, with the token's secret as the bus password, on the pinned certificate. It
// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), for
// as long as a machine asking the controller would (EnrolPatience). An answer is returned as the holder gave it: the caller
// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), , for
// EnrolPatience. An answer is returned as the holder gave it: the caller
// verifies it (VerifyRedeemAnswer) before it keeps anything.
func Redeem(ctx context.Context, to Approach, node, secret string, req delivery.Redeem, timeout time.Duration) (delivery.RedeemAnswer, error) {
if to.Transport != OnNATS {
internal/link/enrol_shape_test.go
-67
View File
@@ -1,67 +0,0 @@
package link
import (
"encoding/json"
"os"
"testing"
"github.com/novox/mesh-host/internal/identity"
)
// What this node says when it joins, written out so the mesh's own suite can accept it.
//
// The two ends are separate structs in separate repositories. Every field here is one somebody
// could rename on one side, and the failure would be silent: enrolment succeeds, a key is simply
// absent, and the node looks joined until the first thing sealed to it cannot be opened. That is
// exactly the shape of fault this project keeps finding late.
//
// Skipped unless MESH_ENROL_OUT names a file, so this is a check somebody runs deliberately
// rather than a dependency between two repositories.
func TestWhatThisNodeSaysWhenItJoins(t *testing.T) {
path := os.Getenv("MESH_ENROL_OUT")
if path == "" {
t.Skip("set MESH_ENROL_OUT to write the enrolment request the mesh's suite reads")
}
// A real one. Generated the way enrolment generates them rather than typed as literals, so a
// key that stopped being a key would be caught here rather than travelling.
mine, err := identity.Generate("workstation")
if err != nil {
t.Fatal(err)
}
overlay, err := identity.GenerateOverlayKey()
if err != nil {
t.Fatal(err)
}
sealing, err := identity.GenerateSealingKey()
if err != nil {
t.Fatal(err)
}
serving, err := identity.GenerateServingKey()
if err != nil {
t.Fatal(err)
}
request := EnrolRequest{
Node: "workstation",
Secret: "a-one-time-secret",
PublicKey: mine.Public,
OverlayKey: overlay.Public,
SealingKey: sealing.Public,
ServingKey: serving.Public,
Profile: map[string]any{"seat": true},
}
body, err := json.MarshalIndent(request, "", " ")
if err != nil {
t.Fatal(err)
}
if err := os.WriteFile(path, append(body, '\n'), 0o644); err != nil {
t.Fatal(err)
}
// The private half of the sealing key goes beside it, so the mesh's suite can prove what it
// sealed is openable rather than merely present.
if err := os.WriteFile(path+".sealing-private", []byte(sealing.Private), 0o600); err != nil {
t.Fatal(err)
}
t.Logf("wrote %s", path)
}
internal/link/handover.go
+3 -12
View File
@@ -76,13 +76,9 @@ const MaxHandOverBy = 200
// HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken.
type HandOverCheck struct {
Signer ed25519.PublicKey
// SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer:
// nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7,
// ADR 0550 §2).
SignerNow func() ed25519.PublicKey
Node string
Now func() time.Time
Nonces *Nonces
Node string
Now func() time.Time
Nonces *Nonces
}
// Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory —
@@ -162,11 +158,6 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod
return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error()
}
signer := check.Signer
if check.SignerNow != nil {
if signer = check.SignerNow(); signer == nil {
return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken"
}
}
if len(signer) != ed25519.PublicKeySize ||
!ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) {
return HandOverAsk{}, nil, "the ask is not signed by the mesh"
internal/link/handover_test.go
+15 -38
View File
@@ -148,20 +148,14 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing.
if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" {
t.Fatalf("the handler was given %v", given)
}
// ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key
// is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over.
trusted := m.Signer
following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node,
Now: func() time.Time { return now }, Nonces: &Nonces{}}
// Issue 1126: a link opened on the mesh has no signer (the controller that signed these asks ended), so an ask signed
// by the membership's key, the controller's, is refused and nothing is handed over.
before := len(given)
trusted = nil
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) }))
if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before {
t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a)
}
trusted = m.Signer
if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 {
t.Fatalf("while the controller holds the lease its ask was answered %+v", a)
opened := (&natsLink{node: m.Node}).check()
opened.Now, opened.Nonces = func() time.Time { return now }, &Nonces{}
late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Nonce = strings.Repeat("cd", 16) }))
if a := AnswerHandOver(opened, handle, late); !strings.Contains(a.Refused, "not signed by the mesh") || len(given) != before {
t.Fatalf("a link opened on the mesh answered an ask signed by the controller's key: %+v", a)
}
// A nonce is forgotten once its ask has expired, so the set does not grow for ever.
now = now.Add(HandOverGood + time.Second)
@@ -192,23 +186,17 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) {
// handingLink is a quiet link that hears hand-overs, keeping the handler it was given.
type handingLink struct {
*quietLink
given chan HandOverFunc
signer chan func() ed25519.PublicKey
given chan HandOverFunc
}
func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h }
// FollowSigner keeps what the link is told to verify the controller's asks with.
func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f }
// **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's
// judgement, and nothing else.
func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
m, _ := aMember(t)
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)}
l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)}
q := &Session{Membership: m, Timeout: time.Second}
// The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7).
q.ControllerKey = func() ed25519.PublicKey { return nil }
q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil }
ctx, stop := context.WithCancel(context.Background())
served := make(chan error, 1)
@@ -221,14 +209,6 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
case <-time.After(5 * time.Second):
t.Fatal("the link was never given the queue's hand-over")
}
select {
case f := <-l.signer:
if f() != nil {
t.Fatal("the link was given another key than the queue's")
}
case <-time.After(5 * time.Second):
t.Fatal("the link was never given the key the engine trusts now")
}
stop()
<-served
// 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on
@@ -241,15 +221,13 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
return map[string]any{"module": module, "digest": digest, "diff": diff}, nil
},
Compose: func(module string) (any, error) { return map[string]any{"composed": true, "retried": module}, nil },
// ADR 0565 §6, §15: the node's key beside the controller's mirror's, the mirror's from its writer's test.
// ADR 0565 §6, §15: the node's key of the engine's state packages.
Packages: func(write bool) (any, error) {
key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`)
mirror := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false},` +
`{"package":"nano","modules":["web"],"absent":true}]}`)
if !write {
return PackagesAnswer("n1", key, mirror, nil, nil), nil
return PackagesAnswer("n1", key, nil), nil
}
v := PackagesAnswer("n1", nil, nil, nil, errors.New("not granted"))
v := PackagesAnswer("n1", nil, nil)
wrote := true
v.Wrote = &wrote
return v, nil
@@ -265,11 +243,10 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) {
// The recovery verb compose (issue 774) makes a pass now, with a module's apply tried again when named.
{"compose", EngineToolSubject("compose", "n1"), `"composed":true,"retried":""`},
{"compose", EngineToolSubject("compose", "n1") + " " + `{"module":"route-proxy"}`, `"retried":"route-proxy"`},
// The read verb packages (ADR 0565 §6) names both sources and their difference; with write, what it wrote, and a
// key the engine has not written, or a mirror it may not read, is said and not an error.
{"packages", EngineToolSubject("packages", "n1"), `"differences":[{"package":"nano","absent":true,"engine":[],"mirror":["web"]}],"agree":false`},
// The read verb packages (ADR 0565 §6) names its source; with write, what it wrote, and a key the engine has not
// written is said and not an error.
{"packages", EngineToolSubject("packages", "n1"), `"source":"mesh-host.packages","key":{"node":"n1"`},
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"mirror":null,"mirror_error":"not granted","differences":[],"agree":false,"wrote":true`},
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"key":null,"wrote":true`},
{"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`},
{"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`},
} {
internal/link/have.go
+2 -3
View File
@@ -756,9 +756,8 @@ func (q *Session) NoteCandidates(ctx context.Context, answers map[string]deliver
q.writeHaves(ctx, bus)
}
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn or
// delivery.ComposesController) and why: written at each start of its agent, so a node returning to the controller's
// compositions says so where every have is read (issue 1126).
// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn) and why: written
// at each start of its agent, where every have is read (issue 1126).
func (q *Session) NoteComposes(ctx context.Context, composes, why string) {
q.init()
s := &q.haves
internal/link/have_test.go
+6 -27
View File
@@ -24,7 +24,7 @@ type haveLink struct {
hmu sync.Mutex
kv map[string]haveEntry
sets []string // keys written, in order
events []string // "report" and "have <key>", in the order they happened
events []string // "have <key>", in the order they happened
failing bool
binds int
setTries int
@@ -40,16 +40,6 @@ type haveEntry struct {
func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} }
func (l *haveLink) Report(ctx context.Context, node string, body []byte) error {
err := l.quietLink.Report(ctx, node, body)
if err == nil {
l.hmu.Lock()
l.events = append(l.events, "report")
l.hmu.Unlock()
}
return err
}
func (l *haveLink) Health(context.Context, string, []byte) error { return nil }
func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) {
@@ -241,9 +231,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) {
}
}
// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters
// and profile.
func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile.
func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
l := newHaveLink()
q, _ := aHaveQueue(l)
rep := appliedModules()
@@ -253,13 +242,6 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}}
rep.Profile = map[string]any{"cores": 8}
q.SayFacts(context.Background(), rep)
if len(l.events) < 2 || l.events[0] != "report" {
t.Fatalf("the report was not first: %v", l.events)
}
raw, _ := json.Marshal(l.said()[0])
if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") {
t.Errorf("the report carries the module facts: %s", raw)
}
got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value)
if err != nil {
t.Fatal(err)
@@ -310,16 +292,13 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) {
}
}
// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again.
func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) {
// 1132/AC7: a write the bucket refuses is said in the log, and the next apply writes again.
func TestAFailedHaveWriteIsSaidAndWrittenAgain(t *testing.T) {
l := newHaveLink()
l.failing = true
q, said := aHaveQueue(l)
if !q.SayFacts(context.Background(), appliedModules()) {
t.Fatal("the report did not go out")
}
if len(l.said()) != 1 {
t.Fatalf("reports sent: %d", len(l.said()))
t.Fatal("the facts were not said with a link open")
}
if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") {
t.Errorf("the failure is not in the log: %v", *said)
internal/link/hearing_nats.go
+10 -31
View File
@@ -8,7 +8,6 @@ import (
"os"
"regexp"
"strings"
"sync"
"time"
"github.com/nats-io/nats.go"
@@ -18,11 +17,6 @@ import (
// The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration
// (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts.
// EnrolSubject is where a joining machine asks. One subject for every node, because a machine
// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is
// the whole of what its enrolment user may do.
const EnrolSubject = "mesh.control.enrol"
// AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream.
// The answer is the facts event on its report subject — nobody's inbox is answered.
func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" }
@@ -39,14 +33,11 @@ func natsURL(address string) string {
// natsLink is this node's connection as a JetStream subscription.
type natsLink struct {
// mu guards signerNow (FollowSigner).
mu sync.Mutex
signerNow func() ed25519.PublicKey
conn *nats.Conn
js nats.JetStreamContext
asking *nats.Subscription
asked chan struct{}
handing *nats.Subscription
conn *nats.Conn
js nats.JetStreamContext
asking *nats.Subscription
asked chan struct{}
handing *nats.Subscription
// searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361).
searching *nats.Subscription
// judging hears the controller asking for a read-only judgement of this node's directories (issue 358).
@@ -117,7 +108,9 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e
}
l := &natsLink{
conn: conn, js: js, node: m.Node, user: user, signer: m.Signer,
// No signer: the asks these subjects carry were signed by the controller, which ended by its unassignment
// (issue 1126), so a link opened on the mesh believes none of them (ADR 0563 §7, ADR 0550 §2).
conn: conn, js: js, node: m.Node, user: user,
lost: make(chan error, 1),
asked: make(chan struct{}, 1),
refused: make(chan string, 64),
@@ -163,19 +156,9 @@ func (l *natsLink) hearAsks() {
}
}
// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the
// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2).
func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) {
l.mu.Lock()
l.signerNow = f
l.mu.Unlock()
}
// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust.
// check is how a signed ask is believed on this link: by the link's signer, which a link opened on the mesh has none of.
func (l *natsLink) check() HandOverCheck {
l.mu.Lock()
defer l.mu.Unlock()
return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node}
return HandOverCheck{Signer: l.signer, Node: l.node}
}
// HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh
@@ -323,10 +306,6 @@ func (l *natsLink) Close() {
}
}
func (l *natsLink) Report(ctx context.Context, node string, body []byte) error {
return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body)
}
func (l *natsLink) Health(ctx context.Context, node string, body []byte) error {
return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body)
}
internal/link/messages.go
+3 -67
View File
@@ -1,8 +1,6 @@
package link
import (
"strconv"
"strings"
"time"
"git.novox.be/novox/mesh-sdk/go/delivery"
@@ -47,13 +45,6 @@ type Report struct {
// machine saying what is true of it, not asking for anything.
Carried []int `json:"carried,omitempty"`
// Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand
// among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine,
// the sequence one higher for every facts event, so the controller refuses older facts by this order. On the
// wire as three top-level keys.
Writer string `json:"writer,omitempty"`
Order
// Held is what this adopted node found and is keeping as it was until its module is taken
// (novox/hq ADR 0100). Without it an adopted node reads as converged.
Held []Held `json:"held,omitempty"`
@@ -142,10 +133,6 @@ type Report struct {
// that does not load is a machine filtering nothing while its unit reports success.
Outward []string `json:"outward,omitempty"`
// Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq
// ADR 0105). Not an account of the machine: a report carrying one says nothing else.
Rekey *Rekey `json:"rekey,omitempty"`
// Rollbacks is what this machine's witnesses decided about a core build that was not healthy
// in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the
// engine, the engine about the controller and the node tools. **Said on every report while it
@@ -488,9 +475,8 @@ const WitnessContract = 1
// The core components a witness judges (to-be 45 §8), as a rollback names them.
const (
ComponentEngine = "node-engine"
ComponentController = "controller"
ComponentNodeTools = "node-tools"
ComponentEngine = "node-engine"
ComponentNodeTools = "node-tools"
)
// What a witness concluded, as a rollback says it.
@@ -506,7 +492,7 @@ const (
// RestoreFailed is a restoration that was attempted and did not complete.
RestoreFailed = "restore-failed"
// Unwitnessed is a build whose health could not be judged at all within the bound — the
// witness could not ask (no grant, no lease bucket) — so it is neither proved nor rolled back.
// witness could not ask (no grant, the bus away) — so it is neither proved nor rolled back.
Unwitnessed = "unwitnessed"
// Halted is the node-engine's launcher giving up: the build it would go back to fails too, so
// the fault is the machine's and not a build's.
@@ -531,26 +517,6 @@ type Rollback struct {
At time.Time `json:"at"`
}
// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its
// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one
// with nothing kept on disk.
type Order struct {
Epoch int64 `json:"epoch,omitempty"`
Sequence int64 `json:"sequence,omitempty"`
}
// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a
// Stringer promoted onto every report would print each one as its order alone.
func (o Order) Words() string {
switch {
case o.Epoch > 0:
return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10)
case o.Sequence > 0:
return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch"
}
return "no order"
}
// CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried
// states below; Note is what the host did about it, when it did something.
type CarriedTunnel struct {
@@ -573,36 +539,6 @@ const (
CarriedDown = "down"
)
// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling
// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel
// over, since re-enrolling would rotate every key it holds. Signed with the identity key over
// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key.
type Rekey struct {
// Previous is the overlay key this node held until now, as the mesh records it; the mesh
// refuses a rekey naming another, which is how a replayed one is refused.
Previous string `json:"previous"`
OverlayKey string `json:"overlay_key"`
Tunnel *Tunnel `json:"tunnel"`
Proof []byte `json:"proof"`
}
// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes
// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel.
// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof).
func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte {
var t Tunnel
if tunnel != nil {
t = *tunnel
}
peers := make([]string, 0, len(t.Peers))
for _, p := range t.Peers {
peers = append(peers, p.PublicKey+"@"+p.Address)
}
return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" +
t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" +
t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ","))
}
// Held is one file or container found on an adopted node and kept as it was.
type Held struct {
ID string `json:"id"`
internal/link/messages_test.go
+1 -4
View File
@@ -12,9 +12,6 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
value any
expect []string
}{
// novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence.
{Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}},
[]string{"node", "writer", "epoch", "sequence"}},
{Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"},
[]string{"node", "applied", "failed", "refused"}},
// novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what
@@ -27,7 +24,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) {
[]string{"protocol", "address", "port", "by", "published", "container-port"}},
// novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the
// witness contract this host keeps.
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract},
{Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract},
[]string{"node", "rollbacks", "witness"}},
{Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"},
[]string{"component", "from", "to", "outcome", "why", "at"}},
internal/link/run.go
-45
View File
@@ -3,7 +3,6 @@ package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"time"
@@ -199,12 +198,6 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
}
// And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue
// 356), answered on the link the question came over.
// Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7).
if f, ok := link.(interface {
FollowSigner(func() ed25519.PublicKey)
}); ok && queue.ControllerKey != nil {
f.FollowSigner(queue.ControllerKey)
}
if h, ok := link.(HandOvers); ok {
h.HearHandOvers(queue.HandOver)
}
@@ -290,41 +283,3 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann
}
}
}
// publishReport tells the mesh what this node did, and says whether the broker took it.
// Publish sends one report on this node's own connection and returns: the one-shot path for a
// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same
// account, the same pinned certificate and the same exchange as the running host's reports.
func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error {
link, err := Open(ctx, m, timeout)
if err != nil {
return err
}
defer link.Close()
var said string
if !publishReport(ctx, link, m, report, func(s string) { said = s }, timeout) {
return errors.New(said)
}
return nil
}
func publishReport(ctx context.Context, bus Bus, m Membership, report Report,
say Announce, timeout time.Duration) bool {
report.Node = m.Node
body, err := json.Marshal(report)
if err != nil {
say("cannot encode this node's own report: " + err.Error())
return false
}
publish, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
// Said rather than swallowed. A report that fails to publish leaves the mesh believing this
// node never answered, while the node believes it did — and the two would go on disagreeing
// with nothing anywhere saying so. That shape of fault is the one this project keeps finding.
if err := bus.Report(publish, m.Node, body); err != nil {
say(fmt.Sprintf("applied, and could not tell the mesh: %v", err))
return false
}
return true
}
internal/link/session.go
+9 -45
View File
@@ -2,7 +2,6 @@ package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
@@ -14,16 +13,11 @@ import (
"git.novox.be/novox/mesh-sdk/go/health"
)
// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program
// that wrote them, beside its epoch and sequence.
const Writer = "node-engine"
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks
// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts.
// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this
// machine over it, and what the machine says on it unasked: its health and its haves.
//
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an
// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles
// a whole declaration.
// **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here
// receives, orders or reconciles a whole declaration.
type Session struct {
// Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none.
Fallback func() *delivery.LoginFallback
@@ -54,9 +48,6 @@ type Session struct {
// MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of
// the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal.
MoveSecret MoveSecretFunc
// ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq
// ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's.
ControllerKey func() ed25519.PublicKey
// Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the
// operator asked and heard over it. Nil serves no seat.
Acts *HandActs
@@ -88,12 +79,8 @@ type Session struct {
haveAgainPending bool
haveAgainTries int
// saying is held while a facts event is made and said, so they leave in the order they are made.
// saying is held while a facts event is noted, so the haves are written in the order the facts are made.
saying sync.Mutex
// epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one
// without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2).
epoch int64
sequence int64
// haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132).
haves haveState
@@ -137,7 +124,6 @@ func (q *Session) init() {
if q.Say == nil {
q.Say = func(string) {}
}
q.epoch = time.Now().UnixMilli()
q.changed = make(chan struct{})
q.owed = make(chan struct{}, 1)
})
@@ -160,8 +146,10 @@ func (q *Session) oweHealth() {
}
}
// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them.
// A facts event made with no link open is not kept: the next link says the facts as they are then.
// SayFacts says this node's facts in its haves on the link open now, in the order they are made, and whether a link
// was open to say them on. A facts event made with no link open is not kept: the next link says the facts as they are
// then. They are said in the haves alone: the CONTROL stream they were also published on was the controller's, which
// ended by its unassignment (issue 1126). A write the bucket refuses is said and written again (issue 1132).
func (q *Session) SayFacts(ctx context.Context, r Report) bool {
q.init()
q.saying.Lock()
@@ -172,14 +160,7 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool {
if bus == nil {
return false
}
q.sequence++
r.Node = q.Membership.Node
r.Writer = Writer
r.Order = Order{Epoch: q.epoch, Sequence: q.sequence}
if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) {
return false
}
// After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132).
if q.noteApply(r) {
q.writeHaves(ctx, bus)
}
@@ -309,23 +290,6 @@ func (q *Session) AskTool(ctx context.Context, module, tool string) (health.Answ
return ReadToolAnswer(reply)
}
// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over
// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for.
func (q *Session) AskController(ctx context.Context, body []byte) ([]byte, error) {
q.init()
q.mu.Lock()
bus := q.bus
q.mu.Unlock()
if bus == nil {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}
asker, ok := bus.(ToolBus)
if !ok {
return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked)
}
return asker.Ask(ctx, CLISubject(q.Membership.Node), body)
}
// ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any
// other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272).
var ErrNotAsked = errors.New("not asked")
internal/link/session_test.go
+3 -74
View File
@@ -3,7 +3,6 @@ package link
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"fmt"
"os"
@@ -15,42 +14,16 @@ import (
"github.com/nats-io/nats.go"
)
// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled,
// as the bus's is ("reporting: context canceled"), or while refusing.
// quietLink is a link with no broker behind it.
type quietLink struct {
mu sync.Mutex
reports []Report
refusing bool
lost chan error
lost chan error
}
func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} }
func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error {
if err := ctx.Err(); err != nil {
return err
}
l.mu.Lock()
defer l.mu.Unlock()
if l.refusing {
return errors.New("refused")
}
var r Report
if err := json.Unmarshal(body, &r); err != nil {
return err
}
l.reports = append(l.reports, r)
return nil
}
func (l *quietLink) Lost() <-chan error { return l.lost }
func (l *quietLink) Close() {}
func (l *quietLink) said() []Report {
l.mu.Lock()
defer l.mu.Unlock()
return append([]Report(nil), l.reports...)
}
func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
t.Helper()
public, private, err := ed25519.GenerateKey(nil)
@@ -60,40 +33,6 @@ func aMember(t *testing.T) (Membership, ed25519.PrivateKey) {
return Membership{Node: "n1", Signer: public}, private
}
// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539
// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts.
func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) {
m, _ := aMember(t)
l := newQuietLink()
s := &Session{Membership: m, Timeout: time.Second}
s.Attach(context.Background(), l)
if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) {
t.Fatal("the facts were not taken")
}
got := l.said()
if len(got) != 2 {
t.Fatalf("%d facts events were said", len(got))
}
for _, r := range got {
if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 {
t.Fatalf("the facts do not say who wrote them: %+v", r)
}
}
if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 {
t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order)
}
// A later start orders after this one with nothing kept on disk.
time.Sleep(2 * time.Millisecond)
later := &Session{Membership: m, Timeout: time.Second}
later.Attach(context.Background(), l)
later.SayFacts(context.Background(), Report{})
if last := l.said()[2]; last.Epoch <= got[1].Epoch {
t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch)
}
}
// Facts made with no link open are not kept, and nothing is said: the next link says them as they are then.
func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
m, _ := aMember(t)
@@ -104,7 +43,7 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) {
}
s.Attach(context.Background(), l)
s.Detach(l)
if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 {
if s.Linked() || s.SayFacts(context.Background(), Report{}) {
t.Fatal("facts were said on a link that was let go")
}
}
@@ -140,16 +79,6 @@ func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) {
if err != nil {
t.Fatal(err)
}
want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"},
Retention: nats.WorkQueuePolicy}
if _, err := js.StreamInfo(want.Name); err != nil {
if _, err := js.AddStream(want); err != nil {
t.Fatal(err)
}
}
if err := js.PurgeStream(want.Name); err != nil {
t.Fatal(err)
}
return conn, js
}
internal/link/testdata/witness-grants.conf
Vendored
+3 -5
View File
@@ -3,16 +3,15 @@
# docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf
# MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness
#
# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a
# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool
# runtime, `admin` for the controller writing the lease.
# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host
# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime.
jetstream: enabled
accounts {
MESH {
jetstream: enabled
users = [
{ user: "node.anchor", password: "a", permissions: {
publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] }
publish: { allow: ["$SRV.PING.node-tools.anchor"] }
subscribe: { allow: ["_INBOX.node.anchor.>"] }
} }
{ user: "node.bare", password: "b", permissions: {
@@ -23,7 +22,6 @@ accounts {
subscribe: { allow: ["$SRV.PING.node-tools.>"] }
allow_responses: { max: 1, ttl: "1m" }
} }
{ user: "admin", password: "x" }
]
}
}
internal/link/witnessing.go
+4 -46
View File
@@ -10,14 +10,11 @@ import (
"github.com/nats-io/nats.go"
)
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the
// node tools it placed on this machine, by what the bus says about them rather than by what they
// say about themselves.
// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this
// machine, by what the bus says about them rather than by what they say about themselves.
//
// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read
// by JetStream's direct get — one subject, no stream information, nothing written; and this
// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's —
// so only this machine's runtime can answer it.
// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's
// PING by its name and this machine's, so only this machine's runtime can answer it.
// ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it,
// no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness
@@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered")
// Asker is what a witness asks through. The link open now implements it; nil while there is none.
type Asker interface {
// ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it —
// absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all.
ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error)
// Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer;
// ErrNoAnswer is none in time; ErrCannotAsk is no asking.
Ping(ctx context.Context, service, id string) error
}
// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the
// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard.
func DirectGetSubject(bucket, key string) string {
return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key
}
// PingSubject is the services protocol's PING of one instance of one service.
func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id }
@@ -58,36 +46,6 @@ func (q *Session) Asker() Asker {
return nil
}
func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) {
subject := DirectGetSubject(bucket, key)
before := l.conn.LastError()
msg, err := l.conn.RequestWithContext(ctx, subject, nil)
switch {
case errors.Is(err, nats.ErrNoResponders):
return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket)
case err != nil:
return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before))
}
if msg.Header != nil {
switch status := msg.Header.Get("Status"); status {
case "":
case "404":
return nil, false, nil
default:
return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key,
bucket, status, msg.Header.Get("Description"))
}
switch msg.Header.Get("KV-Operation") {
case "DEL", "PURGE":
return nil, false, nil
}
}
if len(msg.Data) == 0 {
return nil, false, nil
}
return msg.Data, true, nil
}
func (l *natsLink) Ping(ctx context.Context, service, id string) error {
subject := PingSubject(service, id)
before := l.conn.LastError()
internal/link/witnessing_nats_test.go
+2 -62
View File
@@ -2,7 +2,6 @@ package link
import (
"context"
"encoding/json"
"errors"
"os"
"testing"
@@ -12,7 +11,7 @@ import (
)
// What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8):
// whether the direct get and the PING work as the two subjects named, and whether a host without them
// whether the PING works as the subject named, and whether a host without them
// is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf.
func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
url := os.Getenv("MESH_TEST_NATS_GRANTS")
@@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Cleanup(conn.Close)
return conn
}
admin := dial("admin", "x", "_INBOX.admin")
js, err := admin.JetStream()
if err != nil {
t.Fatal(err)
}
_ = js.DeleteKeyValue("mesh-controller_lease")
kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1})
if err != nil {
t.Fatal(err)
}
anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")}
bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")}
ask := func() (context.Context, context.CancelFunc) {
return context.WithTimeout(context.Background(), 2*time.Second)
}
// Nobody holds it.
ctx, cancel := ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("an empty lease read as %v, %v", found, err)
}
cancel()
// Held.
if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder")
cancel()
if err != nil || !found {
t.Fatalf("a held lease read as %v, %v", found, err)
}
var holder struct {
Instance string `json:"instance"`
}
if json.Unmarshal(value, &holder); holder.Instance == "" {
t.Fatalf("the lease's value is %s", value)
}
// Given back.
if err := kv.Delete("holder"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found {
t.Fatalf("a lease given back read as %v, %v", found, err)
}
cancel()
// A host without the grant cannot ask — which is not "nobody holds it".
ctx, cancel = ask()
if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err)
}
cancel()
// No runtime on the bus: not there to answer.
ctx, cancel = ask()
ctx, cancel := ask()
if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) {
t.Fatalf("a PING nobody serves is %v, want no answer", err)
}
@@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) {
t.Fatalf("a host without the grant heard %v, want it unable to ask", err)
}
cancel()
// No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it".
if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil {
t.Fatal(err)
}
ctx, cancel = ask()
if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) {
t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err)
}
cancel()
}
internal/meshcli/ask.go
-92
View File
@@ -1,92 +0,0 @@
package meshcli
import (
"context"
"encoding/json"
"errors"
"fmt"
"time"
"git.novox.be/novox/mesh-sdk/go/cli"
"github.com/novox/mesh-host/internal/link"
)
// The controller answers a line as it answers every call (novox/hq ADR 0272, review): in an envelope, `result` or
// `error`, and within its window for an answer — so a line still running is answered `running` with its call, and
// followed by the same account on the same node until it ends. Its side holds the same field names (mesh-controller
// internal/link, the mesh-cli field names' test).
// ErrNotAsked is the link's: a request it knows it never delivered.
var ErrNotAsked = link.ErrNotAsked
// AskFunc asks the controller one request on this node's subject and returns its reply.
type AskFunc func(ctx context.Context, body []byte) ([]byte, error)
// followEvery is how often a line still running is asked about again.
var followEvery = 2 * time.Second
// envelope is a call's answer: its result, or its error.
type envelope struct {
Result json.RawMessage `json:"result"`
Error string `json:"error"`
}
// runningResult is what a call still running answers in its result.
type runningResult struct {
Running bool `json:"running"`
Call string `json:"call"`
}
// AskTheController asks the line and follows it to its answer, within ctx. What it says is one of three things,
// never confused: what the line printed; that nothing ran, and why (the request was never taken); or that what came
// of it is not known — it may have run — naming the call to look at when one is known.
func AskTheController(ctx context.Context, ask AskFunc, asked Asked) cli.Answer {
body, _ := json.Marshal(asked)
reply, err := ask(ctx, body)
if err != nil {
// **Only a request the link knows it never asked ran nothing**: no link open, nobody listening. Any other
// failure may come after the request was delivered, and the line may have run (review of ADR 0272).
if errors.Is(err, ErrNotAsked) {
return cli.Refusal("the controller was not reached, so nothing ran: " + err.Error())
}
return cli.NotKnown("", "no answer from the controller ("+err.Error()+")")
}
call := ""
for {
var env envelope
if err := json.Unmarshal(reply, &env); err != nil {
return cli.NotKnown(call, "the controller's answer could not be read")
}
if env.Error != "" {
return cli.NotKnown(call, "the controller answered "+env.Error)
}
var r runningResult
_ = json.Unmarshal(env.Result, &r)
if !r.Running {
var a cli.Answer
if err := json.Unmarshal(env.Result, &a); err != nil {
return cli.NotKnown(call, "the controller's answer could not be read")
}
return a
}
if r.Call != "" {
call = r.Call
}
if call == "" {
return cli.NotKnown("", "the controller says the line is running and names no call to follow")
}
// Asked again until it ends, through a link that may drop and come back: a failed follow is tried
// again, and only ctx ending stops it.
for {
select {
case <-ctx.Done():
return cli.NotKnown(call, fmt.Sprintf("no answer within the time mesh-cli waits (%s)", cli.AskWithin))
case <-time.After(followEvery):
}
body, _ := json.Marshal(Asked{Follow: call, Account: asked.Account, UID: asked.UID, Session: asked.Session})
if reply, err = ask(ctx, body); err == nil {
break
}
}
}
}
internal/meshcli/ask_test.go
-123
View File
@@ -1,123 +0,0 @@
package meshcli
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
"testing"
"time"
)
// controllerSays is a controller answering each request in turn from a script, and recording what it was asked.
type controllerSays struct {
mu sync.Mutex
script []func(Asked) ([]byte, error)
asked []Asked
}
func (c *controllerSays) ask(_ context.Context, body []byte) ([]byte, error) {
c.mu.Lock()
defer c.mu.Unlock()
var a Asked
_ = json.Unmarshal(body, &a)
c.asked = append(c.asked, a)
if len(c.script) == 0 {
return nil, context.DeadlineExceeded
}
next := c.script[0]
if len(c.script) > 1 {
c.script = c.script[1:]
}
return next(a)
}
func result(v any) func(Asked) ([]byte, error) {
return func(Asked) ([]byte, error) { return json.Marshal(map[string]any{"result": v}) }
}
func fastFollow(t *testing.T) {
was := followEvery
followEvery = time.Millisecond
t.Cleanup(func() { followEvery = was })
}
// A line still running when the controller first answers is followed, by its call, to what it printed.
func TestALineStillRunningIsFollowedToItsAnswer(t *testing.T) {
fastFollow(t)
running := result(map[string]any{"running": true, "call": "call-9"})
c := &controllerSays{script: []func(Asked) ([]byte, error){running, running, running,
result(map[string]any{"stdout": []byte("pushed\n"), "exit": 0, "terminal": true})}}
a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000})
if string(a.Stdout) != "pushed\n" || !a.Terminal || a.Refused != "" || a.Unknown != "" {
t.Fatalf("followed to %+v", a)
}
for _, f := range c.asked[1:] {
if f.Follow != "call-9" || f.Account != "op" || len(f.Line) != 0 {
t.Fatalf("a follow asked %+v", f)
}
}
}
// **No answer in time is never "nothing ran"** (review of ADR 0272): the line was asked and may have run, and the
// call to look at is named when it is known.
func TestNoAnswerInTimeSaysTheLineMayHaveRun(t *testing.T) {
fastFollow(t)
c := &controllerSays{}
a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"push"}, Account: "op"})
if a.Refused != "" || !strings.Contains(a.Unknown, "may have run") || a.Exit == 0 {
t.Fatalf("a request with no answer reads %+v", a)
}
c = &controllerSays{script: []func(Asked) ([]byte, error){result(map[string]any{"running": true, "call": "call-3"}),
func(Asked) ([]byte, error) { return nil, context.DeadlineExceeded }}}
ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
defer cancel()
a = AskTheController(ctx, c.ask, Asked{Line: []string{"push"}, Account: "op"})
if a.Refused != "" || !strings.Contains(a.Unknown, "call-3") || !strings.Contains(a.Unknown, "may have run") {
t.Fatalf("a running line never answered reads %+v", a)
}
}
// A request the bus never carried to a controller ran nothing, and says so.
func TestARequestNobodyTookSaysNothingRan(t *testing.T) {
c := &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) {
return nil, fmt.Errorf("%w: nothing on this machine answers mesh.control.laptop.cli", ErrNotAsked)
}}}
a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"status"}, Account: "op"})
if !strings.Contains(a.Refused, "nothing ran") || a.Unknown != "" {
t.Fatalf("a request nobody took reads %+v", a)
}
c = &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) {
return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked)
}}}
if a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"status"}}); !strings.Contains(a.Refused, "nothing ran") {
t.Fatalf("no link reads %+v", a)
}
}
// **Any error once the request may have been delivered is not known, never "nothing ran"** (review of ADR 0272):
// only a request the link says it never asked — no link open, nobody listening — ran nothing.
func TestOnlyARequestNeverAskedSaysNothingRan(t *testing.T) {
for _, c := range []struct {
err error
nothing bool
}{
{fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked), true},
{fmt.Errorf("%w: nothing on this machine answers mesh.control.laptop.cli", ErrNotAsked), true},
{errors.New("nats: connection closed"), false},
{errors.New("mesh.control.laptop.cli: permissions violation"), false},
{context.DeadlineExceeded, false},
} {
err := c.err
ctl := &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) { return nil, err }}}
a := AskTheController(context.Background(), ctl.ask, Asked{Line: []string{"push"}, Account: "op"})
if c.nothing && (!strings.Contains(a.Refused, "nothing ran") || a.Unknown != "") {
t.Errorf("%v reads %+v; it was never asked", err, a)
}
if !c.nothing && (a.Refused != "" || !strings.Contains(a.Unknown, "may have run")) {
t.Errorf("%v reads %+v; it may have been asked", err, a)
}
}
}
internal/meshcli/deadline_test.go
-42
View File
@@ -1,42 +0,0 @@
package meshcli
import (
"context"
"net"
"strings"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/cli"
)
// **A connection that says nothing is bounded from the moment it is accepted** (review of ADR 0272): its deadline is
// set before any lookup, so it is answered and closed within readWithin, holds no slot after, and asks nothing.
func TestAConnectionThatSaysNothingIsAnsweredWithinItsDeadline(t *testing.T) {
was := readWithin
readWithin = 200 * time.Millisecond
t.Cleanup(func() { readWithin = was })
asked := false
socket := serving(t, func(context.Context, Asked) cli.Answer {
asked = true
return cli.Answer{}
})
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
a, err := cli.ReadAnswer(conn)
if err != nil {
t.Fatalf("a silent connection was not answered within its deadline: %v", err)
}
if asked || a.Refused == "" || !strings.Contains(a.Refused, "nothing was asked") {
t.Fatalf("a silent connection was asked (%v) or answered %+v", asked, a)
}
// Its slot was given back: the account's next line is not busy.
b, err := cli.Ask(context.Background(), socket, []string{"status"})
if err != nil || strings.Contains(b.Refused, "busy") || !asked {
t.Fatalf("the next line after a silent one reads %+v (%v), asked %v", b, err, asked)
}
}
internal/meshcli/meshcli.go
-339
View File
@@ -1,339 +0,0 @@
// Package meshcli is the node-engine's side of `mesh-cli` (novox/hq ADR 0272): a socket on this machine that any
// account may connect to, the account read from the kernel, and the line asked of the controller.
//
// **The node-engine vouches; it does not decide.** It adds what only it can know — which account on this machine
// is asking, from the kernel's peer credentials, never from anything the program says — and sends the request out
// over its own link, on its own node's subject. The controller judges whether the line runs as its terminal, as an
// ordinary call, or not at all (ADR 0272 §4). Nothing received here is run or applied on the machine.
//
// **No listening control surface** (novox/hq ADR 0004, ADR 0272 §3): the socket has no network address, so
// nothing on another machine reaches it, and nothing sent to it acts on this one.
package meshcli
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"git.novox.be/novox/mesh-sdk/go/cli"
)
// Asked is what the node-engine sends the controller on its own node's subject: the line, and the account the
// kernel says is asking. The controller's side holds the same field names (a test on each side).
type Asked struct {
Line []string `json:"line,omitempty"`
// Account is the login of the user id the kernel gave, from this machine's user database.
Account string `json:"account"`
// UID is the user id itself, so a renamed or duplicated login is still the same id.
UID uint32 `json:"uid"`
// Follow is a call a line runs as, asked about again by the account that asked it (AskTheController).
Follow string `json:"follow,omitempty"`
// Session is the login session the asking process runs in — its systemd scope, `session-<id>.scope` under the
// account's own slice — as the kernel's cgroup says it; empty for anything else: a system service, a user unit,
// a process another account's session started. Only a login session is a person at a terminal (novox/hq ADR
// 0272): the controller makes nothing else the terminal.
Session string `json:"session,omitempty"`
// Stdin is what mesh-cli's standard input held, at most cli.MaxStdin: carried to the controller with the line
// and nowhere else. The engine says it in no journal, keeps it in no state, and never sends it again: a follow
// carries none. The controller gives it to a line that runs as its terminal alone.
Stdin []byte `json:"stdin,omitempty"`
}
// Forward asks the controller and follows the line to what came of it (AskTheController): what it printed, that
// nothing ran and why, or that it is not known.
type Forward func(ctx context.Context, asked Asked) cli.Answer
// readWithin bounds how long a connection may take to say its line: a program that connects and says nothing
// holds nothing for long. A variable so a test need not wait.
var readWithin = 30 * time.Second
// refusedWithin bounds how long the engine waits for the request of a line it refuses before it is asked: a client
// that has connected writes its request within microseconds, so a refused connection holds nothing for long. Never
// longer than readWithin. A variable so a test can say.
var refusedWithin = 1 * time.Second
// refusalsWaiting bounds how many refusals wait for their request at the same time, on the whole engine; drainCap
// bounds how much of a refused request is read, and it is read into a small buffer and dropped. The socket is open
// to every account, so a refusal must cost the engine little however many connections ask (review of ADR 0272).
// Variables so a test can say.
var (
refusalsWaiting = make(chan struct{}, 32)
drainCap = 4 << 10
)
// drain reads and drops what a refused client wrote, up to the end of its request line, the end of the stream, the
// connection's deadline or drainCap bytes, whichever comes first. Nothing is kept.
func drain(r io.Reader) {
var buf [512]byte
for read := 0; read < drainCap; {
n, err := r.Read(buf[:min(len(buf), drainCap-read)])
read += n
if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil {
return
}
}
}
// atOnce bounds how many lines are asked at the same time, and perAccount how many of them one account's: a person
// types one at a time, so more is a program on the machine. One over either bound is answered busy at once — the
// engine never stops accepting for it, so one account cannot hold the socket from every other (review of ADR 0272).
var (
atOnce = 8
perAccount = 2
)
// slots counts the lines being asked, in all and per user id.
type slots struct {
mu sync.Mutex
all int
byUID map[uint32]int
}
func (s *slots) take(uid uint32) bool {
s.mu.Lock()
defer s.mu.Unlock()
if s.all >= atOnce || s.byUID[uid] >= perAccount {
return false
}
s.all++
s.byUID[uid]++
return true
}
func (s *slots) give(uid uint32) {
s.mu.Lock()
defer s.mu.Unlock()
s.all--
if s.byUID[uid]--; s.byUID[uid] <= 0 {
delete(s.byUID, uid)
}
}
// peerOf reads the connecting process's credentials from the kernel: its user id and process id. A variable so a
// test can say what the kernel would; it is the only source of the account.
var peerOf = peerCredentials
// sessionOf reads the login session a process runs in, from the kernel. A variable so a test can say.
var sessionOf = func(pid int32, uid uint32) string {
raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid))
if err != nil {
return ""
}
return userSessionIn(string(raw), uid)
}
// sessionsDir is where logind keeps a record of each session it holds. A variable so a test can say.
var sessionsDir = "/run/systemd/sessions"
// userSessionIn is the login session a cgroup names, when logind's own record of it says it is a person's: this
// account's (UID), of class `user` — not a greeter's or the user manager's — and of this very scope. The cgroup
// alone names a scope; only the record says whose session it is. No record, or one that does not say so, is no
// session.
func userSessionIn(cgroup string, uid uint32) string {
scope := loginSessionIn(cgroup, uid)
id, ok := strings.CutPrefix(strings.TrimSuffix(scope, ".scope"), "session-")
if !ok || id == "" || strings.ContainsAny(id, "/.") {
return ""
}
raw, err := os.ReadFile(filepath.Join(sessionsDir, id))
if err != nil {
return ""
}
// **The record is systemd's private format**, not an interface it promises: `KEY=value` lines logind writes for
// itself. Only UID, CLASS and SCOPE are read, each at its first occurrence — logind writes each once, and a later
// line saying otherwise is not the record's word. A systemd that changes the format, renames a key or stops
// writing the file fails closed here: no session is read, so nobody is the terminal until this reads it again.
record := map[string]string{}
for _, line := range strings.Split(string(raw), "\n") {
k, v, ok := strings.Cut(line, "=")
if !ok {
continue
}
if _, said := record[k]; !said {
record[k] = v
}
}
if record["UID"] != strconv.FormatUint(uint64(uid), 10) || record["CLASS"] != "user" || record["SCOPE"] != scope {
return ""
}
return scope
}
// loginSessionIn is the login session scope a process's cgroup names, for its own account: the unified hierarchy's
// line, `0::/user.slice/user-<uid>.slice/session-<id>.scope`, and nothing below or beside it. Anything else is no
// login session — a service under system.slice, a unit under the account's user manager (user@<uid>.service), a
// scope of another account's.
func loginSessionIn(cgroup string, uid uint32) string {
for _, line := range strings.Split(cgroup, "\n") {
path, ok := strings.CutPrefix(line, "0::")
if !ok {
continue
}
prefix := fmt.Sprintf("/user.slice/user-%d.slice/", uid)
scope, ok := strings.CutPrefix(path, prefix)
if !ok || strings.Contains(scope, "/") || !strings.HasPrefix(scope, "session-") || !strings.HasSuffix(scope, ".scope") {
return ""
}
return scope
}
return ""
}
// sameNamespaces refuses a peer whose mount or user namespace is not the engine's. A variable so a test can say.
var sameNamespaces = namespacesOf
// lookupAccount names a user id from this machine's user database.
var lookupAccount = func(uid uint32) (string, error) {
u, err := user.LookupId(strconv.FormatUint(uint64(uid), 10))
if err != nil {
return "", err
}
return u.Username, nil
}
// Serve listens on socket until ctx ends, and answers each connection: one request, one answer.
//
// The directory is made 0755 and the socket 0666: every account on the machine may ask, because the account is
// what is vouched for and the controller decides what it may do. A file at the socket's path that is not a socket
// is never removed: the engine says so and serves nothing.
func Serve(ctx context.Context, socket string, forward Forward, say func(string)) error {
dir := filepath.Dir(socket)
if err := os.MkdirAll(dir, 0o755); err != nil {
return fmt.Errorf("mesh-cli's socket directory: %w", err)
}
// The directory is the engine's: a link, or one another account owns, could put the socket where that
// account replaces it with its own. Said, and nothing served.
if problem := notOwnDirectory(dir); problem != "" {
return fmt.Errorf("mesh-cli is not served on this machine: %s", problem)
}
if err := os.Chmod(dir, 0o755); err != nil {
return fmt.Errorf("mesh-cli's socket directory: %w", err)
}
if info, err := os.Lstat(socket); err == nil {
if info.Mode()&os.ModeSocket == 0 {
return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; "+
"mesh-cli is not served on this machine until it is moved", socket)
}
// A socket left by the engine before this one: nothing listens on it any more.
if err := os.Remove(socket); err != nil {
return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err)
}
}
l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"})
if err != nil {
return fmt.Errorf("listening for mesh-cli at %s: %w", socket, err)
}
if err := os.Chmod(socket, 0o666); err != nil {
l.Close()
return fmt.Errorf("opening mesh-cli's socket to every account: %w", err)
}
go func() {
<-ctx.Done()
l.Close()
}()
held := &slots{byUID: map[uint32]int{}}
var wg sync.WaitGroup
defer wg.Wait()
for {
conn, err := l.AcceptUnix()
if err != nil {
if ctx.Err() != nil {
return nil
}
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
continue
}
return fmt.Errorf("mesh-cli's socket: %w", err)
}
// Bounded from the first byte: nothing below may wait on a connection for longer than this before the
// line is read, whatever the peer does (review of ADR 0272).
readBy := time.Now().Add(readWithin)
_ = conn.SetDeadline(readBy)
wg.Add(1)
go func() {
defer wg.Done()
answer(ctx, conn, readBy, held, forward, say)
}()
}
}
// answer serves one connection; readBy is the deadline set when it was accepted, for reading its line.
func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, held *slots, forward Forward, say func(string)) {
defer conn.Close()
reply := func(a cli.Answer) {
_ = conn.SetWriteDeadline(time.Now().Add(readWithin))
if err := cli.WriteAnswer(conn, a); err != nil && say != nil {
say(fmt.Sprintf("mesh-cli: the answer could not be written back: %v", err))
}
}
// A refusal is written only after the request it answers is read (novox/hq issue 434). The client writes its
// request first and reads the answer after; an engine that answers and closes before that write has landed
// makes the write fail with a broken pipe, and the client never reads the refusal waiting for it — under load a
// line beyond its account's bound read as an error instead of busy. The wait ends at refusedWithin, or earlier
// at the deadline set when the connection was accepted; what was written is drained and never asked. Only
// refusalsWaiting refusals wait at once: over that the refusal is written at once, as before, and only then can a
// client still find its write broken.
refuse := func(why string) {
select {
case refusalsWaiting <- struct{}{}:
by := time.Now().Add(refusedWithin)
if readBy.Before(by) {
by = readBy
}
_ = conn.SetReadDeadline(by)
drain(conn)
<-refusalsWaiting
default:
}
reply(cli.Refusal(why))
}
uid, pid, err := peerOf(conn)
if err != nil {
// Never sent as anyone: an account the kernel did not give is no account at all.
refuse("the node-engine could not read which account is asking, so nothing was asked: " + err.Error())
return
}
// The user id is this machine's account only in this machine's namespaces: a container running with the
// operator's id shares the number and nothing else (review of ADR 0272).
if err := sameNamespaces(pid); err != nil {
refuse("the asking process does not run in this machine's own namespaces, so it is not this " +
"machine's account and nothing was asked: " + err.Error())
return
}
// And the login session, now: judged as the peer was when it connected, before it can exec or exit while its
// line is read (review of ADR 0272).
session := sessionOf(pid, uid)
// Looked up before a slot is taken, so a refused account never holds one while it is refused.
account, err := lookupAccount(uid)
if err != nil || account == "" {
refuse(fmt.Sprintf("user id %d is not in this machine's user database, so nothing was asked", uid))
return
}
if !held.take(uid) {
refuse(fmt.Sprintf("busy: user id %d has %d lines being asked already, or the machine %d; ask "+
"again when one has ended. Nothing was asked", uid, perAccount, atOnce))
return
}
defer held.give(uid)
req, err := cli.ReadRequest(conn)
if err != nil {
reply(cli.Refusal("the request could not be read, so nothing was asked: " + err.Error()))
return
}
// From here the wait is the controller's: as long as mesh-cli waits, and no longer.
_ = conn.SetDeadline(time.Now().Add(cli.AskWithin))
asking, cancel := context.WithTimeout(ctx, cli.AskWithin-30*time.Second)
defer cancel()
reply(forward(asking, Asked{Line: req.Line, Account: account, UID: uid, Session: session, Stdin: req.Stdin}))
}
internal/meshcli/meshcli_test.go
-534
View File
@@ -1,534 +0,0 @@
package meshcli
import (
"context"
"encoding/json"
"errors"
"io"
"net"
"os"
"os/user"
"path/filepath"
"sort"
"strings"
"sync"
"testing"
"time"
"git.novox.be/novox/mesh-sdk/go/cli"
)
// serving starts Serve on a socket of the test's own, and stops it when the test ends.
func serving(t *testing.T, forward Forward) string {
t.Helper()
socket := filepath.Join(t.TempDir(), "run", "cli.sock")
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() { done <- Serve(ctx, socket, forward, func(string) {}) }()
t.Cleanup(func() {
cancel()
if err := <-done; err != nil {
t.Errorf("Serve ended with %v", err)
}
})
for i := 0; i < 200; i++ {
if _, err := os.Stat(socket); err == nil {
return socket
}
time.Sleep(5 * time.Millisecond)
}
t.Fatal("the socket never appeared")
return ""
}
// The account sent is the one the kernel gives for the connecting process, and the answer comes back as it was.
func TestTheAccountAskingIsTheKernelsAndTheAnswerComesBack(t *testing.T) {
var got Asked
socket := serving(t, func(_ context.Context, asked Asked) cli.Answer {
got = asked
return cli.Answer{Stdout: []byte("done\n"), Exit: 0, Terminal: true, Why: "the terminal"}
})
a, err := cli.Ask(context.Background(), socket, []string{"settings", "show", "zsh"})
if err != nil {
t.Fatal(err)
}
if string(a.Stdout) != "done\n" || !a.Terminal || a.Refused != "" {
t.Fatalf("the answer came back as %+v", a)
}
me, err := user.Current()
if err != nil {
t.Skipf("this machine cannot name the test's account: %v", err)
}
if int(got.UID) != os.Getuid() || got.Account != me.Username {
t.Fatalf("asked as %d %q, and the kernel says %d %q", got.UID, got.Account, os.Getuid(), me.Username)
}
if strings.Join(got.Line, " ") != "settings show zsh" {
t.Fatalf("the line was sent as %q", got.Line)
}
if info, err := os.Stat(socket); err != nil || info.Mode().Perm() != 0o666 {
t.Fatalf("the socket is not open to every account: %v %v", info.Mode(), err)
}
}
// Credentials the kernel did not give are no account: refused, and nothing is asked.
func TestAConnectionWhoseAccountCannotBeReadIsNeverAskedAsAnyone(t *testing.T) {
before := peerOf
peerOf = func(*net.UnixConn) (uint32, int32, error) { return 0, 0, errors.New("no credentials") }
t.Cleanup(func() { peerOf = before })
asked := false
socket := serving(t, func(context.Context, Asked) cli.Answer {
asked = true
return cli.Answer{}
})
a, err := cli.Ask(context.Background(), socket, []string{"status"})
if err != nil {
t.Fatal(err)
}
if asked || a.Refused == "" || a.Exit == 0 {
t.Fatalf("a connection with no account was asked (%v) or not refused: %+v", asked, a)
}
}
// A request over the bound is refused and never asked.
func TestARequestOverTheBoundIsRefused(t *testing.T) {
asked := false
socket := serving(t, func(context.Context, Asked) cli.Answer {
asked = true
return cli.Answer{}
})
a, err := cli.Ask(context.Background(), socket, []string{strings.Repeat("x", cli.MaxRequest)})
if err != nil {
t.Fatal(err)
}
if asked || a.Refused == "" {
t.Fatalf("a request over the bound was asked (%v): %+v", asked, a)
}
}
// A file at the socket's path that is not a socket is never removed.
func TestAFileThatIsNotASocketIsKept(t *testing.T) {
dir := t.TempDir()
socket := filepath.Join(dir, "cli.sock")
if err := os.WriteFile(socket, []byte("mine"), 0o644); err != nil {
t.Fatal(err)
}
err := Serve(context.Background(), socket, nil, nil)
if err == nil || !strings.Contains(err.Error(), "not a socket") {
t.Fatalf("a file at the socket's path was not refused: %v", err)
}
if body, _ := os.ReadFile(socket); string(body) != "mine" {
t.Fatal("the file at the socket's path was touched")
}
}
// The request the controller reads holds these field names; the controller's side holds the same list
// (mesh-controller, the mesh-cli request's test).
func TestTheRequestToTheControllerKeepsItsFieldNames(t *testing.T) {
body, err := json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1})
if err != nil {
t.Fatal(err)
}
body, _ = json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1, Session: "session-3.scope"})
if got := keysOf(t, body); got != "account line session uid" {
t.Fatalf("the request's fields are %q", got)
}
body, _ = json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1})
if got := keysOf(t, body); got != "account line uid" {
t.Fatalf("the request's fields are %q", got)
}
body, _ = json.Marshal(Asked{Line: []string{"secret"}, Account: "a", UID: 1, Stdin: []byte("x")})
if got := keysOf(t, body); got != "account line stdin uid" {
t.Fatalf("a request with standard input has the fields %q", got)
}
body, _ = json.Marshal(Asked{Follow: "call-1", Account: "a", UID: 1})
if got := keysOf(t, body); got != "account follow uid" {
t.Fatalf("a follow's fields are %q", got)
}
// What the controller answers as a line's result, and as a running line's: the same names its side holds.
body, err = json.Marshal(cli.Answer{Stdout: []byte("o"), Stderr: []byte("e"), Exit: 1, Terminal: true, Why: "w",
Refused: "r", Cut: true})
if err != nil {
t.Fatal(err)
}
if got := keysOf(t, body); got != "cut exit refused stderr stdout terminal why" {
t.Fatalf("the answer's fields are %q", got)
}
body, _ = json.Marshal(runningResult{Running: true, Call: "call-1"})
if got := keysOf(t, body); got != "call running" {
t.Fatalf("a running answer's fields are %q", got)
}
}
func keysOf(t *testing.T, body []byte) string {
t.Helper()
var m map[string]any
if err := json.Unmarshal(body, &m); err != nil {
t.Fatal(err)
}
var keys []string
for k := range m {
keys = append(keys, k)
}
sort.Strings(keys)
return strings.Join(keys, " ")
}
// holdALine asks one line and returns once the engine is asking it, so it holds its account's slot. A first line
// that ends or never arrives fails the test at once, with what came of it, instead of hanging the suite (novox/hq
// issue 434).
func holdALine(t *testing.T, socket string, started <-chan struct{}) {
t.Helper()
first := make(chan error, 1)
go func() {
_, err := cli.Ask(context.Background(), socket, []string{"push"})
first <- err
}()
select {
case <-started:
case err := <-first:
t.Fatalf("the first line ended before it was asked: %v", err)
case <-time.After(5 * time.Second):
t.Fatal("the first line was never asked")
}
}
// **One account cannot hold the socket** (review of ADR 0272): its lines beyond its own bound are answered busy at
// once, nothing is asked, and the engine goes on accepting.
func TestOneAccountsLinesBeyondItsBoundAreAnsweredBusy(t *testing.T) {
was := perAccount
perAccount = 1
t.Cleanup(func() { perAccount = was })
release := make(chan struct{})
started := make(chan struct{}, 2)
socket := serving(t, func(context.Context, Asked) cli.Answer {
started <- struct{}{}
<-release
return cli.Answer{}
})
// Released before the engine is stopped: cleanups run last first, and the engine waits for its lines.
t.Cleanup(func() { close(release) })
holdALine(t, socket, started)
type asked struct {
a cli.Answer
err error
}
done := make(chan asked, 1)
go func() {
a, err := cli.Ask(context.Background(), socket, []string{"status"})
done <- asked{a, err}
}()
select {
case got := <-done:
// An error is a busy line the client never read (novox/hq issue 434), not an answer.
if got.err != nil || !strings.Contains(got.a.Refused, "busy") {
t.Fatalf("a second line from the same account reads %+v (%v)", got.a, got.err)
}
case <-time.After(5 * time.Second):
t.Fatal("a second line from the same account waited instead of being answered busy")
}
if len(started) != 0 {
t.Fatal("the busy line was asked")
}
}
// **A refusal waits for the request it answers** (novox/hq issue 434): the client writes its line first and reads
// the answer after. An engine that answered busy and closed before that write landed broke the client's write, and
// the refusal was never read — under the whole suite's load the busy line read as an error. Here the client
// connects and holds its line back: no answer may come before the line, and the line, once written, is answered
// busy. Ordered by the client's own write, so it holds however slow the machine is.
func TestARefusalWaitsForTheRequestItAnswers(t *testing.T) {
was, wasWithin := perAccount, refusedWithin
perAccount, refusedWithin = 1, time.Minute
t.Cleanup(func() { perAccount, refusedWithin = was, wasWithin })
release := make(chan struct{})
started := make(chan struct{}, 2)
socket := serving(t, func(context.Context, Asked) cli.Answer {
started <- struct{}{}
<-release
return cli.Answer{}
})
t.Cleanup(func() { close(release) })
holdALine(t, socket, started)
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Nothing is written yet: an engine that answers now answers before the request. The wait is far inside
// refusedWithin, so an engine that waits is still waiting when it ends.
_ = conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond))
var b [1]byte
if n, err := conn.Read(b[:]); n > 0 || !errors.Is(err, os.ErrDeadlineExceeded) {
t.Fatalf("the engine answered before the request was written (%d bytes, %v)", n, err)
}
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
if err := cli.WriteRequest(conn, cli.Request{Line: []string{"status"}}); err != nil {
t.Fatalf("the request could not be written: %v", err)
}
a, err := cli.ReadAnswer(conn)
if err != nil || !strings.Contains(a.Refused, "busy") {
t.Fatalf("a line written after the engine chose busy reads %+v (%v)", a, err)
}
if len(started) != 0 {
t.Fatal("the busy line was asked")
}
}
// **Refusals that wait are bounded** (novox/hq issue 434, review of ADR 0272): with refusalsWaiting already full, a
// further refused line is answered at once, without waiting for its request.
func TestARefusalOverTheWaitingBoundIsAnsweredAtOnce(t *testing.T) {
was, wasWithin, wasWaiting := perAccount, refusedWithin, refusalsWaiting
perAccount, refusedWithin = 1, time.Minute
refusalsWaiting = make(chan struct{}, 1)
refusalsWaiting <- struct{}{}
t.Cleanup(func() { perAccount, refusedWithin, refusalsWaiting = was, wasWithin, wasWaiting })
release := make(chan struct{})
started := make(chan struct{}, 2)
socket := serving(t, func(context.Context, Asked) cli.Answer {
started <- struct{}{}
<-release
return cli.Answer{}
})
t.Cleanup(func() { close(release) })
holdALine(t, socket, started)
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
// Nothing is written: a refusal that waited would wait refusedWithin, a minute, for it.
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
a, err := cli.ReadAnswer(conn)
if err != nil || !strings.Contains(a.Refused, "busy") {
t.Fatalf("a refusal over the waiting bound reads %+v (%v)", a, err)
}
}
// countingReader is an endless request with no line end, counting what was read of it.
type countingReader struct{ read int }
func (c *countingReader) Read(p []byte) (int, error) {
for i := range p {
p[i] = 'x'
}
c.read += len(p)
return len(p), nil
}
// **A refused request is drained, not kept** (novox/hq issue 434): what a refused client writes is read up to
// drainCap bytes and no further, however much it sends.
func TestARefusedRequestIsDrainedNoFurtherThanItsCap(t *testing.T) {
r := &countingReader{}
drain(r)
if r.read != drainCap {
t.Fatalf("drained %d bytes of an endless request; the cap is %d", r.read, drainCap)
}
after := &countingReader{}
drain(io.MultiReader(strings.NewReader("{\"line\":[\"status\"]}\n"), after))
if after.read != 0 {
t.Fatalf("draining read %d bytes past the end of the request line", after.read)
}
}
// **A process in another namespace is not this machine's account** (review of ADR 0272): a container running with
// the operator's user id shares the id and not the machine; refused, and nothing is asked.
func TestAPeerInAnotherNamespaceIsRefused(t *testing.T) {
before := sameNamespaces
sameNamespaces = func(pid int32) error { return errors.New("its mount namespace is not the engine's") }
t.Cleanup(func() { sameNamespaces = before })
asked := false
socket := serving(t, func(context.Context, Asked) cli.Answer {
asked = true
return cli.Answer{}
})
a, err := cli.Ask(context.Background(), socket, []string{"status"})
if err != nil {
t.Fatal(err)
}
if asked || !strings.Contains(a.Refused, "namespace") {
t.Fatalf("a peer in another namespace was asked (%v): %+v", asked, a)
}
}
// **The session a line comes from is the kernel's word, and only a login session counts** (review of ADR 0272): a
// service running as the operator — the tool runner, a user unit — is not a person at a terminal. The engine reads
// the peer's cgroup and sends the login session's scope, or nothing; the controller makes no other terminal.
func TestTheLoginSessionAskingIsReadFromTheKernel(t *testing.T) {
for _, c := range []struct{ cgroup, session string }{
{"0::/user.slice/user-1000.slice/session-3.scope\n", "session-3.scope"},
{"0::/user.slice/user-1000.slice/session-c1.scope\n", "session-c1.scope"},
{"0::/user.slice/user-1000.slice/user@1000.service/app.slice/tmux.scope\n", ""},
{"0::/system.slice/node-tools.service\n", ""},
{"0::/user.slice/user-1001.slice/session-3.scope\n", ""}, // another account's session
{"12:pids:/user.slice\n0::/user.slice/user-1000.slice/session-9.scope/sub\n", ""},
// Below a session scope, in a scope of its own: only the check that the scope is the account slice's direct
// child refuses it.
{"0::/user.slice/user-1000.slice/session-9.scope/x.scope\n", ""},
{"", ""},
} {
if got := loginSessionIn(c.cgroup, 1000); got != c.session {
t.Errorf("%q read as %q, want %q", c.cgroup, got, c.session)
}
}
// And it is sent: the test process's own, whatever it is, as the kernel says it.
var got Asked
socket := serving(t, func(_ context.Context, asked Asked) cli.Answer {
got = asked
return cli.Answer{}
})
if _, err := cli.Ask(context.Background(), socket, []string{"status"}); err != nil {
t.Fatal(err)
}
raw, _ := os.ReadFile("/proc/self/cgroup")
if want := userSessionIn(string(raw), uint32(os.Getuid())); got.Session != want {
t.Fatalf("sent session %q, the kernel says %q", got.Session, want)
}
}
// **A login session is a person's: systemd's own record says so** (review of ADR 0272): the scope the cgroup names
// must be a session logind keeps for this very account, of class `user` — not a greeter's, not the user manager's,
// not one whose record names another account — and read at accept, with the namespaces, before anything else.
func TestALoginSessionIsAUsersByItsOwnRecord(t *testing.T) {
dir := t.TempDir()
was := sessionsDir
sessionsDir = dir
t.Cleanup(func() { sessionsDir = was })
write := func(id, body string) {
if err := os.WriteFile(filepath.Join(dir, id), []byte(body), 0o644); err != nil {
t.Fatal(err)
}
}
write("9", "UID=1000\nUSER=op\nCLASS=user\nSCOPE=session-9.scope\nACTIVE=1\n")
write("3", "UID=1000\nCLASS=greeter\nSCOPE=session-3.scope\n")
write("5", "UID=1000\nCLASS=user\nSCOPE=session-6.scope\n")
// The account the record names is not the slice's: only the record's UID check refuses it.
write("8", "UID=1001\nCLASS=user\nSCOPE=session-8.scope\n")
// A name with a dot, held as a record that would pass: only the check of the id's characters refuses it.
write("a.b", "UID=1000\nCLASS=user\nSCOPE=session-a.b.scope\n")
// A key said twice: the first is the record's word (systemd writes each once); a later one is not read.
write("11", "UID=1001\nCLASS=user\nSCOPE=session-11.scope\nUID=1000\n")
write("12", "UID=1000\nCLASS=user\nSCOPE=session-12.scope\nCLASS=greeter\nUID=1001\n")
for _, c := range []struct {
cgroup string
uid uint32
session string
}{
{"0::/user.slice/user-1000.slice/session-9.scope\n", 1000, "session-9.scope"},
{"0::/user.slice/user-1000.slice/x.scope\n", 1000, ""},
{"0::/user.slice/user-1000.slice/session-9.scope\n", 1001, ""}, // asked as another uid than the slice's
{"0::/user.slice/user-1000.slice/session-8.scope\n", 1000, ""}, // the record names uid 1001
{"0::/user.slice/user-1000.slice/session-3.scope\n", 1000, ""}, // a greeter's
{"0::/user.slice/user-1000.slice/session-5.scope\n", 1000, ""}, // a record naming another scope
{"0::/user.slice/user-1000.slice/session-7.scope\n", 1000, ""}, // no record at all
{"0::/user.slice/user-1000.slice/session-a.b.scope\n", 1000, ""},
{"0::/user.slice/user-1000.slice/session-...scope\n", 1000, ""},
{"0::/user.slice/user-1000.slice/session-11.scope\n", 1000, ""}, // first UID is 1001
{"0::/user.slice/user-1000.slice/session-12.scope\n", 1000, "session-12.scope"}, // first UID and CLASS stand
} {
if got := userSessionIn(c.cgroup, c.uid); got != c.session {
t.Errorf("%q as uid %d read as %q, want %q", c.cgroup, c.uid, got, c.session)
}
}
}
// **The session is read at accept, with the namespaces, before the line** (review of ADR 0272): the peer is judged as
// it was when it connected. The client sends its line only once the session has been read; were the session read
// after the line — in the forward, say — the line would never be sent and the test fails.
func TestTheSessionIsReadBeforeTheLine(t *testing.T) {
var mu sync.Mutex
var order []string
note := func(s string) { mu.Lock(); order = append(order, s); mu.Unlock() }
read := make(chan struct{}, 1)
beforeNS, beforeSession := sameNamespaces, sessionOf
sameNamespaces = func(int32) error { note("namespaces"); return nil }
sessionOf = func(int32, uint32) string { note("session"); read <- struct{}{}; return "session-1.scope" }
t.Cleanup(func() { sameNamespaces, sessionOf = beforeNS, beforeSession })
var got Asked
socket := serving(t, func(_ context.Context, asked Asked) cli.Answer {
note("forward")
got = asked
return cli.Answer{}
})
conn, err := net.Dial("unix", socket)
if err != nil {
t.Fatal(err)
}
defer conn.Close()
select {
case <-read:
case <-time.After(3 * time.Second):
t.Fatal("the session was not read before the line was sent")
}
note("line sent")
if err := cli.WriteRequest(conn, cli.Request{Line: []string{"status"}}); err != nil {
t.Fatal(err)
}
_ = conn.SetDeadline(time.Now().Add(5 * time.Second))
if _, err := cli.ReadAnswer(conn); err != nil {
t.Fatal(err)
}
mu.Lock()
defer mu.Unlock()
if strings.Join(order, ",") != "namespaces,session,line sent,forward" || got.Session != "session-1.scope" {
t.Fatalf("read in the order %v, sent %q", order, got.Session)
}
}
// mesh-cli's standard input reaches the controller with its line, whole, and only with the line: a follow of the
// same call carries none, and the engine says it nowhere (novox/hq ADR 0259 §10, ADR 0272).
func TestStandardInputGoesToTheControllerWithTheLineAndNowhereElse(t *testing.T) {
secret := []byte("123456789:the-bot-token-given-at-the-terminal\n")
var said []string
var mu sync.Mutex
var asked [][]byte
ask := func(_ context.Context, body []byte) ([]byte, error) {
mu.Lock()
defer mu.Unlock()
asked = append(asked, append([]byte(nil), body...))
if len(asked) == 1 {
return []byte(`{"result":{"running":true,"call":"call-7"}}`), nil
}
return []byte(`{"result":{"exit":0,"terminal":true}}`), nil
}
was := followEvery
followEvery = time.Millisecond
t.Cleanup(func() { followEvery = was })
socket := filepath.Join(t.TempDir(), "run", "cli.sock")
ctx, cancel := context.WithCancel(context.Background())
done := make(chan error, 1)
go func() {
done <- Serve(ctx, socket, func(ctx context.Context, a Asked) cli.Answer { return AskTheController(ctx, ask, a) },
func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() })
}()
defer func() { cancel(); <-done }()
for i := 0; i < 200; i++ {
if _, err := os.Stat(socket); err == nil {
break
}
time.Sleep(5 * time.Millisecond)
}
a, err := cli.AskWith(context.Background(), socket, cli.Request{Line: []string{"secret", "accept", "anchor",
"telegram", "telegram-token", "--from", "-"}, Stdin: secret})
if err != nil || a.Exit != 0 {
t.Fatalf("%v %+v", err, a)
}
mu.Lock()
defer mu.Unlock()
if len(asked) != 2 {
t.Fatalf("asked %d times, want the line and one follow", len(asked))
}
var first, follow Asked
_ = json.Unmarshal(asked[0], &first)
_ = json.Unmarshal(asked[1], &follow)
if string(first.Stdin) != string(secret) {
t.Fatalf("the controller was not given the standard input whole: %q", first.Stdin)
}
if len(follow.Stdin) != 0 || strings.Contains(string(asked[1]), "stdin") {
t.Fatalf("a follow carried the standard input again: %s", asked[1])
}
for _, s := range said {
if strings.Contains(s, "bot-token") {
t.Fatalf("the engine said the standard input: %s", s)
}
}
}
internal/meshcli/peer.go
-19
View File
@@ -1,19 +0,0 @@
package meshcli
import "net"
// The kernel's word on who is at the other end of a socket, shared with the node-engine's other local socket, the
// write of a filled file (internal/filling, novox/hq ADR 0349 §4): one reading of it, so the two never judge a peer
// differently.
// PeerCredentials is the user id and process id of the process at the other end, as the kernel recorded them when
// it connected (SO_PEERCRED). Nothing the process says is read.
func PeerCredentials(conn *net.UnixConn) (uint32, int32, error) { return peerCredentials(conn) }
// SameNamespaces refuses a process whose mount or user namespace is not the engine's: a user id is this machine's
// account only in this machine's namespaces.
func SameNamespaces(pid int32) error { return namespacesOf(pid) }
// NotOwnDirectory says what is wrong with a socket's directory, or nothing: it must be a directory, not a link,
// owned by the account the engine runs as.
func NotOwnDirectory(dir string) string { return notOwnDirectory(dir) }
internal/ownresync/ownresync.go
-319
View File
@@ -1,319 +0,0 @@
// Package ownresync hands the vault the value of this machine's own secret that runs, where the vault holds another
// (novox/hq issue 1126, cause C4).
//
// The vault's `move-in` never replaces a value it holds, so its first import of an own secret stays fixed; where the
// controller later made that secret anew (a rotation at start, a re-issue), the new value runs and the vault keeps the
// old one. The node's shadow composer sees it: composing from the vault's value, it opens both seals with this
// machine's key and its answer's shown body says delivery.MaskedAnotherValue. After each shadow answer this step asks the
// vault's `move-in` with replace_running: true and what runs, for each own secret so masked, once per running value, and
// says what it asked and what the vault answered. It is move-in's, the verb this node may ask already, so no seat and no
// bus grant changes for it.
//
// **Only while this node applies the controller's compositions**, and only for a module whose composition that runs is
// the one its want names: then the controller's value is what runs, and what runs is the truth the vault lacks. Once the
// node composes its own, the vault's value is the source and nothing is handed. **The switch** is the vault's
// (`own_take_switch`, a person's act): while it is off the vault refuses every ask, and this step says so once per value.
// The vault also refuses any value it changed in the last 15 minutes, so a re-issue the node has not applied yet is
// never put back.
package ownresync
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"sort"
"sync"
"git.novox.be/novox/mesh-sdk/go/catalogue"
"git.novox.be/novox/mesh-sdk/go/delivery"
"github.com/novox/mesh-host/internal/selfcompose"
)
// Verb is the vault's verb this step asks, with ReplaceRunning true in both steps.
const Verb = "move-in"
// ReplaceRunning is move-in's argument that makes it take the running value over the one the vault holds.
const ReplaceRunning = "replace_running"
// Env is what the step reads and asks.
type Env struct {
Node string
// Composes is whose compositions the node applies now: delivery.ComposesController, delivery.ComposesOwn, or ""
// while its agent has not decided. The step hands nothing but for ComposesController.
Composes func() string
// Running is what the node runs now (agent.Everything).
Running func() []delivery.Composed
// Get reads one key's latest value; nil and no error when there is none.
Get func(ctx context.Context, bucket, key string) ([]byte, error)
// Unseal opens a value sealed to this machine; Seal seals a value to a raw X25519 key in base64 (identity.Seal).
Unseal func(sealed string) ([]byte, error)
Seal func(to string, value []byte) (string, error)
// Ask asks one of the vault's verbs in this engine's own name (link.Session.AskVault).
Ask func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error)
Say func(string)
}
// Outcome is what the step did for one own secret: asked or not, and what was said.
type Outcome struct {
Module string
Name string
Asked bool
Said string
// Retry says a read or an ask the bus did not answer stopped it: no answer, kept as none, and the step is to be
// taken again after a bound (issue 1126, 2026-10-28: "want.shanks.brother could not be read ... no responders").
Retry bool
}
// Resync is the step of one node agent's run.
type Resync struct {
env Env
mu sync.Mutex
// done is, by module/name, the running value's fingerprint the vault was asked with or why it was not: each is
// asked, and each reason said, once.
done map[string]string
}
// New is the step for one node.
func New(env Env) *Resync {
if env.Say == nil {
env.Say = func(string) {}
}
return &Resync{env: env, done: map[string]string{}}
}
// Take reads the node's shadow answers and hands the vault what runs of each own secret they say another value of.
func (r *Resync) Take(ctx context.Context, answers map[string]delivery.CandidateAnswer) []Outcome {
seen := map[delivery.OwnValueDiffers]bool{}
var drift []delivery.OwnValueDiffers
proposals := make([]string, 0, len(answers))
for g := range answers {
proposals = append(proposals, g)
}
sort.Strings(proposals)
for _, g := range proposals {
for _, o := range delivery.OwnValuesDiffer(answers[g]) {
if !seen[o] {
seen[o] = true
drift = append(drift, o)
}
}
}
if len(drift) == 0 {
return nil
}
composes := ""
if r.env.Composes != nil {
composes = r.env.Composes()
}
var out []Outcome
for _, o := range drift {
res := r.one(ctx, composes, o)
if res.Said != "" || res.Retry {
out = append(out, res)
}
}
return out
}
// one is the step for one own secret: Said is empty when it was done before for the same value and reason.
func (r *Resync) one(ctx context.Context, composes string, o delivery.OwnValueDiffers) Outcome {
res := Outcome{Module: o.Module, Name: o.Name}
key := o.Module + "/" + o.Name
if composes == "" {
return res // the agent has not decided whose compositions it applies: taken again once it has
}
if composes != delivery.ComposesController {
return r.said(key, "not:composes", res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
"nothing is handed: this node composes its own (%q), so the vault's value is the source", key, composes))
}
sealed, why, failed := r.running(ctx, o)
if failed != nil {
// The bus did not answer: no reason to keep, and taken again (Retry); said once per reason.
res.Retry = true
return r.said(key, "retry:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
"nothing is handed yet: %s; asked again", key, why))
}
if why != "" {
return r.said(key, "not:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+
"nothing is handed: %s", key, why))
}
sum := sha256.Sum256([]byte(sealed))
fp := hex.EncodeToString(sum[:])
r.mu.Lock()
asked := r.done[key] == fp
r.mu.Unlock()
if asked {
return res
}
value, err := r.env.Unseal(sealed)
if err != nil {
return r.said(key, "not:unseal:"+fp, res, fmt.Sprintf("%s: what runs does not open with this machine's key, so "+
"nothing is handed: %v", key, err))
}
defer wipe(value)
said, err := r.hand(ctx, o, value)
if err != nil {
if ctx.Err() != nil {
return res // the run ended: asked again in the next
}
if selfcompose.BusFailed(err) {
// The vault did not answer on the bus: no word of the vault's, so not kept as asked (Retry).
res.Retry = true
return r.said(key, "retry:ask", res, fmt.Sprintf("%s: the vault's %s with replace_running did not answer, "+
"and is asked again: %v", key, Verb, err))
}
said = fmt.Sprintf("%s: the vault's %s with replace_running refused: %v", key, Verb, err)
}
// Once per running value, whatever the vault answered: a refusal is the vault's word, said once, and a new value
// that runs asks again.
r.mu.Lock()
r.done[key] = fp
r.mu.Unlock()
res.Asked, res.Said = true, said
r.env.Say(said)
return res
}
// said says why once per reason, and keeps the reason as done.
func (r *Resync) said(key, reason string, res Outcome, why string) Outcome {
r.mu.Lock()
again := r.done[key] == reason
r.done[key] = reason
r.mu.Unlock()
if again {
return res
}
res.Said = why
r.env.Say(why)
return res
}
// running is the sealed value of o in the composition that runs, when that composition is the one the module's want
// names; or why it is not handed, with the error of a read the bus did not answer, which is no answer.
func (r *Resync) running(ctx context.Context, o delivery.OwnValueDiffers) (sealed, why string, failed error) {
var comp *delivery.Composed
if r.env.Running != nil {
for _, c := range r.env.Running() {
if c.Module == o.Module {
c := c
comp = &c
break
}
}
}
if comp == nil {
return "", o.Module + " runs no composition here", nil
}
wantKey, err := delivery.WantKey(r.env.Node, o.Module)
if err != nil {
return "", err.Error(), nil
}
raw, err := r.env.Get(ctx, delivery.AssignerWantsBucket, wantKey)
if err != nil {
return "", fmt.Sprintf("its want %s could not be read (%v)", wantKey, err), err
}
if raw == nil {
return "", fmt.Sprintf("it has no want %s", wantKey), nil
}
want, err := delivery.DecodeWant(raw)
if err != nil {
return "", fmt.Sprintf("its want %s is not one: %v", wantKey, err), nil
}
if want.Self || want.Removed || want.Composed == "" {
return "", fmt.Sprintf("its want %s names no composition of the controller's", wantKey), nil
}
// What runs is the composition the want names, as the module's have says: applied and running.
haveKey, err := delivery.HaveKey(r.env.Node, o.Module)
if err != nil {
return "", err.Error(), nil
}
raw, err = r.env.Get(ctx, delivery.HaveBucket, haveKey)
if err != nil {
return "", fmt.Sprintf("its have %s could not be read (%v)", haveKey, err), err
}
if raw == nil {
return "", fmt.Sprintf("it has no have %s", haveKey), nil
}
have, err := delivery.DecodeHave(raw)
if err != nil {
return "", fmt.Sprintf("its have %s is not one: %v", haveKey, err), nil
}
if have.Composed != want.Composed || have.Phase != delivery.PhaseRunning {
return "", fmt.Sprintf("what runs (%s, %s) is not yet the composition its want names (%s)", have.Composed,
have.Phase, want.Composed), nil
}
// On a node the resource's id carries its module's name (`only-office.needs-jwt`); delivery.OwnNeedName reads it.
id := o.Module + "." + catalogue.NeedID(o.Name)
for _, res := range comp.Resources {
var body struct {
ID string `json:"id"`
Sealed any `json:"sealed"`
}
if json.Unmarshal(res, &body) != nil {
continue
}
if name, own := delivery.OwnNeedName(o.Module, body.ID); !own || name != o.Name {
continue
}
if s, ok := body.Sealed.(string); ok && s != "" {
return s, "", nil
}
return "", fmt.Sprintf("its resource %s carries no value sealed to this machine", id), nil
}
return "", fmt.Sprintf("what runs holds no resource %s", id), nil
}
// hand asks the vault's two steps with value, and checks what the vault holds after by its fingerprint.
func (r *Resync) hand(ctx context.Context, o delivery.OwnValueDiffers, value []byte) (string, error) {
raw, err := r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "module": o.Module, "name": o.Name,
"why": "this node's shadow answer says the vault holds another value than runs here"})
if err != nil {
return "", err
}
var opened struct {
Call string `json:"call"`
SealTo string `json:"seal_to"`
Secret string `json:"secret"`
}
if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" {
return "", fmt.Errorf("the vault answered no call and no key to seal to")
}
sealed, err := r.env.Seal(opened.SealTo, value)
if err != nil {
return "", fmt.Errorf("the vault's key for the call is not a sealing key: %w", err)
}
raw, err = r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "call": opened.Call, "sealed": sealed})
if err != nil {
return "", err
}
var kept struct {
Secret string `json:"secret"`
From int `json:"from_generation"`
To int `json:"to_generation"`
Unchanged bool `json:"unchanged"`
Fingerprint string `json:"fingerprint"`
Said string `json:"said"`
}
if err := json.Unmarshal(raw, &kept); err != nil || kept.Secret == "" {
return "", fmt.Errorf("the vault answered %s with no secret", opened.Secret)
}
sum := sha256.Sum256(value)
if kept.Fingerprint != "sha256:"+hex.EncodeToString(sum[:]) {
return "", fmt.Errorf("the vault says it holds %s at generation %d, but not the value handed in", kept.Secret, kept.To)
}
if kept.Unchanged {
return fmt.Sprintf("handed the vault what runs of %s: it held that value already (generation %d)", kept.Secret, kept.To), nil
}
return fmt.Sprintf("handed the vault what runs of %s: generation %d replaced by %d, the value that runs", kept.Secret,
kept.From, kept.To), nil
}
func wipe(b []byte) {
for i := range b {
b[i] = 0
}
}
internal/ownresync/ownresync_test.go
-117
View File
@@ -1,117 +0,0 @@
package ownresync
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"strings"
"testing"
"git.novox.be/novox/mesh-sdk/go/delivery"
)
// Issue 1126, cause C4: after a shadow answer saying the vault holds another value of an own secret than runs, the node
// hands the vault what runs, in the verb's two steps, once per running value; and only while it applies the
// controller's compositions and runs the one its want names.
func TestTheNodeHandsTheVaultWhatRunsOfADriftedOwnSecret(t *testing.T) {
const running = "the-value-that-runs"
digest := "sha256:" + strings.Repeat("a", 64)
fp := "src2:" + strings.Repeat("b", 64)
sum := sha256.Sum256([]byte(running))
answers := map[string]delivery.CandidateAnswer{"g": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateDiffers,
Differs: []delivery.ModuleDiffers{{Module: "only-office", Why: "the resource differs", Bodies: []delivery.ResourceBodies{
{ID: "only-office.needs-jwt", Composed: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`, Running: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`},
{ID: "only-office.needs-salt", Composed: `{"sealed":"<masked: another seal of the same value>"}`}}}}}}
for name, row := range map[string]struct {
composes string
runs string // the have's composition digest
refuse string // the vault's refusal
asks int // the vault's verb asked, both passes
says string
// Issue 1126, 2026-10-28: a read or an ask the bus did not answer is no answer: taken again (Retry), never kept.
wantRead string // the want's read: "" answers it, "fails" the bus does not answer, "absent" no key
retry bool
}{
"applies the controller's, runs its want": {delivery.ComposesController, digest, "", 2, "generation 1 replaced by 2", "", false},
"composes its own": {delivery.ComposesOwn, digest, "", 0, "composes its own", "", false},
"not decided yet": {"", digest, "", 0, "", "", false},
"runs not yet what its want names": {delivery.ComposesController, "sha256:" + strings.Repeat("c", 64), "", 0, "not yet the composition", "", false},
"the vault refuses": {delivery.ComposesController, digest, "switched off", 1, "switched off", "", false},
"the vault does not answer": {delivery.ComposesController, digest, "nats: no responders available for request", 2, "asked again", "", true},
"the want's read does not answer": {delivery.ComposesController, digest, "", 0, "asked again", "fails", true},
"it has no want": {delivery.ComposesController, digest, "", 0, "it has no want", "absent", false},
} {
var asked []map[string]any
var said []string
want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest})
have, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: row.runs, Phase: delivery.PhaseRunning, Health: delivery.HealthOK})
r := New(Env{Node: "novox", Composes: func() string { return row.composes },
Running: func() []delivery.Composed {
return []delivery.Composed{{Node: "novox", Module: "only-office", FP: fp, Resources: []json.RawMessage{
// As novox runs it (its shadow answer, 2026-10-28): the id carries the module's name.
json.RawMessage(`{"id":"only-office.needs-jwt","module":"only-office","type":"file",` +
`"path":"/var/lib/only-office/jwt.secret","sealed":"box-of-the-running-value"}`)}}}
},
Get: func(_ context.Context, bucket, key string) ([]byte, error) {
switch {
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "fails":
return nil, errors.New("the bucket mesh-delivery-assigner_wants: nats: no responders available for request")
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "absent":
return nil, nil
case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office":
return want, nil
case bucket == delivery.HaveBucket && key == "novox.only-office":
return have, nil
}
return nil, nil
},
Unseal: func(sealed string) ([]byte, error) {
if sealed != "box-of-the-running-value" {
return nil, errors.New("not this machine's")
}
return []byte(running), nil
},
Seal: func(to string, value []byte) (string, error) { return to + "(" + string(value) + ")", nil },
Ask: func(_ context.Context, verb string, args map[string]any) (json.RawMessage, error) {
asked = append(asked, args)
if verb != "move-in" || args["replace_running"] != true {
t.Errorf("%s: asked %s with %v, not move-in with replace_running", name, verb, args)
}
if row.refuse != "" {
return nil, errors.New(row.refuse)
}
if args["call"] == nil {
if args["module"] != "only-office" || args["name"] != "jwt" || args["node"] != nil {
t.Errorf("%s: the first step names %v", name, args)
}
return json.RawMessage(`{"call":"c1","seal_to":"key","secret":"own/novox/only-office/jwt"}`), nil
}
if args["sealed"] != "key("+running+")" {
t.Errorf("%s: handed %v, not what runs sealed to the call's key", name, args["sealed"])
}
return json.RawMessage(`{"secret":"own/novox/only-office/jwt","from_generation":1,"to_generation":2,` +
`"fingerprint":"sha256:` + hex.EncodeToString(sum[:]) + `"}`), nil
},
Say: func(s string) { said = append(said, s) }})
first := r.Take(context.Background(), answers)
again := r.Take(context.Background(), answers)
if len(asked) != row.asks {
t.Errorf("%s: the vault asked %d times, want %d: %v", name, len(asked), row.asks, asked)
}
if retry := len(first) == 1 && first[0].Retry; retry != row.retry {
t.Errorf("%s: taken again %v, want %v: %+v", name, retry, row.retry, first)
}
if row.retry && (len(again) != 1 || !again[0].Retry || again[0].Said != "") || !row.retry && len(again) != 0 {
t.Errorf("%s: the same answer did or said something again: %+v", name, again)
}
all := strings.Join(said, "\n")
if row.says == "" && (len(first) != 0 || all != "") || row.says != "" && (len(first) != 1 || !strings.Contains(all, row.says)) {
t.Errorf("%s: did %+v and said %q, want %q", name, first, all, row.says)
}
if strings.Contains(all, running) {
t.Errorf("%s: the value is in what the node said", name)
}
}
}
internal/selfcompose/packages.go
+3 -3
View File
@@ -15,10 +15,10 @@ import (
// **The packages this node's compositions declare, kept by its engine** (novox/hq ADR 0565 §6, §15; issue 1953). The
// composer keeps the package resources of each composition it handed, and after a pass that handed something new (or
// the run's first) it works out the node's key of the engine's state packages from them (delivery.DeclaredPackagesOf,
// the controller mirror's layout and order) and writes it only when the value differs from what the key holds (ADR 0289
// in the layout and order the controller's mirror had) and writes it only when the value differs from what the key holds (ADR 0289
// §1): no timer, and no composing of another node. A module held keeps the packages of what was handed of it, as it keeps
// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it beside the
// controller's mirror until the lease write, and with write writes it again (ADR 0565 §15).
// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it, and with
// write writes it again (ADR 0565 §15).
// PackagesStore is this node's key of the engine's state packages (link.EnginePackages).
type PackagesStore interface {
internal/selfcompose/selfcompose_test.go
+7 -7
View File
@@ -1057,7 +1057,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
go func() { sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond }); close(kept) }()
time.Sleep(30 * time.Millisecond)
sb.missing(delivery.PlacementCandidatesBucket, false)
sb.made(7)
sb.made(6)
rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}})
tp, _ := delivery.EncodePlacement(trunk)
sb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
@@ -1321,7 +1321,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
nctx, ncancel := context.WithCancel(ctx)
defer ncancel()
go func() { _ = nsh.Run(nctx) }()
nb.made(7)
nb.made(6)
nb.put(delivery.PlacementBucket, "placement.n1", tp, 1)
nb.put(delivery.HaveBucket, "n1.node", rec, 1)
for name, m := range mods {
@@ -1335,19 +1335,19 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk})
oldKey, _ := delivery.PlacementCandidateKey("older", "n1")
nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2)
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} {
for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, delivery.BuilderBuildsBucket} {
nb.send(bucket, agent.Update{Caught: true})
}
select {
case a := <-nanswered:
if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, delivery.AssignerWantsBucket) || g.Proposal != "same" {
t.Errorf("a shadow waiting for its wants answered %+v", g)
if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, ResolvedBucket) || g.Proposal != "same" {
t.Errorf("a shadow waiting for its values answered %+v", g)
}
case <-time.After(5 * time.Second):
t.Fatal("a shadow not ready after its wait answered nothing")
}
nmu.Lock()
if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+delivery.AssignerWantsBucket) {
if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+ResolvedBucket) {
t.Errorf("a shadow not ready said %q", nsaid)
}
nmu.Unlock()
@@ -1355,7 +1355,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) {
scancel()
<-kept
setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time
nb.send(delivery.AssignerWantsBucket, agent.Update{Caught: true})
nb.send(ResolvedBucket, agent.Update{Caught: true})
select {
case a := <-nanswered:
if g := a["same"]; g.Verdict != delivery.CandidateSame || !strings.Contains(g.Note, "this answer took over") {
internal/selfcompose/shadow.go
+2 -36
View File
@@ -32,8 +32,6 @@ type Shadow struct {
mu sync.Mutex
candidates map[string]map[string]candidateKey // by proposal, by node
answers map[string]delivery.CandidateAnswer
// wants are the node's wants by module (agent.NodeModule for its own); a change of one answers again.
wants map[string]delivery.Want
// ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy
// from the declaration kept before the switch (issue 1126, probe #532 on shanks).
ran map[string]string
@@ -64,7 +62,7 @@ type answeredAt struct {
}
// inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it.
var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves", "wants"}
var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves"}
// wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at):
// its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words.
@@ -99,7 +97,7 @@ type candidateKey struct {
// NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key.
func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow {
return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{},
answers: map[string]delivery.CandidateAnswer{}, wants: map[string]delivery.Want{}, ran: map[string]string{}}
answers: map[string]delivery.CandidateAnswer{}, ran: map[string]string{}}
}
// Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch
@@ -141,7 +139,6 @@ func (s *Shadow) Run(ctx context.Context) error {
defer cancel()
s.mu.Lock()
s.candidates = map[string]map[string]candidateKey{}
s.wants = map[string]delivery.Want{}
s.ran = map[string]string{}
s.answered = map[string]answeredAt{}
s.moved = [len(inputWords)]uint64{}
@@ -159,8 +156,6 @@ func (s *Shadow) Run(ctx context.Context) error {
{ResolvedBucket, node + ".>", (*Composer).takeValue, nil},
// What this node runs moved: an apply ends in its module's have.
{delivery.HaveBucket, node + ".>", nil, nil},
// Its wants.
{delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil},
}
type tagged struct {
in int
@@ -225,8 +220,6 @@ func (s *Shadow) Run(ctx context.Context) error {
case t.in == 5:
folded := c.takeModuleHave(t.u)
changed = s.takeRan(t.u) || folded
case t.in == 6:
changed = s.takeWant(t.u)
default:
changed = c.takeChanged(ins[t.in], t.u)
}
@@ -685,33 +678,6 @@ func sameAnswer(a, b delivery.CandidateAnswer) bool {
return errA == nil && errB == nil && string(x) == string(y)
}
// takeWant takes one of the node's wants, and says whether it changed.
func (s *Shadow) takeWant(u agent.Update) bool {
module := ""
var w delivery.Want
if _, err := delivery.ParseWantNodeKey(u.Key); err == nil {
module = agent.NodeModule
} else if node, m, err := delivery.ParseWantKey(u.Key); err == nil && node == s.c.env.Node {
module = m
if !u.Deleted {
if w, err = delivery.DecodeWant(u.Value); err != nil || w.Removed || w.FP == "" {
u.Deleted = true
}
}
} else {
return false
}
s.mu.Lock()
defer s.mu.Unlock()
old, had := s.wants[module]
if u.Deleted {
delete(s.wants, module)
return had
}
s.wants[module] = w
return !had || old.FP != w.FP
}
// sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not
// open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work).
func (c *Composer) sameValue(a, b string) (same, known bool) {
internal/userlist/userlist.go
+89 -93
View File
@@ -13,23 +13,26 @@
// or is deleted is said, the key last heard stands, and the list loaded stays; with no key ever heard nothing is
// written. The list a composition carries is never written.
//
// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition in the node's
// composition bucket, verified as the agent verified it before anything of it is read: what tells the bus node it is
// one, and the file's place. The have of that module says what was written, which list it follows and this node's key,
// which the grants part seals to.
// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition as this node runs
// it (Env.Running: the agent's own record of what runs, its own composition or the declaration kept before the switch),
// never one read from a composition bucket: the controller that wrote `composed-<node>` ended (issue 1126), and the bus
// module on the bus node runs from what the agent holds. That is what tells the bus node it is one, and the file's
// place. The have of that module says what was written, which list it follows and this node's key, which the grants
// part seals to.
//
// **It watches, it never polls**, and looks again after every apply of the bus module and when asked (Again). It reads no
// want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list.
// **It watches, it never polls**: the grants part's key is watched, and what runs is looked at again when the agent says
// it changed, after every apply of the bus module and when asked (Again). It reads no want and writes none, moves no
// fingerprint, and touches nothing of the bus module but its user list.
package userlist
import (
"context"
"crypto/ed25519"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"regexp"
"sort"
"strings"
"sync"
"time"
@@ -49,10 +52,10 @@ type Written = delivery.UserListWritten
// Env is everything the follower needs of the node.
type Env struct {
Node string
Composed agent.Watcher
// Trusted is the keys a composition may be signed by: one any other key signed declares no file.
Trusted func() []ed25519.PublicKey
Node string
// Running is the compositions this node runs now, as its agent holds them (agent.Agent.Everything): where the user
// list's file is is read from these alone (issue 1126). Nil runs nothing.
Running func() []delivery.Composed
// OnDisk is what the resource's file holds on the machine now; nil when there is none.
OnDisk func(resource json.RawMessage) ([]byte, error)
// Write writes the user list's file resource of a module's composition alone, with body as its content, when it
@@ -79,19 +82,18 @@ type Env struct {
}
type source struct {
key string
module string
rev uint64
resource json.RawMessage
wanted string
}
// Follower follows the node's compositions for the bus's user list.
// Follower follows the grants part's user list into the file the bus module this node runs declares.
type Follower struct {
env Env
kick chan struct{}
mu sync.Mutex
srcs map[string]source // by composition key
// looks asks a look at what runs (RunsChanged): written only when the file it names moved.
looks chan struct{}
mu sync.Mutex
srcs map[string]source // by module, of what runs now
// cur is the source last taken, and written whether its write went through.
cur source
written bool
@@ -116,7 +118,8 @@ func New(env Env) *Follower {
if env.Backoff == nil {
env.Backoff = func(int) time.Duration { return time.Minute }
}
f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}}
f := &Follower{env: env, kick: make(chan struct{}, 1), looks: make(chan struct{}, 1), srcs: map[string]source{},
said: map[string]bool{}}
if env.Grants != nil && env.Kept != nil {
if raw, rev := env.Kept(); raw != nil {
if g, err := delivery.DecodeGrantsUserList(raw); err == nil {
@@ -127,8 +130,8 @@ func New(env Env) *Follower {
return f
}
// Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module,
// and when a person asks. It never blocks.
// Again asks the follower to look at what runs and the file again: when what runs changed, after an apply of the bus
// module, and when a person asks. It never blocks.
func (f *Follower) Again() {
select {
case f.kick <- struct{}{}:
@@ -136,21 +139,25 @@ func (f *Follower) Again() {
}
}
// State is the module whose list the follower writes ("" when no composition of this node carries one) and what it
// last did.
// RunsChanged asks the follower to look at what this node runs again: the list is written again only when the file the
// bus module declares moved. It never blocks.
func (f *Follower) RunsChanged() {
select {
case f.looks <- struct{}{}:
default:
}
}
// State is the module whose list the follower writes ("" when nothing this node runs carries one) and what it last did.
func (f *Follower) State() (string, Written) {
f.mu.Lock()
defer f.mu.Unlock()
return f.mod, f.state
}
// Run follows the compositions until ctx ends.
// Run follows the grants part's list until ctx ends.
func (f *Follower) Run(ctx context.Context) error {
updates, err := f.env.Composed.Watch(ctx, f.env.Node+".>")
if err != nil {
return fmt.Errorf("the user list cannot be followed: %w", err)
}
caught := false
f.RunsChanged() // the first look at what runs, at once
// The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk
// buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link)
// never writes a list older than the key; a watch refused or ended is tried again on the backoff, and meanwhile the
@@ -185,17 +192,11 @@ func (f *Follower) Run(ctx context.Context) error {
return nil
case <-f.kick:
again = true
f.look()
case <-f.looks:
f.look()
case <-retry:
retry = nil
case u, ok := <-updates:
if !ok {
return fmt.Errorf("the watch of the compositions ended")
}
if u.Caught {
caught = true
} else {
f.take(u)
}
case u, ok := <-grants:
if !ok {
failed("its watch ended")
@@ -210,9 +211,6 @@ func (f *Follower) Run(ctx context.Context) error {
}
f.takeGrants(u)
}
if !caught {
continue
}
open()
if opened && !grantsCaught && grants != nil {
continue // the switch is not heard yet: nothing is written before it is
@@ -226,11 +224,37 @@ func (f *Follower) Run(ctx context.Context) error {
}
}
// busNode says some composition of this node carries the bus's user list.
// busNode says something this node runs carries the bus's user list, or did.
func (f *Follower) busNode() bool {
f.mu.Lock()
defer f.mu.Unlock()
return f.cur.key != "" || len(f.srcs) > 0
return f.cur.module != "" || len(f.srcs) > 0
}
// look reads what this node runs now for the bus's user list: each module whose composition carries it.
func (f *Follower) look() {
var running []delivery.Composed
if f.env.Running != nil {
running = f.env.Running()
}
srcs := map[string]source{}
for _, c := range running {
if c.Node != "" && c.Node != f.env.Node {
continue
}
raw, _, _, ok := userList(c.Resources)
if !ok {
continue
}
module := c.Module
if module == "" {
module = agent.NodeModule
}
srcs[module] = source{module: module, resource: raw}
}
f.mu.Lock()
f.srcs = srcs
f.mu.Unlock()
}
// takeGrants keeps the grants part's key as heard. A key deleted, or a value it cannot read, is said and changes nothing:
@@ -272,51 +296,6 @@ func (f *Follower) key() string {
return f.env.Key()
}
// take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted.
func (f *Follower) take(u agent.Update) {
node, module, _, isNode, err := delivery.ParseComposedKey(u.Key)
if err != nil || node != f.env.Node || isNode {
return
}
f.mu.Lock()
defer f.mu.Unlock()
if u.Deleted {
delete(f.srcs, u.Key)
return
}
c, err := delivery.DecodeComposed(u.Value)
if err != nil {
delete(f.srcs, u.Key)
return
}
var trusted []ed25519.PublicKey
if f.env.Trusted != nil {
trusted = f.env.Trusted()
}
if _, err := delivery.VerifyComposed(c, trusted); err != nil {
// The agent says it is refused; a refused composition is never a source of the list.
delete(f.srcs, u.Key)
return
}
if c.Node != f.env.Node || c.Module != module {
// Signed, and for another node or module than its key says: a re-put of another composition is no source.
delete(f.srcs, u.Key)
f.sayOnceLocked(fmt.Sprintf("the composition under %s is %s's of %s, and is no source of the bus's user list",
u.Key, c.Node, c.Module))
return
}
raw, content, digest, ok := userList(c.Resources)
if !ok {
delete(f.srcs, u.Key)
return
}
wanted := digest
if wanted == "" {
wanted = Digest([]byte(content))
}
f.srcs[u.Key] = source{key: u.Key, module: module, rev: u.Revision, resource: raw, wanted: wanted}
}
// settle writes the grants part's list: when its key moved, or when asked.
func (f *Follower) settle(ctx context.Context, again bool) { f.settleGrants(ctx, again) }
@@ -337,18 +316,26 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) {
g = *f.grants
}
rev := f.grantsRev
// What runs now: the module written before while it still carries the list, else the first by name; what was
// written before when nothing runs that carries it now (a look between two applies).
file := f.cur
for _, s := range f.srcs {
if s.rev > file.rev {
file = s
if s, ok := f.srcs[f.cur.module]; ok {
file = s
} else if len(f.srcs) > 0 {
names := make([]string, 0, len(f.srcs))
for m := range f.srcs {
names = append(names, m)
}
sort.Strings(names)
file = f.srcs[names[0]]
}
if file.key == "" {
if file.module == "" {
f.mu.Unlock()
return // no composition of this node declares the file: not the bus node
return // nothing this node runs declares the file: not the bus node
}
moved := file.module != f.cur.module || pathOf(file.resource) != pathOf(f.cur.resource)
f.cur = file
if !again && f.state.Source != "" && f.state.Revision == rev && f.written {
if !again && !moved && f.state.Source != "" && f.state.Revision == rev && f.written {
f.mu.Unlock()
return
}
@@ -522,6 +509,15 @@ func userList(resources []json.RawMessage) (json.RawMessage, string, string, boo
return nil, "", "", false
}
// pathOf is where a file resource says its file is; "" when it says none.
func pathOf(raw json.RawMessage) string {
var head struct {
Path string `json:"path"`
}
_ = json.Unmarshal(raw, &head)
return head.Path
}
// Digest is content's digest as the have says it, `sha256:<hex>`.
func Digest(content []byte) string {
sum := sha256.Sum256(content)
internal/userlist/userlist_test.go
+36 -82
View File
@@ -2,7 +2,6 @@ package userlist
import (
"context"
"crypto/ed25519"
"encoding/json"
"errors"
"strings"
@@ -17,17 +16,6 @@ import (
const node = "novox"
func key() (ed25519.PublicKey, ed25519.PrivateKey) {
pub, k, err := ed25519.GenerateKey(nil)
if err != nil {
panic(err)
}
return pub, k
}
var controllerPub, controllerKey = key()
var _, strangerKey = key()
func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) }
// feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once
@@ -69,19 +57,14 @@ func list(users ...string) string {
return b.String()
}
// comp is a composition of module at fp holding the user list content (none when empty), signed by signer; by digest
// when byDigest, as the controller sends a file over 64 KiB; for another node or module than its key when asked.
func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p put) (string, []byte) {
t.Helper()
owner := module
if p.asModule != "" {
owner = p.asModule
}
server, _ := json.Marshal(map[string]any{"id": owner + ".server-conf", "type": "file", "module": owner,
// comp is a composition of module at fp as this node runs it, holding the user list content (none when empty); by
// digest when byDigest, as a file over 64 KiB is given; for another node than this one when asked.
func comp(module, fp, content string, p put) delivery.Composed {
server, _ := json.Marshal(map[string]any{"id": module + ".server-conf", "type": "file", "module": module,
"path": "/var/lib/nats-module/conf/nats.conf", "content": "include accounts.conf\n"})
rs := []json.RawMessage{server}
if content != "" {
r := map[string]any{"id": owner + ".bus-users", "type": "file", "module": owner,
r := map[string]any{"id": module + ".bus-users", "type": "file", "module": module,
"path": "/var/lib/nats-module/conf/accounts.conf", "content": content, "mode": "0600"}
if p.byDigest {
delete(r, "content")
@@ -94,22 +77,7 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p
if p.asNode != "" {
c.Node = p.asNode
}
if p.asModule != "" {
c.Module = p.asModule
}
k, err := delivery.ComposedKey(node, module, fp)
if err != nil {
t.Fatal(err)
}
signed, err := delivery.SignComposed(c, signer)
if err != nil {
t.Fatal(err)
}
raw, err := delivery.EncodeComposed(signed)
if err != nil {
t.Fatal(err)
}
return k, raw
return c
}
// gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to
@@ -125,12 +93,12 @@ type gput struct {
type put struct {
module, fp, content string
stranger, deleted bool
byDigest bool
asNode, asModule string
asNode string
}
// Defends novox/hq ADR 0547 §8: the bus node writes the newest signed list alone, only when it changed, and reports it.
// Defends novox/hq ADR 0547 §8: the bus node writes the grants part's list alone, only when it changed, and reports it,
// into the file the bus module it runs declares (issue 1126, B17: what runs, never a composition bucket).
func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c")
five := list("controller", "a", "b", "c", "d")
@@ -152,13 +120,14 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
// wantSource and wantFollows are what the last have says it took the list from and follows.
wantSource, wantFollows string
}{
// Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module's
// composition declares; the list a composition carries is never written, whatever becomes of the grants key.
{name: "the grants part's list is written into the file a composition declares",
// Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module
// this node runs declares; the list a composition carries is never written, whatever becomes of the grants key.
// B17: the file's place is read from what runs, as the agent holds it, never from a composition bucket.
{name: "the grants part's list is written into the file the bus module it runs declares",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a key that says follow the controller still writes the grants part's list",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}},
{name: "a bus module it runs by digest still names the file",
before: []put{{module: "nats", fp: fpOf("a"), content: two, byDigest: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}},
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a newer composition's list is not written",
before: []put{{module: "nats", fp: fpOf("a"), content: two}},
@@ -167,8 +136,8 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants},
{name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}},
grants: []gput{{follow: delivery.FollowGrants, content: three}}},
{name: "a composition signed by a stranger declares no file",
before: []put{{module: "nats", fp: fpOf("a"), content: two, stranger: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}},
{name: "another node's composition declares no file",
before: []put{{module: "nats", fp: fpOf("a"), content: two, asNode: "ace"}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}},
{name: "an unreadable grants key with none heard before writes nothing, never the composition's list",
before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{unreadable: true}},
onDisk: two, wantErr: "key is not heard", wantRefused: true, wantSource: delivery.GrantsUserListSource},
@@ -201,9 +170,8 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource},
} {
t.Run(c.name, func(t *testing.T) {
f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})}
objects := map[string]string{}
var mu sync.Mutex
var running []delivery.Composed
var writes []string
var reports []Written
reported := make(chan struct{}, 64)
@@ -257,7 +225,12 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
gf.first = append(gf.first, u)
}
}
fl := New(Env{Node: node, Composed: f, Grants: gf,
fl := New(Env{Node: node, Grants: gf,
Running: func() []delivery.Composed {
mu.Lock()
defer mu.Unlock()
return append([]delivery.Composed(nil), running...)
},
Kept: func() ([]byte, uint64) { return keptRaw, 9 },
Keep: func([]byte, uint64) {},
Backoff: func(int) time.Duration { return time.Hour },
@@ -271,8 +244,7 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
out, err := seal.Open(b, nodeKey.PrivateKey)
return []byte(out), err
},
Key: func() string { return nodeKey.PublicKey },
Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} },
Key: func() string { return nodeKey.PublicKey },
OnDisk: func(json.RawMessage) ([]byte, error) {
if c.onDisk == "" {
return nil, nil
@@ -296,30 +268,14 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
},
Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) },
})
update := func(p put) agent.Update {
f.rev++
signer := controllerKey
if p.stranger {
signer = strangerKey
}
k, raw := comp(t, p.module, p.fp, p.content, signer, p)
if p.byDigest {
objects[Digest([]byte(p.content))] = p.content
}
if p.deleted {
return agent.Update{Key: k, Revision: f.rev, Deleted: true}
}
return agent.Update{Key: k, Value: raw, Revision: f.rev}
}
for _, p := range c.before {
f.first = append(f.first, update(p))
running = append(running, comp(p.module, p.fp, p.content, p))
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
done := make(chan error, 1)
go func() { done <- fl.Run(ctx) }()
<-f.watching
// A later put after the first list is written: the first write waits for the grants part's key to catch up.
// What runs changes after the first list is written: the first write waits for the grants part's key.
if len(c.after) > 0 {
for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) {
mu.Lock()
@@ -331,7 +287,10 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
}
}
for _, p := range c.after {
f.ch <- update(p)
mu.Lock()
running = []delivery.Composed{comp(p.module, p.fp, p.content, p)}
mu.Unlock()
fl.RunsChanged()
}
if len(grantsAfter) > 0 {
for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) {
@@ -384,23 +343,18 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) {
if c.wantUnwatched != (st.Unwatched != "") {
t.Fatalf("the have says unwatched %q", st.Unwatched)
}
// While the grants part's list is followed, an apply of the controller's composition writes not its list.
if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" {
cmp, err := delivery.DecodeComposed(raw)
if err != nil {
t.Fatal(err)
}
// While the grants part's list is followed, an apply of the bus module's composition writes not its list.
if cmp := comp("nats", fpOf("a"), two, put{}); c.wantFollows != "" {
held, err := fl.Hold(cmp)
if err != nil {
t.Fatal(err)
}
_, _, _, hasList := userList(held.Resources)
if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 {
t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList)
if _, _, _, hasList := userList(held.Resources); hasList || len(held.Resources) < 1 {
t.Fatalf("an apply while following %s holds the composition's list: %v", c.wantFollows, hasList)
}
// A list on the machine that cannot be read stops the apply; it is never left out to be removed.
if c.wantFollows == delivery.FollowGrants {
readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) {
readErr := New(Env{Node: node, OnDisk: func(json.RawMessage) ([]byte, error) {
return nil, errors.New("permission denied")
}})
readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants}
internal/witness/contract.go
+12 -113
View File
@@ -1,155 +1,54 @@
// Package witness is the node-engine watching a core build it placed — the controller on the control
// node, the node tools on every machine — and restoring the build before it when the new one is not
// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the
// only witness of its successor).
// Package witness is the node-engine watching a core build it placed — the node tools on every machine — and
// restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the
// component being replaced is never the only witness of its successor).
//
// **The contract is this file.** What the engine reads to call a build healthy, from where, in what
// shape, within which bound — said once, here, and held by contract_test.go. The controller's side
// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a
// change to this file and its test.
// shape, within which bound — said once, here, and held by contract_test.go.
package witness
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/novox/mesh-host/internal/link"
)
// The two witnesses, as a process declares which watches it (`witness`). A process that says none is
// placed as ever; one that says nothing is judged by its name's default (Default).
// The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one
// that says nothing is judged by its name's default (Default).
const (
// ByLease: healthy when the controller this machine started holds the controller's lease.
ByLease = "lease"
// ByPing: healthy when this machine's runtime answers the services protocol's PING.
ByPing = "ping"
// ByNone: not judged.
ByNone = "none"
)
// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket
// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five.
const (
LeaseBucket = "mesh-controller_lease"
LeaseKey = "holder"
// LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is
// not holding it, whatever the key still says.
LeaseAge = 15 * time.Second
// Skew is how far the controller's clock and this host's may disagree about when it took the
// lease. One machine, one clock, in practice: a margin, not a tolerance.
Skew = 2 * time.Second
)
// Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every.
const (
// ControllerWithin: the controller holds the lease within sixty seconds of starting.
ControllerWithin = 60 * time.Second
// NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting,
// each PING answered within PingWithin.
NodeToolsWithin = 60 * time.Second
PingWithin = 5 * time.Second
Every = 5 * time.Second
// GiveUp is how long a witness goes on when it cannot ask at all before it says the build is
// unwitnessed: the grant missing, the bucket missing, the bus away the whole time.
// unwitnessed: the grant missing, the bus away the whole time.
GiveUp = 30 * time.Minute
)
// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease
// Holder, field for field by its JSON names. Only what the witness reads is required; a field the
// controller adds later is ignored here.
type ControllerLease struct {
// Instance names one controller process: "controller@<machine> pid <pid> since <RFC 3339>".
Instance string `json:"instance"`
// Host is the machine it runs on, as its own hostname says.
Host string `json:"host,omitempty"`
// Build is the controller's build as it knows it; not read here — the toolchain stamps no
// version, so it says "development build" — and a bundle's identity is its digest, which the
// host already knows.
Build string `json:"build,omitempty"`
Epoch uint64 `json:"epoch,omitempty"`
// Taken is when this instance took the key, Renewed when it last renewed it, by its own clock.
Taken time.Time `json:"taken"`
Renewed time.Time `json:"renewed"`
}
// ParseLease reads the key's value.
func ParseLease(value []byte) (ControllerLease, error) {
var l ControllerLease
if err := json.Unmarshal(value, &l); err != nil {
return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err)
}
return l, nil
}
// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or
// after `since` — the controller this machine started then, and not the one before it — and is
// held now. Why says what it saw when it is not.
func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) {
switch {
case l.Instance == "":
return false, "the lease names no holder"
case host != "" && !sameMachine(l.Host, host):
return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host)
case l.Taken.Before(since.Add(-Skew)):
return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s",
l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339))
case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge:
return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s",
l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge)
}
return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch)
}
// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree.
func sameMachine(a, b string) bool {
short := func(s string) string {
s = strings.ToLower(strings.TrimSpace(s))
if i := strings.IndexByte(s, '.'); i > 0 {
s = s[:i]
}
return s
}
return short(a) == short(b)
}
func latest(a, b time.Time) time.Time {
if b.After(a) {
return b
}
return a
}
// NodeToolsService is the runtime's name on the services protocol, and its instance is this
// machine's node name: what `$SRV.PING.<service>.<node>` asks, so only this machine's runtime can
// answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}).
const NodeToolsService = "node-tools"
// Default is the witness a process is judged by when it names none: the two core processes the mesh
// composes, by the names it composes them under, and nothing else.
// Default is the witness a process is judged by when it names none: the core process the mesh composes on every
// machine, by the name it composes it under, and nothing else.
func Default(process string) string {
switch process {
case "mesh-controller":
return ByLease
case NodeToolsService:
if process == NodeToolsService {
return ByPing
}
return ByNone
}
// Within is the bound for a witness.
func Within(by string) time.Duration {
if by == ByLease {
return ControllerWithin
}
return NodeToolsWithin
}
func Within(string) time.Duration { return NodeToolsWithin }
// Component names what a witness judges, as a rollback says it.
func Component(by string) string {
if by == ByLease {
return link.ComponentController
}
return link.ComponentNodeTools
}
func Component(string) string { return link.ComponentNodeTools }
internal/witness/contract_test.go
+9 -84
View File
@@ -9,82 +9,7 @@ import (
"github.com/novox/mesh-host/internal/link"
)
// The contract, held. Each of these is a line the controller's side relies on or writes; a change on
// either side changes this test (novox/hq to-be 45 §8).
// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names.
func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) {
if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" {
t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder",
LeaseBucket, LeaseKey)
}
if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" {
t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got)
}
if LeaseAge != 15*time.Second {
t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge)
}
}
// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is
// read. The value below is the shape that Holder marshals to.
func TestTheLeaseValueIsTheControllersHolder(t *testing.T) {
written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` +
`"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}`
l, err := ParseLease([]byte(written))
if err != nil {
t.Fatal(err)
}
if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() {
t.Fatalf("the lease was not read whole: %+v", l)
}
// And a field the controller adds later does not stop it being read.
if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil {
t.Fatalf("a lease value with a field this host does not know was refused: %v", err)
}
}
// Healthy is: held now, on this machine, by an instance that took it after the new build started.
func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) {
started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
now := started.Add(30 * time.Second)
held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example",
Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58}
for _, c := range []struct {
name string
l ControllerLease
want bool
says string
}{
{"held by the new instance", held, true, "holds the lease"},
{"nobody", ControllerLease{}, false, "no holder"},
{"the old instance, taken before the restart", func() ControllerLease {
l := held
l.Taken = started.Add(-time.Hour)
return l
}(), false, "before the new build started"},
{"another machine's controller", func() ControllerLease {
l := held
l.Host = "home-server"
return l
}(), false, "not this machine"},
{"taken and no longer renewed", func() ControllerLease {
l := held
l.Renewed = now.Add(-20 * time.Second)
return l
}(), false, "past its"},
{"taken within the clocks' skew of the start", func() ControllerLease {
l := held
l.Taken = started.Add(-time.Second)
return l
}(), true, "holds the lease"},
} {
got, why := c.l.HeldBySince("anchor", started, now)
if got != c.want || !strings.Contains(why, c.says) {
t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says)
}
}
}
// The contract, held (novox/hq to-be 45 §8).
// The runtime is asked by its own name and this machine's: only this machine's runtime can answer.
func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
@@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) {
}
}
// What is judged by default is the mesh's two core processes, by the names the controller composes
// them under, and nothing else.
func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) {
for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} {
// What is judged by default is the mesh's core process on every machine, by the name it is composed under, and
// nothing else: the controller ended (issue 1126), so its name is judged as any other.
func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) {
for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} {
if got := Default(name); got != want {
t.Errorf("%s is judged by %q, want %q", name, got, want)
}
}
if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five")
if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second {
t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five")
}
}
// A verdict on the wire names its component as the controller's condition does.
func TestAVerdictNamesItsComponent(t *testing.T) {
raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack})
if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" {
raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack})
if !strings.Contains(string(raw), `"component":"node-tools"`) {
t.Fatalf("a verdict names its component as %s", raw)
}
}
internal/witness/draining_test.go
+3 -3
View File
@@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string {
// after its build was moved into a 0700 directory, and then deleted, could not run its own verbs.
func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) {
root := t.TempDir()
name := "mesh-controller"
name := "node-tools"
start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC)
// Build A placed and running.
p, err := Place(root, name, ByLease, buildA, "", "", start)
p, err := Place(root, name, ByPing, buildA, "", "", start)
if err != nil {
t.Fatal(err)
}
@@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T)
}
// Build B placed while A still runs: A is moved aside, and reachable at its new path.
place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute))
place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute))
for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} {
info, err := os.Stat(dir)
if err != nil {
internal/witness/watch.go
+5 -20
View File
@@ -14,15 +14,14 @@ import (
// concluded (to-be 45 §8).
//
// **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer
// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a
// from the bus — the PING delivered — so a machine whose own link is down, or a
// bus that refuses the question, never rolls a build back for the host's own trouble. A look that
// could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed:
// neither proved nor rolled back, and said.
type Watcher struct {
Root string
// Node is this machine's node name, the instance its runtime answers PING as; Host its hostname,
// as the controller's lease names its machine.
Node, Host string
// Node is this machine's node name, the instance its runtime answers PING as.
Node string
// Asker is the link open now, or nil.
Asker func() link.Asker
Run Runner
@@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
every = Every
}
now := w.now()
healthy, why, err := w.judge(ctx, s, now)
healthy, why, err := w.judge(ctx, s)
w.hold(func() {
// Read again in turn: an apply may have placed another build meanwhile, which starts its own
@@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) {
}
// judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not.
func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) {
func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) {
var asker link.Asker
if w.Asker != nil {
asker = w.Asker()
@@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri
asking, cancel := context.WithTimeout(ctx, PingWithin)
defer cancel()
switch s.Witness {
case ByLease:
value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey)
if err != nil {
return false, "", err
}
if !found {
return false, "nobody holds the controller's lease", nil
}
lease, err := ParseLease(value)
if err != nil {
return false, err.Error(), nil
}
held, why := lease.HeldBySince(w.Host, s.Started, now)
return held, why, nil
case ByPing:
err := asker.Ping(asking, s.Process, w.Node)
switch {
internal/witness/watch_test.go
+47 -86
View File
@@ -13,8 +13,8 @@ import (
"github.com/novox/mesh-host/internal/link"
)
// Each rollback path, induced against a real directory: a controller that never takes the lease, a
// runtime that never answers PING, a build declared not reversible; and a healthy update, which
// Each rollback path, induced against a real directory: a runtime that never answers PING, a build
// declared not reversible; and a healthy update, which
// deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8).
const (
@@ -62,18 +62,10 @@ func kept(root, name string) bool {
// asker answers as the build it is told is running would.
type asker struct {
mu sync.Mutex
lease func() ([]byte, bool, error)
ping func() error
asked int
}
func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) {
a.mu.Lock()
defer a.mu.Unlock()
a.asked++
return a.lease()
}
func (a *asker) Ping(context.Context, string, string) error {
a.mu.Lock()
defer a.mu.Unlock()
@@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e
}
func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher {
return &Watcher{Root: root, Node: "anchor", Host: "anchor",
return &Watcher{Root: root, Node: "anchor",
Asker: func() link.Asker {
if a == nil {
return nil
@@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) {
}
}
func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) {
return func() ([]byte, bool, error) {
return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`,
instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil
}
}
// A controller that starts and never takes the lease: restored to the build before, once, said once.
func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
// A runtime that starts and never answers PING: restored to the build before, once, said once.
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") {
t.Fatal("the new controller is not running with the build before it kept beside it")
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") {
t.Fatal("the new runtime is not running with the build before it kept beside it")
}
// The old instance's lease, taken an hour ago, is all the bus ever shows.
a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)}
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(ControllerWithin/Every)-1)
if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB {
look(w, &clock, int(NodeToolsWithin/Every)-1)
if len(verdicts) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("rolled back before its bound: %+v", verdicts)
}
look(w, &clock, 1)
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA ||
verdicts[0].Component != link.ComponentController {
t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts)
verdicts[0].Component != link.ComponentNodeTools {
t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts)
}
if running(t, root, "mesh-controller") != buildA {
t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller"))
if running(t, root, "node-tools") != buildA {
t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools"))
}
if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" {
if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" {
t.Fatalf("the machine was asked %v", m.commands)
}
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") {
if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") {
t.Fatalf("the verdict does not say why: %s", verdicts[0].Why)
}
@@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) {
t.Fatalf("what every report says is %+v", s)
}
// The mesh still declares B: refused, A kept running, the verdict still standing.
p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock)
if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") {
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller"))
p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock)
if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") {
t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools"))
}
if len(Standing(root)) != 1 || len(Trials(root)) != 0 {
t.Fatal("refusing the rolled-back build changed what stands or started a trial")
}
// A newer build is a new trial, and ends what was concluded once it is proved.
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 {
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 {
t.Fatal("a newer build after a rollback was not placed on trial")
}
if err := Proved(root, "mesh-controller"); err != nil {
if err := Proved(root, "node-tools"); err != nil {
t.Fatal(err)
}
if len(Standing(root)) != 0 || kept(root, "mesh-controller") {
if len(Standing(root)) != 0 || kept(root, "node-tools") {
t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept")
}
}
// A runtime that never answers PING: restored, once.
func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(NodeToolsWithin/Every)+20)
if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools {
t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts)
}
if running(t, root, "node-tools") != buildA || kept(root, "node-tools") {
t.Fatal("the runtime running is not the build before, or a copy of it is still kept")
}
}
// A healthy update: nothing deleted before it is proved; the build before it deleted once it is.
func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
// The controller placed by a host from before any witness: a record, and no state.
if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil {
// The runtime placed by a host from before any witness: a record, and no state.
if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil {
if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil {
t.Fatal(err)
}
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
started := clock
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
a := &asker{ping: func() error { return link.ErrNoAnswer }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, 5)
if !kept(root, "mesh-controller") {
if !kept(root, "node-tools") {
t.Fatal("the build before was deleted while the new one was still on trial")
}
// A third build while the second is on trial: the one kept is still the one seen healthy.
place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock)
if s, _ := Load(root, "mesh-controller"); s.Previous != buildA {
place(t, root, "node-tools", ByPing, buildC, buildB, "", clock)
if s, _ := Load(root, "node-tools"); s.Previous != buildA {
t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous)
}
started = clock
a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock)
a.ping = func() error { return nil }
look(w, &clock, 2)
if len(verdicts) != 0 || len(m.commands) != 0 {
t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands)
}
if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC {
if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC {
t.Fatal("the build before was not deleted once the new one was proved")
}
}
@@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) {
func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock)
a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }}
a := &asker{ping: func() error { return link.ErrNoAnswer }}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(ControllerWithin/Every)+20)
look(w, &clock, int(NodeToolsWithin/Every)+20)
if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" {
t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts)
@@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") {
t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why)
}
if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
if len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands,
running(t, root, "mesh-controller"))
running(t, root, "node-tools"))
}
if len(Standing(root)) != 1 {
t.Fatal("the not-reversible verdict does not stand")
@@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) {
func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) {
root := t.TempDir()
clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
place(t, root, "mesh-controller", ByLease, buildA, "", "", clock)
place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock)
place(t, root, "node-tools", ByPing, buildA, "", "", clock)
place(t, root, "node-tools", ByPing, buildB, buildA, "", clock)
a := &asker{lease: func() ([]byte, bool, error) {
return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk)
a := &asker{ping: func() error {
return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk)
}}
m := &machine{}
var verdicts []link.Rollback
w := watcher(root, a, m, &clock, &verdicts)
look(w, &clock, int(GiveUp/Every)-1)
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB {
if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB {
t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands)
}
look(w, &clock, 1)
if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 {
t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts)
}
if !kept(root, "mesh-controller") {
if !kept(root, "node-tools") {
t.Fatal("the build before was deleted though the new one was never seen healthy")
}