diff --git a/README.md b/README.md index fd4e736..c70a77e 100644 --- a/README.md +++ b/README.md @@ -171,26 +171,6 @@ carries. The claim is not. The foundation is four container services, and: So `foundation.lock` here is a real bundle with a placeholder's content. Saying that plainly beats shipping a host that claims a foundation it has never raised. -## mesh-cli's socket (novox/hq ADR 0272) - -The engine serves `/run/nox-mesh-host/cli.sock` for `mesh-cli`, the operator's command (`internal/meshcli`). It -reads the asking account from the kernel (`SO_PEERCRED`) and asks the controller on this node's own subject; the -controller decides what the line may do, and the engine runs nothing it receives. One line per connection, each -connection's deadline set the moment it is accepted, two lines at once per account and eight for the machine, -"busy" beyond that. - -**Only a login session can be the terminal.** The engine reads the asking process's cgroup and sends its login -session's scope (`session-.scope` under the account's own `user-.slice`, which logind's own record in -`/run/systemd/sessions/` must say is this account's, of class `user`), or nothing, read the moment the -connection is accepted; the controller makes -nothing else the terminal. A service running as the operator — the tool runner, a unit of the account's user -manager — is at most an ordinary call. - -**A process outside the machine's own namespaces is refused**, whatever its user id: the peer's mount and user -namespace must be the engine's. So `mesh-cli` run inside a flatpak, a toolbox or distrobox container, or any other -container — even one running as the operator's own id — is refused with the reason; run it from the machine's own -shell. - ## A capability is detected, never assumed The reason this is the first thing built rather than a detail of it. diff --git a/cmd/mesh-host/authority_test.go b/cmd/mesh-host/authority_test.go index 78bf7be..21b4f20 100644 --- a/cmd/mesh-host/authority_test.go +++ b/cmd/mesh-host/authority_test.go @@ -9,6 +9,7 @@ import ( "encoding/pem" "os" "path/filepath" + "strings" "testing" "time" @@ -73,7 +74,7 @@ func TestTheAppliedAuthorityIsKeptForTheRecord(t *testing.T) { if _, err := trust.Verify(delivery.KindEnrolment, msg, delivery.SignMessage(controller, enrol, msg)); err == nil { t.Error("a controller-signed message is accepted once the vault's key is read") } - if got := keys().Trusts; delivery.TrustsKey(got, delivery.SignerController, ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || + if got := keys().Trusts; strings.Contains(strings.Join(got, " "), ctrlFP) || !delivery.TrustsKey(got, delivery.SignerVault, vaultFP) || trust.VaultAuthority() != vaultCA || told != 2 { t.Errorf("after the signers are read the record trusts %v (told %d)", got, told) } diff --git a/cmd/mesh-host/engineseat.go b/cmd/mesh-host/engineseat.go index c9ec77f..6cba6a6 100644 --- a/cmd/mesh-host/engineseat.go +++ b/cmd/mesh-host/engineseat.go @@ -333,8 +333,8 @@ func composeFrom(current *agentNow, composer *atomic.Pointer[selfcompose.Compose // packagesReadFor bounds the verb packages' reads and its write. const packagesReadFor = 20 * time.Second -// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages -// beside the controller's mirror's key for it and their difference; with write, the key written again from what the +// packagesFrom answers the verb packages (novox/hq ADR 0565 §6, §15): this node's key of the engine's state packages; +// with write, the key written again from what the // composer that runs now handed, refused in words while the node composes none of its own. func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[selfcompose.Composer]) func(write bool) (any, error) { return func(write bool) (any, error) { @@ -344,8 +344,8 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s if write { c := composer.Load() if c == nil { - return nil, errors.New("this node composes none of its own modules now (it applies the controller's " + - "compositions, or its node agent is starting), so it has no packages of its own to write") + return nil, errors.New("this node composes none of its own modules now (its node agent is starting), " + + "so it has no packages of its own to write") } w, err := c.WritePackages(ctx) if err != nil { @@ -354,12 +354,7 @@ func packagesFrom(node string, session *link.Session, composer *atomic.Pointer[s wrote = &w } key, keyErr := link.EnginePackages{Session: session}.Get(ctx, node) - var mirror []byte - h, mirrorErr := session.KVHistory(ctx, link.MirrorPackagesBucket, node) - if mirrorErr == nil && len(h) > 0 && !h[len(h)-1].Deleted { - mirror = h[len(h)-1].Value - } - v := link.PackagesAnswer(node, key, mirror, keyErr, mirrorErr) + v := link.PackagesAnswer(node, key, keyErr) v.Wrote = wrote return v, nil } diff --git a/cmd/mesh-host/enrol_seat.go b/cmd/mesh-host/enrol_seat.go index 4d253a8..94cb23b 100644 --- a/cmd/mesh-host/enrol_seat.go +++ b/cmd/mesh-host/enrol_seat.go @@ -16,17 +16,17 @@ import ( // enrolThroughSeat joins this machine through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): it redeems the token // with the keys it made, verifies the vault's signed answer against the key the token carries before it keeps anything, keeps -// its membership and its login, dials as the login node-logins names and confirms. handled is false when the bus does not serve -// the seat to this token (the controller still holds its lease, so its enrolment subject is the way in): nothing was sent or -// kept then, and the caller enrols as before. +// its membership and its login, dials as the login node-logins names and confirms. A bus that does not serve the seat to this +// token (link.ErrNoSeat) is said as the error: nothing was sent or kept. func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, node string, mine identity.Identity, - sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) (handled bool, err error) { + sealing identity.SealingKey, serving identity.ServingKey, reported map[string]any, found *link.Tunnel, identityPath string) error { req := delivery.Redeem{Node: node, PublicKey: base64.StdEncoding.EncodeToString(mine.Public), OverlayKey: mine.Overlay.Public, SealingKey: sealing.Public, ServingKey: serving.Public, Profile: reported} if found != nil { + var err error if req.Tunnel, err = json.Marshal(found); err != nil { - return true, err + return err } } req.Proof = base64.StdEncoding.EncodeToString(mine.Sign(delivery.RedeemProof(node, req.PublicKey, req.OverlayKey, req.SealingKey, req.ServingKey))) @@ -61,20 +61,13 @@ func enrolThroughSeat(ctx context.Context, opts options, token identity.Token, n }, } answer, err := j.Run(ctx) - switch { - case errors.Is(err, link.ErrNoSeat): - // The path before the lease write: the controller's enrolment subject is granted to the token only while the - // controller holds its lease, so after the write this fallback is refused too, and that refusal is what is said. - fmt.Printf("the seat mesh-enrolment is not served to this token (%v); enrolling through the controller's enrolment "+ - "subject, the path before the lease write, which the bus refuses once the lease is written\n", err) - return false, nil - case err != nil: - return true, err + if err != nil { + return err } fmt.Printf("\nenrolled as %s through the seat mesh-enrolment, dialled as %s and confirmed\n", answer.Node, answer.Login) fmt.Printf(" identity %s\n", identityPath) fmt.Println(" nothing is applied until the placement of this node names a module") - return true, nil + return nil } // keepJoinedKeys writes the private halves of the keys this machine made, after its identity: an identity naming a key that is diff --git a/cmd/mesh-host/healthowed_test.go b/cmd/mesh-host/healthowed_test.go index 08c5d96..a46871c 100644 --- a/cmd/mesh-host/healthowed_test.go +++ b/cmd/mesh-host/healthowed_test.go @@ -21,7 +21,6 @@ type owedBus struct { rv map[string]uint64 } -func (b *owedBus) Report(context.Context, string, []byte) error { return nil } func (b *owedBus) Health(context.Context, string, []byte) error { return nil } func (b *owedBus) HaveBind(context.Context) (link.HaveBucket, error) { return b, nil } diff --git a/cmd/mesh-host/main.go b/cmd/mesh-host/main.go index 8ca4e66..3045eb9 100644 --- a/cmd/mesh-host/main.go +++ b/cmd/mesh-host/main.go @@ -100,7 +100,7 @@ func versionAt(self, stamped string) string { const usage = `nox-mesh-host — the node-engine A daemon: its launcher runs it, and it applies what the mesh declares for this machine. The -operator's command is nox (mesh-cli), on any node, from their own account (novox/hq ADR 0272); +operator's command is nox (mesh-cli), on any node, from their own account, through the seats' verbs; the subcommands here are for installing a machine, joining it, and rescuing one no mesh can reach, run as root at the machine. @@ -110,8 +110,7 @@ reach, run as root at the machine. key make this machine's tunnel key, or read the one it made, and print the public half: what its join token is issued for (novox/hq ADR 0169) enrol --token T join the mesh — through the tunnel when the token was issued for a key - overlay take take over the tunnel found here (novox/hq ADR 0105): its key becomes this - node's overlay key and the mesh is told, signed; --tunnel when several are up + overlay take refused: the controller that took a rekey ended and no seat takes one now owned what this host has applied and still owns hand-over hand a directory used as found to the mesh: the next apply gives it its declared owner and mode. The operator's line is nox node hand-over on the @@ -809,167 +808,21 @@ func enrol(ctx context.Context, opts options) error { detected := profile.Detect(ctx, profile.Default(nil), opts.timeout) reported := profileAsReported(detected) - // Signed with the identity just generated, so the mesh can tell this machine from anyone else - // who knows its public key (novox/hq issue 083). - proof := mine.Sign(link.EnrolProof(token.Secret, mine.Public, mine.Overlay.Public, - sealing.Public, serving.Public)) - // The token says where to go and which certificate that address must present. It says nothing - // about which bus is there, and does not need to: there is one, and this host knows which - // (novox/hq ADR 0131 — the mesh speaks to one seat and the old transport is gone). - // - // **It used to leave this empty** and mean "whatever the mesh runs today", which was true - // while two buses existed and became a refusal the moment one did: an empty transport is not - // the bus's name, so every enrolment ended at "this token is for the \"\" bus" - // (novox/hq 04-ISSUES/146). Nothing caught it because nothing had enrolled since the bus - // changed. - // Through the seat mesh-enrolment when the bus serves it to this token (novox/hq ADR 0587 §6): the controller's enrolment - // subject below stays for as long as the controller holds its lease. - if handled, err := enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath); handled || err != nil { - return err - } - reply, err := link.Enrol(ctx, - link.Approach{Address: token.Broker, Fingerprint: token.Fingerprint, Transport: link.OnNATS}, - *name, token.Secret, - mine.Public, mine.Overlay.Public, sealing.Public, serving.Public, reported, proof, found, - opts.timeout) - if err != nil { - return err - } - - // The mesh's name for this node wins over what the machine called itself: the token was - // issued for a node record, and that record is what the identity binds to. - mine.Node = reply.Node - mine.Membership = identity.Membership{ - Broker: firstNonEmpty(reply.Broker, token.Broker), - Fingerprint: firstNonEmpty(reply.Fingerprint, token.Fingerprint), - Signer: firstNonEmpty2(reply.Signer, token.Signer), - Password: reply.Password, - // **Which bus this membership is for, said rather than left empty** (novox/hq - // 04-ISSUES/146). The link refuses a membership that names another bus, and an empty name - // is not this one's — so a node enrolled without it came up and reconnected for ever - // against its own record: "this membership is for \"\", and the mesh's bus is nats". The - // reply does not carry it because there is one bus and the host knows which (ADR 0131); - // what was missing was writing that down where the link reads it. - Transport: link.OnNATS, - } - if mine.Membership.Password == "" { - // The mesh did not replace the token's secret, so it is still this node's broker - // password. Said rather than silently kept: a one-time secret living on as a credential - // is worth knowing about. - mine.Membership.Password = token.Secret - fmt.Println("\nnote: the mesh issued no separate broker password, so the token's secret " + - "remains this node's credential") - } - - // Saved only now, and only once the mesh has said it knows this node. A node holding an - // identity the mesh has never recorded would believe it had joined and be believed by - // nobody — worse than not having joined, because nothing would look wrong. - if err := identity.Save(identityPath, mine); err != nil { - return fmt.Errorf( - "the mesh accepted this node as %q and its identity could not be saved: %w\n"+ - "That token is spent, so getting back needs a new one", reply.Node, err) - } - - if !mine.Membership.Joined() { - return fmt.Errorf( - "the mesh accepted this node as %q but did not say how to reach it again, so this "+ - "identity could not be used after a restart. Nothing was saved", reply.Node) - } - - // Written before the identity, so a node that dies between the two has a key file with no - // identity — which enrols again cleanly — rather than an identity naming a key that is not - // there, which looks joined and cannot come up. - if err := os.WriteFile(identity.OverlayKeyPath(opts.state), - []byte(mine.Overlay.Private+"\n"), 0o600); err != nil { - return fmt.Errorf("cannot write this node's overlay key: %w", err) - } - if err := os.WriteFile(identity.SealingKeyPath(opts.state), - []byte(sealing.Private+"\n"), 0o600); err != nil { - return fmt.Errorf("cannot write this node's sealing key: %w", err) - } - if err := identity.WriteServingKey(identity.ServingKeyPath(opts.state), serving); err != nil { - return fmt.Errorf("cannot write this node's serving key: %w", err) - } - - fmt.Printf("\nenrolled as %s\n", reply.Node) - fmt.Printf(" identity %s\n", identityPath) - fmt.Printf(" queue %s\n", reply.Queue) - return nil + // Through the seat mesh-enrolment (novox/hq ADR 0587 §6), the one way in: the controller's enrolment subject ended + // with the controller (issue 1126). + return enrolThroughSeat(ctx, opts, token, *name, mine, sealing, serving, reported, found, identityPath) } -// overlayCommand is `mesh-host overlay take`: this node takes the tunnel found on its machine over -// after it enrolled (novox/hq ADR 0105). For a node that enrolled before the mesh knew to take a -// tunnel over — re-enrolling would rotate its identity, sealing and serving keys, and with them -// every credential the mesh sealed to it. -func overlayCommand(ctx context.Context, opts options) error { +// overlayCommand is `mesh-host overlay take`: a node taking the tunnel found on its machine over after it enrolled +// (novox/hq ADR 0105). Refused, and nothing is changed: the rekey it sent was taken by the controller alone, which +// ended by its unassignment (issue 1126), and no seat takes one now; a key written here that no part of the mesh knows +// would raise the mesh's interface on a key its hub does not peer. +func overlayCommand(_ context.Context, opts options) error { if opts.file != "take" { return errors.New("overlay take [--tunnel ] — the one thing `overlay` does here") } - identityPath := identity.Path(opts.state) - mine, err := identity.Load(identityPath) - if err != nil { - return err - } - found, err := tunnel.Find(ctx, apply.ExecRunner, opts.tunnel) - if err != nil { - return fmt.Errorf("%w. Nothing was changed", err) - } - taken, rekey, err := rekeyOnto(mine, found) - if err != nil { - return err - } - fmt.Printf("taking over %s: this node's overlay key becomes the tunnel's, %s\n", found, taken.Overlay.Public) - fmt.Printf(" identity, sealing and serving keys are untouched\n") - - // Told first, then written: a mesh told and a machine not yet written is put right by running - // this again (the mesh refuses the stale second rekey and changes nothing; the files are - // rewritten the same). A machine written and a mesh not told would raise the mesh's interface - // on a key the mesh does not know at the next restart. - if err := link.Publish(ctx, link.Membership{ - Node: mine.Node, Broker: mine.Membership.Broker, Fingerprint: mine.Membership.Fingerprint, - Password: mine.Membership.Password, Signer: mine.Membership.Signer, - Transport: mine.Membership.Transport, - }, link.Report{Node: mine.Node, Rekey: &rekey}, opts.timeout); err != nil { - return fmt.Errorf("the mesh could not be told; nothing was written here: %w", err) - } - fmt.Printf(" told the mesh signed rekey sent; `node show %s` on the controller says whether it took\n", mine.Node) - - if err := os.WriteFile(identity.OverlayKeyPath(opts.state), - []byte(taken.Overlay.Private+"\n"), 0o600); err != nil { - return fmt.Errorf("the mesh was told and this node's overlay key could not be written: %w — run this again", err) - } - if err := identity.Save(identityPath, taken); err != nil { - return fmt.Errorf("the mesh was told and this node's identity could not be saved: %w — run this again", err) - } - fmt.Printf(" written %s and the identity; the mesh's interface reads the key when the next push restarts it\n", - identity.OverlayKeyPath(opts.state)) - fmt.Printf(" next on the controller: `overlay place %s --hub --endpoint :%d …`, `plan %s --json`, then push\n", - mine.Node, found.Port, mine.Node) - return nil -} - -// rekeyOnto is the identity with the found tunnel's key as its overlay key, and the signed rekey -// that tells the mesh. Pure, so it can be held to: node, sealing and serving keys are the same -// bytes in and out; only the overlay key moves. Run again after a take, the previous key it names -// is the one before the take, so the mesh can tell a repeat from a replay. -func rekeyOnto(mine identity.Identity, found tunnel.Found) (identity.Identity, link.Rekey, error) { - overlay, err := identity.OverlayKeyFrom(found.PrivateKey()) - if err != nil { - return identity.Identity{}, link.Rekey{}, err - } - previous := mine.Overlay.Public - if previous == overlay.Public && mine.OverlayBefore != "" { - previous = mine.OverlayBefore - } - taken := mine - taken.Overlay = overlay - if previous != overlay.Public { - taken.OverlayBefore = previous - } - presented := carried(found) - rekey := link.Rekey{Previous: previous, OverlayKey: overlay.Public, Tunnel: presented} - rekey.Proof = mine.Sign(link.RekeyProof(mine.Node, previous, overlay.Public, presented)) - return taken, rekey, nil + return errors.New("overlay take is refused: the controller that took a rekey ended (issue 1126) and no seat " + + "takes one now. Nothing was changed") } // carried is a found tunnel as it is presented to the mesh: everything but its private key. @@ -1120,9 +973,6 @@ func runLink(ctx context.Context, opts options) error { Membership: membership, // And the public halves and certificate its record says (novox/hq ADR 0555 §3), read at each facts report. Keys: recordKeys(opts.state, trust.Trusts), - // And no controller's key: every ask the controller would sign is refused, never verified with the membership's - // signer (novox/hq ADR 0563 §7; issue 1126, the controller ended by its unassignment). - ControllerKey: noControllerKey, // And the first have this build writes is what proves it to the launcher (ADR 0539 §9, ADR 0227 §8). HaveWritten: proof.haveWritten, // And the operator's hand-over of a directory used as found, carried by the controller from the @@ -1213,9 +1063,6 @@ func runLink(ctx context.Context, opts options) error { }, BusyAskEvery) // And the core builds this host placed are judged, whichever host placed them (to-be 45 §8). go watchWhatThisHostPlaced(aside, mine.Node, queue, say) - // And mesh-cli is answered on this machine: the account from the kernel, the line asked of the controller - // (novox/hq ADR 0272). - go serveMeshCLI(aside, queue, say) // And a filled file is written on the provisioner client's request, one at a time with the apply (novox/hq ADR // 0349 §4). go serveFilling(aside, opts, mine.Membership.Signer, queue, mine.Node, say) @@ -1236,8 +1083,6 @@ func runLink(ctx context.Context, opts options) error { // be a control surface on every machine, reachable by anything that can reach the machine, in // exchange for saving twenty seconds — and the whole security argument rests on there not being // one. A signal is delivered by the service manager to a process it already supervises. -// (mesh-cli's socket is no such surface: it has no network address and nothing sent to it acts on -// the machine — it is asked of the controller, novox/hq ADR 0272 §3.) // // SIGHUP, because that is the signal a long-running program conventionally reads as *look again*, // and nothing here is being reloaded from a file that a different signal would suit better. diff --git a/cmd/mesh-host/meshcli.go b/cmd/mesh-host/meshcli.go deleted file mode 100644 index 8960c27..0000000 --- a/cmd/mesh-host/meshcli.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "context" - - "git.novox.be/novox/mesh-sdk/go/cli" - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/meshcli" -) - -// serveMeshCLI answers mesh-cli on this machine for as long as this engine serves (novox/hq ADR 0272): the account -// from the kernel, the line asked of the controller over the link open at the time. A socket that cannot be served -// is said once, and this engine goes on applying; mesh-cli then says the engine does not answer. -func serveMeshCLI(ctx context.Context, queue *link.Session, say func(string)) { - if err := meshcli.Serve(ctx, cli.Socket, forwardOver(queue), say); err != nil { - say("mesh-cli is not answered on this machine: " + err.Error()) - } -} - -// forwardOver asks the controller over the queue's link, and follows the line to what came of it. -func forwardOver(queue *link.Session) meshcli.Forward { - return func(ctx context.Context, asked meshcli.Asked) cli.Answer { - return meshcli.AskTheController(ctx, queue.AskController, asked) - } -} diff --git a/cmd/mesh-host/nodeagent.go b/cmd/mesh-host/nodeagent.go index 0a12e7e..c9a140a 100644 --- a/cmd/mesh-host/nodeagent.go +++ b/cmd/mesh-host/nodeagent.go @@ -3,7 +3,6 @@ package main import ( "context" "crypto/ed25519" - "encoding/base64" "encoding/json" "errors" "fmt" @@ -375,42 +374,6 @@ func resourcesOf(cs []delivery.Composed) []declaration.Resource { return out } -// operatorKeysFile is the file beside the node's state that holds the operator's public keys, one ed25519 key in base64 a -// line (a line that starts with # is a note). The switch's last declaration writes it as a file resource, beside the -// controller's key the node was enrolled with (novox/hq ADR 0539 §3, §11); a node adopted later has it at its adoption. -const operatorKeysFile = "operator-keys" - -// trustedKeys are the keys a composition in the node's composition bucket may be signed by: the one given (none for the -// agent; the enrolment's signer for the user-list follower, which reads only where the bus's user-list file is from -// such a composition, never its list) and the operator's, read from operatorKeysFile each time it is asked, so a key -// that arrives while the agent runs is trusted at the next composition. A file that cannot be read trusts the one given -// alone, and says so. -func trustedKeys(opts options, controllerNow func() ed25519.PublicKey) func() []ed25519.PublicKey { - return func() []ed25519.PublicKey { - keys := []ed25519.PublicKey{} - if controller := controllerNow(); len(controller) == ed25519.PublicKeySize { - keys = append(keys, controller) - } - raw, err := os.ReadFile(filepath.Join(filepath.Dir(opts.state), operatorKeysFile)) - if err != nil { - return keys - } - for _, line := range strings.Split(string(raw), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - key, err := base64.StdEncoding.DecodeString(line) - if err != nil || len(key) != ed25519.PublicKeySize { - fmt.Fprintf(os.Stderr, "mesh-host: a line of %s is not an ed25519 public key in base64, and is not trusted\n", operatorKeysFile) - continue - } - keys = append(keys, ed25519.PublicKey(key)) - } - return keys - } -} - // keptOnce reads the kept declaration once and splits it by module: its resources (the node's own are those that say // none, ADR 0539 §9) and its filled paths. Once the read is retired (keptRetired) nothing is read. func keptOnce(opts options, signer ed25519.PublicKey) func() (map[string][]json.RawMessage, map[string][]string) { @@ -511,10 +474,15 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus } } // The bus's user list, the grants part's alone (ADR 0547 §8, clause 11; issue 1126), written into the file the - // bus module's composition in the node's bucket declares: on every node, and a node whose compositions carry no - // list writes nothing. Those compositions were signed with the enrolment's signer and nobody writes them now; they - // are read for the file's place alone, never for a list, until the place is read elsewhere (milestone B17). - users := userListFollower(opts, session, composedBucket, trustedKeys(opts, func() ed25519.PublicKey { return signer }), say, func() { + // bus module this node runs declares, as its agent holds what runs (its own composition, or the declaration kept + // before the switch): never read from the node's composition bucket, which the controller wrote. On every node; + // a node that runs no list writes nothing. Looked at again whenever what runs changed (RunsChanged, Settled). + users := userListFollower(opts, session, func() []delivery.Composed { + if now := current.get(); now != nil { + return now.Everything() + } + return nil + }, say, func() { // This engine wrote the list, so the bus reloads its accounts and answers no JetStream request for a few // seconds: what failed in them is tried again once that is over, not at its next wait. time.AfterFunc(reloadSettles, func() { @@ -558,12 +526,14 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus Settled: func(module string) { knockFilling(module+" settled", say) judgeWhatRunsNow(opts.state, signer) + users.RunsChanged() }, // And once the haves are read at the start, and whenever a module is taken as running without an apply, so // a refusal from the engine's start does not wait for some module to settle (issue 924). RunsChanged: func(why string) { knockFilling(why, say) judgeWhatRunsNow(opts.state, signer) + users.RunsChanged() }, Say: say, // A failure of the bus not answering for a moment is tried again: 2 s, 5 s, 15 s, 60 s, then every 5 min. @@ -648,14 +618,9 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus shadowHooks := hooks shadowHooks.refused, shadowHooks.certify = nil, nil shadowEnv := composerEnv(session, node, say, shadowHooks) - // And, after each answer, what runs of an own secret the answer says the vault holds another value of: handed to - // the vault only while a node applied the controller's compositions (issue 1126, cause C4; ownresync), so nothing now. - composes := &composesNow{} - resync := ownResync(session, node, say, shadowEnv, composes) shadow := selfcompose.NewShadow(shadowEnv, func(ctx context.Context, answers map[string]delivery.CandidateAnswer) { session.NoteCandidates(ctx, delivery.WithValuesDiffer(answers)) - go resync.take(ctx, answers) }) go shadow.Keep(run, link.Backoff) // this run's alone: a switch begins a run, and its shadow, again // Its own composer, always (issue 1126, A7): a composition made elsewhere (the controller's, in the node's @@ -664,7 +629,6 @@ func nodeAgent(ctx context.Context, opts options, signer ed25519.PublicKey, trus composer := selfComposer(session, node, hooks, say) machinesNow.Store(composer) // the machines a failing network part points at, by the placements (issue 774) composerNow.Store(composer) // what the verb compose asks a pass of, this run's alone - composes.set(delivery.ComposesOwn) env.Composed, env.SelfComposed, env.Policy = composer, true, composer.Policy env.HoldsData = composer.HoldsData // what runs from the kept declaration says Data as its manifest does session.NoteComposes(linked, delivery.ComposesOwn, "it composes its own modules, and applies no composition made elsewhere") diff --git a/cmd/mesh-host/nodeagent_integration_test.go b/cmd/mesh-host/nodeagent_integration_test.go index 8a958f3..a00bef5 100644 --- a/cmd/mesh-host/nodeagent_integration_test.go +++ b/cmd/mesh-host/nodeagent_integration_test.go @@ -5,7 +5,6 @@ package main import ( "context" "crypto/ed25519" - "encoding/base64" "encoding/json" "os" "path/filepath" @@ -45,7 +44,6 @@ type itBed struct { opts options session *link.Session controll ed25519.PrivateKey - operator ed25519.PrivateKey said []string // refusals are the bus's asynchronous errors on the node's connection: each permissions violation it met. refusals []string @@ -137,20 +135,12 @@ func newItBedAs(t *testing.T, principal buscompose.Principal) *itBed { t.Fatal(err) } _, b.controll, _ = ed25519.GenerateKey(nil) - _, b.operator, _ = ed25519.GenerateKey(nil) return b } -// start runs the node agent under a session on the bus, trusting the controller's key, and the operator's when asked. -func (b *itBed) start(trustOperator bool) (stop func()) { +// start runs the node agent under a session on the bus. +func (b *itBed) start() (stop func()) { b.t.Helper() - if trustOperator { - pub := b.operator.Public().(ed25519.PublicKey) - if err := os.WriteFile(filepath.Join(filepath.Dir(b.opts.state), operatorKeysFile), - []byte("# the operator\n"+base64.StdEncoding.EncodeToString(pub)+"\n"), 0o600); err != nil { - b.t.Fatal(err) - } - } b.session = &link.Session{Membership: link.Membership{Node: itNode}, Timeout: 2 * time.Second, Say: b.say, Keys: func() link.RecordKeys { return link.RecordKeys{} }} b.session.Attach(context.Background(), link.OverNATS{Conn: b.nodeConn, JS: b.nodeJS, Node: itNode}) @@ -440,10 +430,6 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { builds := kvOf(delivery.BuilderBuildsBucket) candidates := kvOf(delivery.PlacementCandidatesBucket) packages := kvOf(link.PackagesBucket) - // The controller's stream the node's facts go into, so the node's key carries the facts it says. - if _, err := b.js.AddStream(&nats.StreamConfig{Name: "CONTROL", Subjects: []string{link.ReportSubject(itNode)}}); err != nil { - t.Fatal(err) - } path := filepath.Join(b.dir, "web.conf") man := `{"module":"web","version":"1","config":{"greeting":{"type":"string","description":"what the file says","default":"hello"}},` + `"resources":[{"id":"conf","type":"file","path":"` + path + `","mode":"0644","content":"greeting=${config:greeting}\n"},` + @@ -497,7 +483,7 @@ func h8(t *testing.T, b *itBed, checkBuild bool) (func(), h8state) { t.Fatal(err) } - stop := b.start(false) + stop := b.start() // Not placed yet: the agent applies the controller's compositions, of which there are none. The catalogue's seed then // places the node, and the agent begins again composing its own (ADR 0549 §4), with no restart of the engine. time.Sleep(300 * time.Millisecond) diff --git a/cmd/mesh-host/nodekey.go b/cmd/mesh-host/nodekey.go index 5881baf..5b1aa9e 100644 --- a/cmd/mesh-host/nodekey.go +++ b/cmd/mesh-host/nodekey.go @@ -26,12 +26,10 @@ const ( ) // heardKey is the vault's own public key as its answer to node-key gave it (ADR 0547 clause 9, issue 1945): the key the -// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is; kick is the follower of -// the user list on the link open now, told when the key is first heard or changes. +// bus node hands the loaded list's hashes sealed to. One for the process, as the key stated is. type heardKey struct { - mu sync.Mutex - pem string - kick func() + mu sync.Mutex + pem string } // vaultKey is the vault's key this engine heard. @@ -43,8 +41,7 @@ func (h *heardKey) get() string { return h.pem } -// heard keeps the key a node-key answer gave, and tells the follower when it is a first or changed one. An answer -// with no key (a vault before this one) changes nothing. +// heard keeps the key a node-key answer gave. An answer with no key (a vault before this one) changes nothing. func (h *heardKey) heard(answer json.RawMessage) { var a struct { VaultKey string `json:"vault_key"` @@ -53,24 +50,8 @@ func (h *heardKey) heard(answer json.RawMessage) { return } h.mu.Lock() - changed := a.VaultKey != h.pem h.pem = a.VaultKey - kick := h.kick h.mu.Unlock() - if changed && kick != nil { - kick() - } -} - -// following names the follower told when the key changes, and tells it at once when a key is known. -func (h *heardKey) following(kick func()) { - h.mu.Lock() - h.kick = kick - known := h.pem != "" - h.mu.Unlock() - if known && kick != nil { - kick() - } } // nodeKeyReask is how often the key is stated again though it did not change, so a changed vault key is heard: the diff --git a/cmd/mesh-host/nodekey_test.go b/cmd/mesh-host/nodekey_test.go index 0bae9f5..6084ced 100644 --- a/cmd/mesh-host/nodekey_test.go +++ b/cmd/mesh-host/nodekey_test.go @@ -162,29 +162,22 @@ func asksAgainUntilItHearsTheVaultsKey(t *testing.T) { } // The statement a host makes names this contract (issue 786), one above the vault's box. And the vault's key its -// answer gives is kept, told to the follower once when first heard and again only when changed; an answer without one -// (an older vault) or with no public key changes nothing (ADR 0547 clause 9). +// answer gives is kept; an answer without one (an older vault) or with no public key changes nothing (ADR 0547 clause +// 9). func TestTheNodeKeyContractIsThirteen(t *testing.T) { if link.NodeKeyContract != 13 { t.Errorf("NodeKeyContract %d", link.NodeKeyContract) } h := &heardKey{} - kicks := 0 - h.following(func() { kicks++ }) h.heard(json.RawMessage(`{"recorded":true}`)) h.heard(json.RawMessage(`{"recorded":true,"vault_key":"not a key"}`)) - if h.get() != "" || kicks != 0 { - t.Fatalf("an answer with no key was kept: %q, %d kicks", h.get(), kicks) + if h.get() != "" { + t.Fatalf("an answer with no key was kept: %q", h.get()) } pem := "-----BEGIN PUBLIC KEY-----\nAAAA\n-----END PUBLIC KEY-----\n" answer, _ := json.Marshal(map[string]any{"recorded": true, "vault_key": pem}) h.heard(answer) - h.heard(answer) - if h.get() != pem || kicks != 1 { - t.Fatalf("key %q, %d kicks, want the key and one", h.get(), kicks) - } - h.following(func() { kicks += 10 }) - if kicks != 11 { - t.Fatalf("a follower named after the key was heard was not told: %d", kicks) + if h.get() != pem { + t.Fatalf("key %q, want the key", h.get()) } } diff --git a/cmd/mesh-host/ownresync.go b/cmd/mesh-host/ownresync.go deleted file mode 100644 index 21d254e..0000000 --- a/cmd/mesh-host/ownresync.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-host/internal/identity" - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/ownresync" - "github.com/novox/mesh-host/internal/selfcompose" -) - -// composesNow is whose compositions this run of the agent applies, once decided ("" before). -type composesNow struct { - mu sync.Mutex - now string -} - -func (c *composesNow) set(s string) { c.mu.Lock(); c.now = s; c.mu.Unlock() } -func (c *composesNow) get() string { c.mu.Lock(); defer c.mu.Unlock(); return c.now } - -// resyncStep is the step that hands the vault what runs of an own secret a shadow answer says another value of (issue -// 1126, cause C4; internal/ownresync): one pass at a time, and the last answers taken again once the agent has decided -// whose compositions it applies. -type resyncStep struct { - r *ownresync.Resync - one sync.Mutex - mu sync.Mutex - last map[string]delivery.CandidateAnswer - // tries and pending are the step taken again after a bound when a read or an ask the bus did not answer stopped it - // (Outcome.Retry, issue 1126, 2026-10-28), one wait at a time. - tries int - pending bool -} - -func ownResync(session *link.Session, node string, say func(string), env selfcompose.Env, composes *composesNow) *resyncStep { - return &resyncStep{r: ownresync.New(ownresync.Env{Node: node, Composes: composes.get, Running: env.Running, - Get: env.Get, Unseal: env.Unseal, Seal: identity.Seal, Say: say, - Ask: func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error) { - return session.AskVault(ctx, verb, args) - }})} -} - -// take keeps answers as the last and takes them. -func (s *resyncStep) take(ctx context.Context, answers map[string]delivery.CandidateAnswer) { - s.mu.Lock() - s.last = answers - s.mu.Unlock() - s.again(ctx) -} - -// again takes the last answers, and once more after a bound (selfcompose.LaterWait) while the bus did not answer for one. -func (s *resyncStep) again(ctx context.Context) { - s.one.Lock() - defer s.one.Unlock() - s.mu.Lock() - last := s.last - s.mu.Unlock() - if last == nil { - return - } - retry := false - for _, o := range s.r.Take(ctx, last) { - retry = retry || o.Retry - } - s.mu.Lock() - defer s.mu.Unlock() - if !retry { - s.tries = 0 - return - } - if s.pending || ctx.Err() != nil { - return - } - wait := selfcompose.LaterWait(s.tries) - s.tries++ - s.pending = true - go func() { - select { - case <-ctx.Done(): - case <-time.After(wait): - } - s.mu.Lock() - s.pending = false - s.mu.Unlock() - if ctx.Err() == nil { - s.again(ctx) - } - }() -} diff --git a/cmd/mesh-host/rekey_test.go b/cmd/mesh-host/rekey_test.go deleted file mode 100644 index 7b19edd..0000000 --- a/cmd/mesh-host/rekey_test.go +++ /dev/null @@ -1,94 +0,0 @@ -package main - -import ( - "crypto/ed25519" - "strings" - "testing" - - "github.com/novox/mesh-host/internal/identity" - "github.com/novox/mesh-host/internal/link" - "github.com/novox/mesh-host/internal/tunnel" -) - -// novox/hq ADR 0105: `overlay take` moves this node's overlay key onto the found tunnel's and -// nothing else — identity, sealing and serving keys stay as they were — and tells the mesh with a -// proof signed by the identity key, over the previous key, the new one and the tunnel. - -func anEnrolledNode(t *testing.T) identity.Identity { - t.Helper() - mine, err := identity.Generate("anchor") - if err != nil { - t.Fatal(err) - } - mine.Overlay, err = identity.GenerateOverlayKey() - if err != nil { - t.Fatal(err) - } - mine.Membership = identity.Membership{Broker: "198.51.100.1:5671", Fingerprint: "sha256:aa", - Signer: make([]byte, ed25519.PublicKeySize), Password: "p"} - return mine -} - -func aFoundTunnel(t *testing.T) tunnel.Found { - t.Helper() - private, err := identity.GenerateOverlayKey() - if err != nil { - t.Fatal(err) - } - found, err := tunnel.Parse([]byte("[Interface]\nPrivateKey = " + private.Private + "\nListenPort = 51900\n" + - "Address = 192.0.2.1/24\n[Peer]\nPublicKey = PEER-A=\nAllowedIPs = 192.0.2.2/32\n")) - if err != nil { - t.Fatal(err) - } - found.Interface, found.Unit, found.Config = "wg0", "wg-quick@wg0", "/etc/wireguard/wg0.conf" - return found -} - -func TestTakingATunnelMovesOnlyTheOverlayKeyAndSignsForIt(t *testing.T) { - mine := anEnrolledNode(t) - found := aFoundTunnel(t) - before := mine.Overlay.Public - - taken, rekey, err := rekeyOnto(mine, found) - if err != nil { - t.Fatal(err) - } - if taken.Overlay.Public != found.PublicKey || taken.Overlay.Private != found.PrivateKey() { - t.Fatal("the overlay key is not the tunnel's") - } - if string(taken.Public) != string(mine.Public) || string(taken.Private) != string(mine.Private) || - taken.Node != mine.Node || taken.Membership.Password != mine.Membership.Password || - taken.Membership.Broker != mine.Membership.Broker { - t.Fatal("something other than the overlay key moved") - } - if taken.OverlayBefore != before { - t.Errorf("the key before the take was not kept: %q", taken.OverlayBefore) - } - if rekey.Previous != before || rekey.OverlayKey != found.PublicKey || rekey.Tunnel == nil || - rekey.Tunnel.PublicKey != found.PublicKey || len(rekey.Tunnel.Peers) != 1 { - t.Fatalf("the rekey does not say what moved: %+v", rekey) - } - if !ed25519.Verify(ed25519.PublicKey(mine.Public), - link.RekeyProof("anchor", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { - t.Fatal("the rekey is not signed by this node's identity key over what it says") - } - if ed25519.Verify(ed25519.PublicKey(mine.Public), - link.RekeyProof("laptop", before, found.PublicKey, rekey.Tunnel), rekey.Proof) { - t.Fatal("the proof is not bound to the node") - } - for _, said := range []string{rekey.Previous, rekey.OverlayKey, rekey.Tunnel.Interface} { - if strings.Contains(said, found.PrivateKey()) { - t.Fatal("the private key travels") - } - } - - // Run again after the take — the mesh not yet told, or told and refused — the previous key it - // names is still the one before the take, so the mesh can tell a repeat from a replay. - again, second, err := rekeyOnto(taken, found) - if err != nil { - t.Fatal(err) - } - if second.Previous != before || again.OverlayBefore != before || again.Overlay.Public != found.PublicKey { - t.Fatalf("a take run again does not name the key before the first: %+v", second) - } -} diff --git a/cmd/mesh-host/trust.go b/cmd/mesh-host/trust.go index 23a5ec4..29f2702 100644 --- a/cmd/mesh-host/trust.go +++ b/cmd/mesh-host/trust.go @@ -36,10 +36,6 @@ type meshTrust struct { // newMeshTrust trusts nothing until the vault's key is read. func newMeshTrust() *meshTrust { return &meshTrust{} } -// noControllerKey is the controller's key as this engine trusts it: none. A link told it refuses every ask signed as -// the controller's, where a link told nothing would verify it with the membership's signer (link.HandOverCheck). -func noControllerKey() ed25519.PublicKey { return nil } - // onChange names who is told when what it trusts changed. func (t *meshTrust) onChange(f func()) { t.mu.Lock() diff --git a/cmd/mesh-host/userlist.go b/cmd/mesh-host/userlist.go index bf79c2f..54cb6f1 100644 --- a/cmd/mesh-host/userlist.go +++ b/cmd/mesh-host/userlist.go @@ -2,7 +2,6 @@ package main import ( "context" - "crypto/ed25519" "encoding/json" "errors" "fmt" @@ -26,13 +25,12 @@ import ( // userListFollower is the follower of the bus's user list for one link: the node's composition bucket, the keys the // agent trusts, the write alone, and the have. -func userListFollower(opts options, session *link.Session, composedBucket string, trusted func() []ed25519.PublicKey, +func userListFollower(opts options, session *link.Session, running func() []delivery.Composed, say link.Announce, wrote func()) *userlist.Follower { return userlist.New(userlist.Env{ - Node: session.Membership.Node, - Composed: bucketWatch{session, composedBucket}, - Trusted: trusted, - OnDisk: userListOnDisk, + Node: session.Membership.Node, + Running: running, + OnDisk: userListOnDisk, Write: func(ctx context.Context, module string, raw json.RawMessage, body []byte) (string, bool, error) { digest, changed, err := writeUserListAlone(session.WithLink(ctx), opts, raw, body, say) if changed && wrote != nil { diff --git a/cmd/mesh-host/witnessing.go b/cmd/mesh-host/witnessing.go index d4aa7a0..2044f4e 100644 --- a/cmd/mesh-host/witnessing.go +++ b/cmd/mesh-host/witnessing.go @@ -146,9 +146,8 @@ func launcherReplaced(pid int, proc, path string) bool { // watchWhatThisHostPlaced judges the core builds this host placed, until ctx ends (to-be 45 §8). func watchWhatThisHostPlaced(ctx context.Context, node string, queue *link.Session, say func(string)) { - host, _ := os.Hostname() w := &witness.Watcher{ - Root: apply.DaemonRoot(), Node: node, Host: host, + Root: apply.DaemonRoot(), Node: node, Asker: func() link.Asker { return queue.Asker() }, Run: witness.Runner(apply.ExecRunner), Hold: func(f func()) { diff --git a/go.mod b/go.mod index 5ed422a..119dc8a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/novox/mesh-host go 1.26.0 require ( - git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 + git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 github.com/nats-io/nats-server/v2 v2.11.17 github.com/nats-io/nats.go v1.54.0 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index 5937b7a..ebd4e0f 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,5 @@ -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4 h1:xeuMBwOHNKrVUdGVg7Bdv5tkdueBvngwJFyk6n10rp0= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101151945-b2a27ed5cec4/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c h1:/YCxh+NYfqry3PxZYY1nQXaccupobxpsFt8c9BYuGBg= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261101235037-c21d8fd9692c/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21 h1:4I4ayo0fZYP4PBfEX1sGDDd6arnw5IQJySJ3jcFobPM= -git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102004146-dff0ce2abb21/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791 h1:7SYIDPMxbPBr8dMB6k9fYsgVt4ZyWVL41q4LMFsfAvs= +git.novox.be/novox/mesh-sdk/go v0.1.14-0.20261102112556-abf5eace7791/go.mod h1:ouERNpZizzwZ0HLEtAypsY9FR20FyUgUYRni6J7kVCY= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0= github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI= github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo= diff --git a/internal/apply/process_witness_test.go b/internal/apply/process_witness_test.go index fa51ff7..0be3ba1 100644 --- a/internal/apply/process_witness_test.go +++ b/internal/apply/process_witness_test.go @@ -72,36 +72,36 @@ func TestARolledBackBuildIsNotPlacedAgain(t *testing.T) { t.Cleanup(func() { unitDir, daemonRoot = wasUnits, wasBundles }) run := func(ctx context.Context, name string, args ...string) (string, error) { return "", nil } - oldBody, oldDigest := anArchive(t, map[string]string{"mesh-controller": "old\n"}) - newBody, newDigest := anArchive(t, map[string]string{"mesh-controller": "new\n"}) - controller := func(body []byte, digest, env string) string { - return `{"id":"mesh-controller.controller","type":"process","name":"mesh-controller","source":"` + serving(t, body) + - `","digest":"` + digest + `","run":["./mesh-controller","serve"],"env":{"X":"` + env + `"}}` + oldBody, oldDigest := anArchive(t, map[string]string{"node-tools": "old\n"}) + newBody, newDigest := anArchive(t, map[string]string{"node-tools": "new\n"}) + runtime := func(body []byte, digest, env string) string { + return `{"id":"node-tools.runtime","type":"process","name":"node-tools","source":"` + serving(t, body) + + `","digest":"` + digest + `","run":["./node-tools"],"env":{"X":"` + env + `"}}` } - _, state, err := Apply(context.Background(), archHost(t), declare(t, controller(oldBody, oldDigest, "1")), store.State{}, + _, state, err := Apply(context.Background(), archHost(t), declare(t, runtime(oldBody, oldDigest, "1")), store.State{}, store.OriginDeclared, run, nil, nil) if err != nil { t.Fatal(err) } - _, state, err = Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "1")), state, + _, state, err = Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "1")), state, store.OriginDeclared, run, nil, nil) if err != nil { t.Fatal(err) } - if _, err := witness.Restore(context.Background(), bundles, "mesh-controller", "never took the lease", witness.Runner(run), + if _, err := witness.Restore(context.Background(), bundles, "node-tools", "never answered", witness.Runner(run), time.Now()); err != nil { t.Fatal(err) } // The mesh still declares the new build, and its unit changes: the process is re-placed. - report, _, err := Apply(context.Background(), archHost(t), declare(t, controller(newBody, newDigest, "2")), state, + report, _, err := Apply(context.Background(), archHost(t), declare(t, runtime(newBody, newDigest, "2")), state, store.OriginDeclared, run, nil, nil) if err != nil { t.Fatal(err) } - if got, _ := os.ReadFile(filepath.Join(bundles, "mesh-controller", "mesh-controller")); string(got) != "old\n" { + if got, _ := os.ReadFile(filepath.Join(bundles, "node-tools", "node-tools")); string(got) != "old\n" { t.Fatalf("the rolled-back build was placed again: %q", got) } - if o := outcomeOf(report, "mesh-controller.controller"); !strings.Contains(o.Detail, "rolled back on this machine") { + if o := outcomeOf(report, "node-tools.runtime"); !strings.Contains(o.Detail, "rolled back on this machine") { t.Fatalf("the apply does not say why it kept the build before: %+v", o) } } diff --git a/internal/filling/filling.go b/internal/filling/filling.go index 24ec940..4954d78 100644 --- a/internal/filling/filling.go +++ b/internal/filling/filling.go @@ -35,10 +35,9 @@ import ( "time" "git.novox.be/novox/mesh-sdk/go/declaration" - "github.com/novox/mesh-host/internal/meshcli" ) -// Socket is where the node-engine accepts a write of a filled file, beside mesh-cli's socket. +// Socket is where the node-engine accepts a write of a filled file. const Socket = "/run/nox-mesh-host/fill.sock" // MaxRequest bounds one request, its content encoded in it: a configuration file, never an archive. @@ -293,11 +292,11 @@ func UIDOf(account string) (uint32, error) { return uint32(id), nil } -// peerOf and sameNamespaces are the kernel's word on the asking process, as mesh-cli's socket reads it. Variables so +// peerOf and sameNamespaces are the kernel's word on the asking process (peer_linux.go). Variables so // a test can say. var ( - peerOf = meshcli.PeerCredentials - sameNamespaces = meshcli.SameNamespaces + peerOf = peerCredentials + sameNamespaces = namespacesOf ) // within bounds how long an accepted connection may take to say its request, and refusedWithin how long a busy one @@ -352,7 +351,7 @@ func Serve(ctx context.Context, socket string, w Writer, say func(string)) error if err := os.MkdirAll(dir, 0o755); err != nil { return fmt.Errorf("the filled files' socket directory: %w", err) } - if problem := meshcli.NotOwnDirectory(dir); problem != "" { + if problem := notOwnDirectory(dir); problem != "" { return fmt.Errorf("filled files are not written on request on this machine: %s", problem) } if err := os.Chmod(dir, 0o755); err != nil { diff --git a/internal/filling/filling_test.go b/internal/filling/filling_test.go index 656d363..c50fc34 100644 --- a/internal/filling/filling_test.go +++ b/internal/filling/filling_test.go @@ -532,7 +532,7 @@ func TestTheSocketGoesOnAfterAnAcceptFails(t *testing.T) { } } -// The socket's directory is given the mode the socket needs, as mesh-cli's is. +// The socket's directory is given the mode the socket needs. func TestTheSocketsDirectoryIsOpenedToEveryAccount(t *testing.T) { dir, err := os.MkdirTemp("", "fill") if err != nil { diff --git a/internal/meshcli/peer_linux.go b/internal/filling/peer_linux.go similarity index 99% rename from internal/meshcli/peer_linux.go rename to internal/filling/peer_linux.go index a6d0ca5..12f456f 100644 --- a/internal/meshcli/peer_linux.go +++ b/internal/filling/peer_linux.go @@ -1,6 +1,6 @@ //go:build linux -package meshcli +package filling import ( "errors" diff --git a/internal/meshcli/peer_other.go b/internal/filling/peer_other.go similarity index 97% rename from internal/meshcli/peer_other.go rename to internal/filling/peer_other.go index f9df07f..7bf16fb 100644 --- a/internal/meshcli/peer_other.go +++ b/internal/filling/peer_other.go @@ -1,6 +1,6 @@ //go:build !linux -package meshcli +package filling import ( "errors" diff --git a/internal/link/asking.go b/internal/link/asking.go deleted file mode 100644 index 2a53261..0000000 --- a/internal/link/asking.go +++ /dev/null @@ -1,53 +0,0 @@ -package link - -import ( - "context" - "fmt" - "time" -) - -// The enrolment conversation, as the host's own words for it. -// -// **Its own seam rather than part of Link**, because almost nothing about it is the same. The -// credential is a one-time secret rather than this node's own; there is no declaration to hear; the -// whole exchange is a single question asked and possibly asked again. And the stakes differ: a node -// that fails here is not in the mesh at all, where a node that fails in Link has merely lost touch -// with one it belongs to. - -// Approach is how a node reaches a mesh it does not yet belong to. -// -// The three things a token carries about where to go, and nothing about who is asking: the address, -// the certificate that address must present, and which bus is at the other end — the mesh's own -// (hearing.go), and a token naming any other is refused before anything is sent. -type Approach struct { - Address string - Fingerprint string - Transport string -} - -// Asking is one open enrolment conversation. -type Asking interface { - // Ask puts the request to the mesh and waits for one answer, or says why none came. - // - // Called again, with the same bytes, while the mesh says "try again": the keys this node - // generated are the ones it keeps, so the same request is the same enrolment and the mesh holds - // the token for it (novox/hq issue 083). - Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) - - // Close lets go of the connection made with the token. - Close() -} - -// Present opens an enrolment conversation with the mesh. -// -// The connection is made before anything is sent, and the certificate is checked while it is being -// made — so a node pointed at the wrong bus finds out before its token has left the machine (ADR -// 0004). -func Present(ctx context.Context, to Approach, node, secret string, - timeout time.Duration) (Asking, error) { - - if to.Transport != OnNATS { - return nil, fmt.Errorf("this token is for the %q bus, and the mesh's bus is %s", to.Transport, OnNATS) - } - return presentNats(ctx, to, node, secret, timeout) -} diff --git a/internal/link/asking_nats.go b/internal/link/asking_nats.go deleted file mode 100644 index ed8bebd..0000000 --- a/internal/link/asking_nats.go +++ /dev/null @@ -1,186 +0,0 @@ -package link - -import ( - "context" - "crypto/rand" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "time" - - "github.com/nats-io/nats.go" -) - -// The enrolment conversation on the bus being built. -// -// **The reply address is the whole of what changes**, and it changes for a reason the transport -// forces rather than a preference. Core NATS request/reply puts the caller's inbox in the message's -// reply field and a plain responder answers it — but this request goes into a stream, and a message -// a JetStream consumer delivers has had that field claimed for the consumer's own ack address. So by -// the time the controller reads the request, the transport's reply field names where the -// *controller* must acknowledge. Verified against a running server (design 25 §2). -// -// The address therefore travels as a field of the request, and this subscribes it before publishing: -// a node that published first could miss an answer to a question nobody was listening for. - -// enrolInbox is where a node enrolling waits. -// -// Under `_INBOX.enrol..`, which is exactly what its enrolment user may subscribe and no -// wider — so an answer sealed to one machine cannot be read by another enrolling beside it. The -// random tail is this attempt's own: a reply left over from an attempt that timed out is not the -// answer to this question, which is what the correlation id does on the other transport. -func enrolInbox(node string) (string, error) { - tail := make([]byte, 8) - if _, err := rand.Read(tail); err != nil { - return "", fmt.Errorf("cannot make a reply address: %w", err) - } - return "_INBOX.enrol." + node + "." + hex.EncodeToString(tail), nil -} - -type natsAsking struct { - conn *nats.Conn - js nats.JetStreamContext - inbox string - answers *nats.Subscription - lost chan error -} - -func presentNats(_ context.Context, to Approach, node, secret string, - timeout time.Duration) (Asking, error) { - - config, err := PinnedConfig(to.Fingerprint) - if err != nil { - return nil, err - } - - inbox, err := enrolInbox(node) - if err != nil { - return nil, err - } - - lost := make(chan error, 1) - // The user is this token's own — `enrol.`, which may publish the enrolment subject and - // subscribe its own inbox and nothing else (design 25 §6). The secret is its password, the same - // string the request claims, so the server proves somebody holds the token and the request - // proves the same thing to the controller without it having to ask the server who connected. - // **Its own inbox space, because that is the only one it may listen in** (novox/hq - // 04-ISSUES/146). A JetStream publish waits for the stream's acknowledgement on an inbox the - // client picks, and the client's default is `_INBOX.` — which this user may not - // subscribe to, so the enrolment failed with a permissions violation on a subject nobody had - // chosen. The permission is `_INBOX.enrol..>` (design 25 §6), so the client is told to - // pick its inboxes there; the reply address below is in the same space for the same reason. - conn, err := nats.Connect(natsURL(to.Address), - nats.Secure(config), - nats.CustomInboxPrefix("_INBOX.enrol."+node), - nats.UserInfo("enrol."+node, secret), - nats.Name("mesh-host/enrol/"+node), - nats.Timeout(timeout), - nats.NoReconnect(), - nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { - select { - case lost <- fmt.Errorf("the bus closed the connection: %w", err): - default: - } - }), - ) - if err != nil { - if errors.Is(err, ErrWrongCertificate) { - return nil, err - } - // Not quoted back with the credential: the secret is one-time and still a secret. - return nil, fmt.Errorf("cannot reach the bus at %s as %s: %w", to.Address, node, err) - } - js, err := conn.JetStream() - if err != nil { - conn.Close() - return nil, fmt.Errorf("the bus at %s has no JetStream: %w", to.Address, err) - } - - // Subscribed before anything is published, so an answer cannot arrive before there is anywhere - // for it to land. - answers, err := conn.SubscribeSync(inbox) - if err != nil { - conn.Close() - return nil, fmt.Errorf("this node cannot listen for the mesh's answer: %w", err) - } - if err := conn.Flush(); err != nil { - conn.Close() - return nil, fmt.Errorf("this node's reply address did not reach the bus: %w", err) - } - - return &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: lost}, nil -} - -func (a *natsAsking) Close() { - if a.answers != nil { - _ = a.answers.Unsubscribe() - } - if a.conn != nil { - a.conn.Close() - } -} - -// Ask publishes the request with this attempt's reply address written into it, and waits there. -func (a *natsAsking) Ask(ctx context.Context, request []byte, wait time.Duration) ([]byte, error) { - addressed, err := withReplyTo(request, a.inbox) - if err != nil { - return nil, err - } - - publish, cancel := context.WithTimeout(ctx, wait) - defer cancel() - // Into the stream and awaited: an enrolment the bus never accepted must fail here rather than be - // assumed, because the node has nothing else to go on. - // - // **Once, however many times it is sent** (novox/hq 04-ISSUES/146). The client re-publishes when - // an acknowledgement is slow, and the mesh enrolled the machine on each copy — minting a second - // credential, which replaced the first, which is the one the node had already been given. The - // machine then reconnected for ever as a user whose password the mesh had rotated out from under - // it, and the controller's log said "enrolled anchor" twice in the same second. - // - // The id is the message: the same bytes carry the same id, so the stream discards the client's - // own retry, and a genuine second attempt — which carries a new reply address — is a different - // message and is let through. - sum := sha256.Sum256(addressed) - if _, err := a.js.Publish(EnrolSubject, addressed, - nats.MsgId(hex.EncodeToString(sum[:])), nats.Context(publish)); err != nil { - return nil, fmt.Errorf("cannot ask the mesh to enrol this node: %w", err) - } - - // Waited for rather than assumed. A published message that nothing answers means the controller - // is not running, and a node that carried on regardless would believe it had joined a mesh that - // has never heard of it. - // - // **The wait may legitimately be several store-window cycles long**: the controller naks the - // request with a delay while its store is restarting, and the node is waiting on the other side - // of that — which is exactly the combination that would have delivered the answer to a caller - // who had given up, had the address travelled in the transport's field. - answered, cancelAnswer := context.WithTimeout(ctx, wait) - defer cancelAnswer() - for { - msg, err := a.answers.NextMsgWithContext(answered) - switch { - case err == nil: - return msg.Data, nil - case errors.Is(err, context.DeadlineExceeded): - select { - case reason := <-a.lost: - return nil, reason - default: - } - return nil, fmt.Errorf( - "the bus accepted this node's connection and nothing answered within %s. The mesh's "+ - "bus is running and its controller is not", wait) - case errors.Is(err, context.Canceled): - return nil, ctx.Err() - default: - select { - case reason := <-a.lost: - return nil, reason - default: - } - return nil, fmt.Errorf("waiting for the mesh's answer: %w", err) - } - } -} diff --git a/internal/link/asking_nats_test.go b/internal/link/asking_nats_test.go deleted file mode 100644 index 919b9d9..0000000 --- a/internal/link/asking_nats_test.go +++ /dev/null @@ -1,192 +0,0 @@ -package link - -import ( - "context" - "encoding/json" - "errors" - "strings" - "testing" - "time" - - "github.com/nats-io/nats.go" -) - -// The enrolment round trip against a real server. -// -// **This is the test that keeps a reason from becoming folklore.** The reply address travels in the -// request's payload because a JetStream consumer's delivery has had the transport's reply field -// claimed for its own ack address — which is a fact about a server, not a rule anybody can check by -// reading. Both halves are asserted here: that the field really is eaten, and that the answer -// reaches the node anyway. -// -// docker run -d --rm --name t -p 14223:4222 nats:2.10-alpine -js -// MESH_TEST_NATS=nats://127.0.0.1:14223 go test ./internal/link/ -run TestNatsAnEnrolment - -// asking is a conversation on a bus with no TLS. Built directly rather than through Present because -// the pin is what Present adds and PinnedConfig's own tests cover it; what is under test here is the -// address the answer comes back on. -func asking(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, node string) *natsAsking { - t.Helper() - inbox, err := enrolInbox(node) - if err != nil { - t.Fatal(err) - } - answers, err := conn.SubscribeSync(inbox) - if err != nil { - t.Fatal(err) - } - if err := conn.Flush(); err != nil { - t.Fatal(err) - } - a := &natsAsking{conn: conn, js: js, inbox: inbox, answers: answers, lost: make(chan error, 1)} - // **Not a.Close.** Close closes the connection, and this connection is the test's own, shared with - // the stand-in. Cleanups run last-in first-out and the stand-in was made first, so its cleanup - // would run on a closed connection: Unsubscribe returns ErrConnectionClosed without deleting the - // consumer it made, and every request after that fails. The consumer then outlives the test on a - // workqueue stream, which allows one consumer per filter, and the next test cannot subscribe - // (issue 604). The connection is closed by aBus, last of all. - t.Cleanup(func() { _ = answers.Unsubscribe() }) - return a -} - -// theMeshAnswers stands in for the controller: it consumes the enrolment off the stream, reads the -// reply address out of the payload — never from the transport field — and answers there. It reports -// what the transport field actually held, which is the claim design 25 §2 rests on. -func theMeshAnswers(t *testing.T, conn *nats.Conn, js nats.JetStreamContext, - reply EnrolReply) <-chan string { - t.Helper() - sawReplyField := make(chan string, 1) - // **A consumer of this test's own, gone before it starts and gone when it ends.** The stand-in - // used to be one durable name shared by every test, released by Unsubscribe alone. Unsubscribe - // only asks the server to let go: it deletes the consumer only when this subscription made it, - // and the server drops the binding on its own time. A test that started in that window (or one - // that found the consumer left by an earlier run on a long-lived bus) was refused with "consumer - // is already bound to a subscription". Per-test names remove the sharing; the delete-and-wait - // before and after removes the dependence on the server's timing. - durable := standinName(t) - stream, err := js.StreamNameBySubject(EnrolSubject) - if err != nil { - t.Fatal(err) - } - standinGone(t, js, stream, durable) - sub, err := js.Subscribe(EnrolSubject, func(msg *nats.Msg) { - select { - case sawReplyField <- msg.Reply: - default: - } - var addressed struct { - ReplyTo string `json:"reply_to"` - } - if err := json.Unmarshal(msg.Data, &addressed); err != nil || addressed.ReplyTo == "" { - _ = msg.Ack() - return - } - body, _ := json.Marshal(reply) - // Published explicitly to the address the payload named, never msg.Respond — which would - // send it to whatever the transport's reply field holds, and that is the point. - _ = conn.Publish(addressed.ReplyTo, body) - _ = msg.Ack() - }, nats.Durable(durable), nats.ManualAck()) - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { - _ = sub.Unsubscribe() - standinGone(t, js, stream, durable) - }) - return sawReplyField -} - -// standinName is the stand-in's durable name for one test: the test's name with every character a -// consumer name may not hold replaced. -func standinName(t *testing.T) string { - t.Helper() - return "controller-standin-" + strings.Map(func(r rune) rune { - if r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-' || r == '_' { - return r - } - return '_' - }, t.Name()) -} - -// standinGone deletes the durable consumer if it exists and waits until the server no longer reports -// it, so that nothing after this call can meet its binding. -func standinGone(t *testing.T, js nats.JetStreamContext, stream, durable string) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for { - _, err := js.ConsumerInfo(stream, durable) - if errors.Is(err, nats.ErrConsumerNotFound) { - return - } - if errors.Is(err, nats.ErrConnectionClosed) { - t.Fatalf("the connection closed before consumer %s was deleted: %v", durable, err) - } - _ = js.DeleteConsumer(stream, durable) - if time.Now().After(deadline) { - t.Fatalf("consumer %s on %s still exists 5s after it was deleted", durable, stream) - } - time.Sleep(20 * time.Millisecond) - } -} - -// An enrolment is answered on the address the request carried, and the transport's own reply field -// held something else entirely. -func TestNatsAnEnrolmentIsAnsweredOnTheAddressInItsPayload(t *testing.T) { - conn, js := aBus(t) - const node = "joining" - - sawReplyField := theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node, Password: "p"}) - a := asking(t, conn, js, node) - - request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"}) - answer, err := a.Ask(context.Background(), request, 8*time.Second) - if err != nil { - t.Fatalf("no answer reached the node: %v", err) - } - var reply EnrolReply - if err := json.Unmarshal(answer, &reply); err != nil { - t.Fatal(err) - } - if !reply.Accepted || reply.Node != node { - t.Fatalf("the answer was not the mesh's: %+v", reply) - } - - // And the field the answer would have gone to, had it used the transport's: the consumer's own - // ack address. If a future server stopped doing this, the payload-borne address would still - // work and this line is what would say the reason had changed. - select { - case field := <-sawReplyField: - if field == a.inbox { - t.Fatalf("the transport's reply field held this node's inbox (%s), so the payload "+ - "address is no longer load-bearing — check design 25 §2 before relying on it", field) - } - if field == "" { - t.Fatal("the transport's reply field was empty rather than claimed, which is a third " + - "behaviour from the two design 25 §2 describes") - } - case <-time.After(2 * time.Second): - t.Fatal("the stand-in never saw the request") - } -} - -// A request that names no reply address is not answered, and the node says so as a mesh that is not -// running rather than hanging. The controller has nowhere to send an answer, which is the failure -// the payload field exists to make impossible — asserted so that a request built without it fails -// loudly here rather than quietly on a machine. -func TestNatsAnEnrolmentWithNoReplyAddressIsNotAnswered(t *testing.T) { - conn, js := aBus(t) - const node = "silent" - - theMeshAnswers(t, conn, js, EnrolReply{Accepted: true, Node: node}) - a := asking(t, conn, js, node) - - // Published without going through Ask, so the reply address is genuinely absent. - request, _ := json.Marshal(EnrolRequest{Node: node, Secret: "t"}) - if _, err := js.Publish(EnrolSubject, request); err != nil { - t.Fatal(err) - } - if _, err := a.Ask(context.Background(), []byte(`{"node":"`+node+`"}`), 0); err == nil { - t.Fatal("a node with no answer coming was told it had one") - } -} diff --git a/internal/link/bus.go b/internal/link/bus.go index dda5a4d..0670097 100644 --- a/internal/link/bus.go +++ b/internal/link/bus.go @@ -10,28 +10,14 @@ import ( "github.com/nats-io/nats.go" ) -// Bus is what a host needs of the mesh's bus, in the mesh's own words. -// -// A host says exactly two things unprompted: what it applied, and that it is here. They are not -// the same kind of statement and the difference is the whole of this interface — one must arrive -// and one must not be insisted on. -// -// **The host still imports nothing of the mesh's own** (novox/hq ADR 0005): this is its own -// interface over its own client libraries, not a contract shared with the controller. The two -// agree because a conformance fixture holds them to one envelope, which is the only kind of -// agreement that survives being in different repositories. -type Bus interface { - // Report says what this node applied. **It must arrive.** A report that fails leaves the - // mesh believing the node never answered while the node believes it did, and the two go on - // disagreeing with nothing anywhere saying so — the shape of fault this project keeps - // finding. Returns false when it could not be delivered, so the caller can say so. - Report(ctx context.Context, node string, body []byte) error -} +// Bus is the link open now as the session holds it. What it can do is asked of it by what it implements (HaveBus, +// HealthBus, ToolBus, FilesBus): the report it once published into the controller's CONTROL stream went with the +// controller (issue 1126), and what the node applied is said in its haves. +type Bus interface{} // --- The bus the mesh runs on today ----------------------------------------------------------- -// OverNATS is the bus as a connection. A report goes through JetStream because it must survive -// the controller's store restarting; a heartbeat does not, because it must not. +// OverNATS is the bus as a connection. type OverNATS struct { Conn *nats.Conn JS nats.JetStreamContext @@ -39,11 +25,6 @@ type OverNATS struct { Node string } -// ReportSubject is this node's own, and no other node's: a host's account may -// publish `mesh.control..>` and nothing wider, so the subject is the authority on -// which node a report is about. -func ReportSubject(node string) string { return "mesh.control." + node + ".report" } - // HealthSubject is where this node says its long-running resources' health between reports (novox/hq // ADR 0240): its own, inside the grant every host already has, and on core NATS like the heartbeat — a // statement lost is said again within a minute while anything is not healthy. @@ -65,11 +46,6 @@ func (b OverNATS) Health(ctx context.Context, node string, body []byte) error { return b.Conn.FlushWithContext(flush) } -// CLISubject is where this node's engine asks the controller a line `mesh-cli` was given on this machine -// (novox/hq ADR 0272 §3): its own node's, inside the grant every engine already has (`mesh.control..>`), so the -// bus server is the authority on which node asked. Core NATS, never the CONTROL stream, which names its own subjects. -func CLISubject(node string) string { return "mesh.control." + node + ".cli" } - // ToolSubject is a module's tool on one machine's node tools — the subject a node-engine asks a declared // tool check on (novox/hq ADR 0240, to-be 48 §3), granted it for its own machine's instance and no other. func ToolSubject(module, tool, node string) string { @@ -90,17 +66,6 @@ func (b OverNATS) Ask(ctx context.Context, subject string, body []byte) ([]byte, return msg.Data, nil } -func (b OverNATS) Report(ctx context.Context, node string, body []byte) error { - // Into the CONTROL stream and awaited: this is the message the store-window guarantee is - // about (novox/hq ADR 0083). The controller naks with a delay while its store is away and - // the message is redelivered; a publish the bus never accepted must fail here rather than - // be assumed. - if _, err := b.JS.Publish(ReportSubject(node), body, nats.Context(ctx)); err != nil { - return fmt.Errorf("reporting: %w", err) - } - return nil -} - // HaveBind binds the have bucket the controller made (novox/hq ADR 0429 §4, issue 1132), once for a pass of writes. // **Only binds**: a node-engine's account reaches no part of the JetStream API beyond what it is granted, and the bucket // is the controller's to create and to configure, so a bucket that is not there is an error said as what it is. Every diff --git a/internal/link/enginepackages.go b/internal/link/enginepackages.go index 50e7adf..3e7d16c 100644 --- a/internal/link/enginepackages.go +++ b/internal/link/enginepackages.go @@ -17,10 +17,6 @@ import ( // mesh-host.packages (delivery.EnginePackagesRead). const PackagesBucket = "mesh-host_" + delivery.EnginePackagesState -// MirrorPackagesBucket is the controller's declared-packages mirror (ADR 0507 §5), which the engine reads its own node's -// key of until the lease write, to compare (the node seat's verb packages). -const MirrorPackagesBucket = "mesh-controller_declared-packages" - // packagesStore binds PackagesBucket over the link open now: bound, never created. func (l *natsLink) packagesStore() (nats.KeyValue, error) { kv, err := l.js.KeyValue(PackagesBucket) @@ -83,51 +79,33 @@ func (p EnginePackages) Put(_ context.Context, key string, value []byte) error { return err } -// PackagesView is the node seat's verb packages' answer (ADR 0565 §6): this node's key of the engine's state packages, -// and until the lease write the controller's mirror's key for the node and every package they hold differently. +// PackagesView is the node seat's verb packages' answer (ADR 0565 §6): this node's key of the engine's state packages. +// The controller's mirror it was once compared with ended with the controller (issue 1126). type PackagesView struct { Node string `json:"node"` // Key is the node's key as the bus holds it (Source); null when the engine has written none. Source string `json:"source"` Key *delivery.DeclaredPackages `json:"key"` KeyError string `json:"key_error,omitempty"` - // Mirror is the controller's mirror's key for the node (MirrorSource); null when it holds none or cannot be read. - MirrorSource string `json:"mirror_source"` - Mirror *delivery.DeclaredPackages `json:"mirror"` - MirrorError string `json:"mirror_error,omitempty"` - // Differences is every package the two hold differently, read only when both are; Agree when there is none. - Differences []delivery.PackageDifference `json:"differences"` - Agree bool `json:"agree"` // Wrote says, for write, whether the key was written again (false: it already held what the engine composed). Wrote *bool `json:"wrote,omitempty"` Note string `json:"note,omitempty"` } -// PackagesAnswer is the verb's answer from the two keys as read (nil: no key), each with its read's error. -func PackagesAnswer(node string, key, mirror []byte, keyErr, mirrorErr error) PackagesView { - v := PackagesView{Node: node, Source: delivery.EnginePackagesRead, MirrorSource: "mesh-controller.declared-packages", - Differences: []delivery.PackageDifference{}} - read := func(raw []byte, err error, into **delivery.DeclaredPackages, said *string) { - if err == nil && raw != nil { - var d delivery.DeclaredPackages - if err = json.Unmarshal(raw, &d); err == nil { - *into = &d - } - } - if err != nil { - *said = err.Error() +// PackagesAnswer is the verb's answer from the key as read (nil: no key), with its read's error. +func PackagesAnswer(node string, key []byte, keyErr error) PackagesView { + v := PackagesView{Node: node, Source: delivery.EnginePackagesRead} + if keyErr == nil && key != nil { + var d delivery.DeclaredPackages + if keyErr = json.Unmarshal(key, &d); keyErr == nil { + v.Key = &d } } - read(key, keyErr, &v.Key, &v.KeyError) - read(mirror, mirrorErr, &v.Mirror, &v.MirrorError) switch { - case v.Key != nil && v.Mirror != nil: - v.Differences = delivery.DiffDeclaredPackages(*v.Key, *v.Mirror) - v.Agree = len(v.Differences) == 0 - case v.Key == nil && v.KeyError == "": - v.Note = "the engine has written no key for this node yet: pacman reads the controller's mirror for it" - case v.Mirror == nil && v.MirrorError == "": - v.Note = "the controller's mirror holds no key for this node: nothing to compare" + case keyErr != nil: + v.KeyError = keyErr.Error() + case v.Key == nil: + v.Note = "the engine has written no key for this node yet" } return v } diff --git a/internal/link/enrol.go b/internal/link/enrol.go index f0a9d5c..da23fe2 100644 --- a/internal/link/enrol.go +++ b/internal/link/enrol.go @@ -1,70 +1,21 @@ package link import ( - "context" - "encoding/base64" - "encoding/json" "errors" - "fmt" "time" ) -// The wire format shared with the control plane, which defines it separately because this binary -// requires nothing present and does not import it. A test on each side asserts the field names. -const ( - Exchange = "mesh" - KeyEnrol = "enrol" -) +// What a joining machine keeps of the enrolment's words: the tunnel it found, as it travels, and how long it asks +// while the seat mesh-enrolment says to ask again (Redeem). The enrolment through the controller's subject on the +// CONTROL stream ended with the controller (issue 1126). -// QueueFor is the queue this node consumes from — the only one its account may read. -func QueueFor(node string) string { return "node." + node } - -// EnrolRequest is what this node says when joining. -type EnrolRequest struct { - Node string `json:"node"` - Secret string `json:"secret"` - PublicKey []byte `json:"public_key"` - - // OverlayKey is the public half of this node's key on the private network — a different key - // from PublicKey above, generated at the same moment and for a different purpose. - // - // Sent with enrolment because the overlay is the first declaration a node receives, and the - // mesh cannot compose it without this. Asking for it afterwards would mean a node is enrolled - // and unreachable for a round trip, which is the state everything else here works to avoid. - OverlayKey string `json:"overlay_key,omitempty"` - - // SealingKey is the public half of the key secrets are sealed to. A third key, and the - // reasoning is the same one twice over: the mesh must be able to send this node something - // nothing else can read, and it must never be able to read it either. - SealingKey string `json:"sealing_key,omitempty"` - - // ServingKey is the public half of the key this node serves TLS with on its internal name. - // The mesh signs a certificate binding it; the private half never leaves the machine, so - // there is nothing to seal and nothing that could be stolen from the mesh's copy. - ServingKey string `json:"serving_key,omitempty"` - - Profile map[string]any `json:"profile,omitempty"` - - // Proof is this node's identity key signing EnrolProof over this request: that the presenter - // holds the private half of PublicKey. The mesh asks for it before letting an enrolment finish - // on a token this key already spent (novox/hq issue 083). - Proof []byte `json:"proof,omitempty"` - - // ReplyTo is where the mesh's answer goes, as a field of the request rather than the - // transport's own reply address (design 25 §2). Written by the transport that needs it — - // withReplyTo, once per attempt — because a request going into a stream has had the transport's - // reply field claimed for the consumer's ack address before the controller ever reads it. - // - // Empty on the bus the mesh runs on today, where the delivery carries the reply queue and the - // field means what it has always meant. Named here so both sides of the wire hold the same - // field name, which is what the shape test on each side is for. - ReplyTo string `json:"reply_to,omitempty"` - - // Tunnel is the tunnel this node found and whose key it took as its overlay key (novox/hq ADR - // 0105): everything about it but that key. Sent with the keys because it is one of them — - // OverlayKey above IS this tunnel's public key when this is set — and the mesh composes the - // hub's address, the range and the carried peers from it before the first declaration. - Tunnel *Tunnel `json:"tunnel,omitempty"` +// Approach is how a node reaches a mesh it does not yet belong to: the address, the certificate that address must +// present, and which bus is at the other end, the mesh's own (hearing.go); a token naming any other is refused before +// anything is sent. +type Approach struct { + Address string + Fingerprint string + Transport string } // Tunnel is a found tunnel as it travels: no private key. @@ -86,36 +37,6 @@ type TunnelPeer struct { Address string `json:"address"` } -// EnrolReply is what the mesh says back. -type EnrolReply struct { - Accepted bool `json:"accepted"` - - // TryAgain is the mesh saying it cannot answer right now — its store is restarting, or the - // token is held for a moment by another enrolment — and that nothing was spent. The same - // request is asked again (novox/hq issue 083). - TryAgain bool `json:"try_again,omitempty"` - - Node string `json:"node,omitempty"` - Queue string `json:"queue,omitempty"` - - // What this node keeps so it can come back on its own. Without these a restart would need a - // person with a new token, which would make disconnection a crisis rather than the ordinary - // situation novox/hq ADR 0004 says it is. - Password string `json:"password,omitempty"` - Broker string `json:"broker,omitempty"` - Fingerprint string `json:"fingerprint,omitempty"` - Signer []byte `json:"signer,omitempty"` - - Refusal string `json:"refusal,omitempty"` -} - -// EnrolProof is what a node signs with its identity key when it enrols: the token and every key it -// presents, so a proof cannot be moved to another request. The mesh builds the same bytes. -func EnrolProof(secret string, public []byte, overlay, sealing, serving string) []byte { - return []byte("novox-mesh-enrol\x00" + secret + "\x00" + base64.StdEncoding.EncodeToString(public) + - "\x00" + overlay + "\x00" + sealing + "\x00" + serving) -} - // ErrRefused is what a node gets when the mesh will not have it. var ErrRefused = errors.New("the mesh refused this enrolment") @@ -126,94 +47,3 @@ const EnrolPatience = 2 * time.Minute // AskAgainAfter is the pause between asks while the mesh says "try again". const AskAgainAfter = 3 * time.Second - -// ErrNotNow is a mesh that said "try again" for longer than this node would keep asking. -var ErrNotNow = errors.New("the mesh could not answer this enrolment") - -// answered decides what one reply means: done, ask again, or stop with an error. Separate from the -// broker so it can be held to that by a test. -func answered(reply EnrolReply, asking time.Duration) (again bool, err error) { - switch { - case reply.Accepted: - return false, nil - case reply.TryAgain && asking < EnrolPatience: - return true, nil - case reply.TryAgain: - // Said with what to do. The mesh holds the token for this attempt's keys for as long as - // this node kept asking, so a new attempt — with keys of its own — waits that out first. - return false, fmt.Errorf("%w for %s: %s. The token was not spent: wait about %s and run "+ - "enrol again with it; if it is then refused, issue a new one", - ErrNotNow, EnrolPatience, reply.Refusal, EnrolPatience) - default: - return false, fmt.Errorf("%w: %s", ErrRefused, reply.Refusal) - } -} - -// Enrol presents this node's key and its one-time secret, and waits to be told it is known. -// -// The broker has already authenticated this connection: the account was created when the token -// was issued and the secret is its password. So this is not how the node gets in — it is what it -// says once it is in, and the secret travels again because the control plane must not have to ask -// the broker who connected. -func Enrol(ctx context.Context, to Approach, node, secret string, public []byte, - overlayKey, sealingKey, servingKey string, profile map[string]any, proof []byte, - tunnel *Tunnel, timeout time.Duration) (EnrolReply, error) { - - asking, err := Present(ctx, to, node, secret, timeout) - if err != nil { - return EnrolReply{}, err - } - defer asking.Close() - - request := EnrolRequest{Node: node, Secret: secret, PublicKey: public, - OverlayKey: overlayKey, SealingKey: sealingKey, ServingKey: servingKey, Profile: profile, - Proof: proof, Tunnel: tunnel} - body, err := json.Marshal(request) - if err != nil { - return EnrolReply{}, err - } - - // Asked, and asked again with the same request while the mesh says "try again": the keys this - // node generated are the ones it keeps, so the same request is the same enrolment, and the mesh - // holds the token for it (novox/hq issue 083). - began := time.Now() - for { - answer, err := asking.Ask(ctx, body, timeout) - if err != nil { - return EnrolReply{}, err - } - var reply EnrolReply - if err := json.Unmarshal(answer, &reply); err != nil { - return EnrolReply{}, fmt.Errorf("the mesh's answer could not be read: %w", err) - } - again, err := answered(reply, time.Since(began)) - if err != nil { - return reply, err - } - if !again { - return reply, nil - } - select { - case <-ctx.Done(): - return EnrolReply{}, ctx.Err() - case <-time.After(AskAgainAfter): - } - } -} - -// withReplyTo writes this attempt's reply address into the request, as a field of its own. -// -// **Written into the bytes rather than carried beside them**, because the whole point is that the -// address survives a stream: a JetStream consumer's delivery has had the transport's reply field -// claimed for its own ack address, so a reply address that is not in the payload is one the -// controller cannot read (design 25 §2). Done by decoding and re-encoding rather than by setting the -// field before marshalling, so one request can be asked again with a fresh address each time without -// the caller knowing that is what happens. -func withReplyTo(request []byte, inbox string) ([]byte, error) { - var fields map[string]any - if err := json.Unmarshal(request, &fields); err != nil { - return nil, fmt.Errorf("this node's own enrolment request cannot be read back: %w", err) - } - fields["reply_to"] = inbox - return json.Marshal(fields) -} diff --git a/internal/link/enrol_again_test.go b/internal/link/enrol_again_test.go deleted file mode 100644 index 7ce0b6f..0000000 --- a/internal/link/enrol_again_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package link - -import ( - "errors" - "strings" - "testing" - "time" -) - -// What one reply means to a node enrolling (novox/hq issue 083): accepted is done; "try again" is -// asked again, until the node's patience runs out and it says the token was not spent; anything -// else is a refusal. -func TestAnEnrollingNodeAsksAgainWhileTheMeshSaysNotNow(t *testing.T) { - if again, err := answered(EnrolReply{Accepted: true}, 0); again || err != nil { - t.Fatalf("an accepted enrolment was not done: again=%v err=%v", again, err) - } - if again, err := answered(EnrolReply{TryAgain: true}, time.Second); !again || err != nil { - t.Fatalf("a mesh saying not now was not asked again: again=%v err=%v", again, err) - } - again, err := answered(EnrolReply{TryAgain: true, Refusal: "restarting"}, EnrolPatience) - if again || !errors.Is(err, ErrNotNow) { - t.Fatalf("a mesh saying not now past the node's patience did not stop it: again=%v err=%v", again, err) - } - if !errorsMention(err, "not spent") { - t.Errorf("giving up did not say the token can be used again: %v", err) - } - if again, err := answered(EnrolReply{Refusal: "that token cannot be used"}, 0); again || !errors.Is(err, ErrRefused) { - t.Fatalf("a refusal was not a refusal: again=%v err=%v", again, err) - } -} - -func errorsMention(err error, what string) bool { - return err != nil && strings.Contains(err.Error(), what) -} diff --git a/internal/link/enrol_proof_test.go b/internal/link/enrol_proof_test.go deleted file mode 100644 index 6d6d47b..0000000 --- a/internal/link/enrol_proof_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package link - -import "testing" - -// The bytes a node signs when it enrols. The mesh builds the same bytes to check the signature, in -// another repository; this known answer is repeated in its test, so the two cannot drift apart -// without one of them failing (novox/hq issue 083). -func TestWhatAnEnrollingNodeSignsIsFixed(t *testing.T) { - got := string(EnrolProof("s", []byte{1, 2, 3}, "o", "e", "v")) - if want := "novox-mesh-enrol\x00s\x00AQID\x00o\x00e\x00v"; got != want { - t.Fatalf("the enrolment proof's message changed: %q, want %q", got, want) - } -} diff --git a/internal/link/enrol_seat.go b/internal/link/enrol_seat.go index c5d7eb7..d6eb34f 100644 --- a/internal/link/enrol_seat.go +++ b/internal/link/enrol_seat.go @@ -18,8 +18,8 @@ import ( // Joining a mesh through the seat mesh-enrolment (novox/hq ADR 0587 §6, issue 774): the node-engine dials the bus as enrol. // with the certificate's fingerprint pinned, calls `redeem` with the public halves of the keys it made where it runs and a // proof signed with its identity key, refuses an answer not signed by the key its token carries, keeps its membership, dials -// again as the login node-logins names for it, and calls `confirm`. The controller's enrolment subject (Enrol) stays for as long -// as the controller holds its lease. +// again as the login node-logins names for it, and calls `confirm`. It is the one way in: the controller's enrolment subject +// ended with the controller (issue 1126). // EnrolmentSeat is the seat that adopts a new machine, and its two verbs a node-engine calls. const ( @@ -33,8 +33,8 @@ func CalledSeatSubject(seat, verb, user string) string { return "mesh.seat." + seat + ".call." + verb + "." + strings.ReplaceAll(user, ".", "~") } -// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call (the -// controller still holds its lease) or no holder answers it. +// ErrNoSeat is the bus saying nothing serves the seat for this caller: the enrolment user may not publish the seat's call, or +// no holder answers it. var ErrNoSeat = errors.New("the seat mesh-enrolment is not served to this machine's token") // VerifyRedeemAnswer holds an answer to what it must be before anything is kept (ADR 0587 §6): it names the node asked for, @@ -80,8 +80,8 @@ func seatAnswer(reply []byte) (json.RawMessage, error) { } // Redeem calls the seat's redeem as enrol., with the token's secret as the bus password, on the pinned certificate. It -// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), for -// as long as a machine asking the controller would (EnrolPatience). An answer is returned as the holder gave it: the caller +// asks again with the same keys while the holder says to (a stopped redeem finishes; the bus node loads the account), , for +// EnrolPatience. An answer is returned as the holder gave it: the caller // verifies it (VerifyRedeemAnswer) before it keeps anything. func Redeem(ctx context.Context, to Approach, node, secret string, req delivery.Redeem, timeout time.Duration) (delivery.RedeemAnswer, error) { if to.Transport != OnNATS { diff --git a/internal/link/enrol_shape_test.go b/internal/link/enrol_shape_test.go deleted file mode 100644 index 7b53f95..0000000 --- a/internal/link/enrol_shape_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package link - -import ( - "encoding/json" - "os" - "testing" - - "github.com/novox/mesh-host/internal/identity" -) - -// What this node says when it joins, written out so the mesh's own suite can accept it. -// -// The two ends are separate structs in separate repositories. Every field here is one somebody -// could rename on one side, and the failure would be silent: enrolment succeeds, a key is simply -// absent, and the node looks joined until the first thing sealed to it cannot be opened. That is -// exactly the shape of fault this project keeps finding late. -// -// Skipped unless MESH_ENROL_OUT names a file, so this is a check somebody runs deliberately -// rather than a dependency between two repositories. -func TestWhatThisNodeSaysWhenItJoins(t *testing.T) { - path := os.Getenv("MESH_ENROL_OUT") - if path == "" { - t.Skip("set MESH_ENROL_OUT to write the enrolment request the mesh's suite reads") - } - - // A real one. Generated the way enrolment generates them rather than typed as literals, so a - // key that stopped being a key would be caught here rather than travelling. - mine, err := identity.Generate("workstation") - if err != nil { - t.Fatal(err) - } - overlay, err := identity.GenerateOverlayKey() - if err != nil { - t.Fatal(err) - } - sealing, err := identity.GenerateSealingKey() - if err != nil { - t.Fatal(err) - } - serving, err := identity.GenerateServingKey() - if err != nil { - t.Fatal(err) - } - - request := EnrolRequest{ - Node: "workstation", - Secret: "a-one-time-secret", - PublicKey: mine.Public, - OverlayKey: overlay.Public, - SealingKey: sealing.Public, - ServingKey: serving.Public, - Profile: map[string]any{"seat": true}, - } - body, err := json.MarshalIndent(request, "", " ") - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, append(body, '\n'), 0o644); err != nil { - t.Fatal(err) - } - // The private half of the sealing key goes beside it, so the mesh's suite can prove what it - // sealed is openable rather than merely present. - if err := os.WriteFile(path+".sealing-private", []byte(sealing.Private), 0o600); err != nil { - t.Fatal(err) - } - t.Logf("wrote %s", path) -} diff --git a/internal/link/handover.go b/internal/link/handover.go index e6103f0..d45ad23 100644 --- a/internal/link/handover.go +++ b/internal/link/handover.go @@ -76,13 +76,9 @@ const MaxHandOverBy = 200 // HandOverCheck is what an ask is held to: the mesh's key, this node's name, the clock, and the nonces taken. type HandOverCheck struct { Signer ed25519.PublicKey - // SignerNow, when set, is the controller's key as the engine trusts it at this ask, read per ask in place of Signer: - // nil once the controller's lease is handed away, which refuses every ask signed by its key (novox/hq ADR 0563 §7, - // ADR 0550 §2). - SignerNow func() ed25519.PublicKey - Node string - Now func() time.Time - Nonces *Nonces + Node string + Now func() time.Time + Nonces *Nonces } // Nonces are the nonces of asks taken, each kept until its ask expires: an ask is taken once. Kept in memory — @@ -162,11 +158,6 @@ func believedBytesWithin(check HandOverCheck, context string, withPath bool, bod return HandOverAsk{}, nil, "the ask is not a signed ask: " + err.Error() } signer := check.Signer - if check.SignerNow != nil { - if signer = check.SignerNow(); signer == nil { - return HandOverAsk{}, nil, "the controller's lease is handed away, and no ask signed by its key is taken" - } - } if len(signer) != ed25519.PublicKeySize || !ed25519.Verify(signer, append([]byte(context), signed.Ask...), signed.Signature) { return HandOverAsk{}, nil, "the ask is not signed by the mesh" diff --git a/internal/link/handover_test.go b/internal/link/handover_test.go index 27cb742..be1b363 100644 --- a/internal/link/handover_test.go +++ b/internal/link/handover_test.go @@ -148,20 +148,14 @@ func TestAHandOverAskIsBelievedOnlySignedForThisNodeUnexpiredAndOnce(t *testing. if strings.Join(given, "; ") != "/srv/notes by jo through mesh-cli on anchor; /refused by jo through mesh-cli on anchor" { t.Fatalf("the handler was given %v", given) } - // ADR 0563 §7, ADR 0550 §2: after the lease write the engine answers no ask signed by the controller's key. The key - // is read per ask from what the engine trusts now; nil (the lease handed away) refuses, and nothing is handed over. - trusted := m.Signer - following := HandOverCheck{Signer: m.Signer, SignerNow: func() ed25519.PublicKey { return trusted }, Node: m.Node, - Now: func() time.Time { return now }, Nonces: &Nonces{}} + // Issue 1126: a link opened on the mesh has no signer (the controller that signed these asks ended), so an ask signed + // by the membership's key, the controller's, is refused and nothing is handed over. before := len(given) - trusted = nil - late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Expires = now.Add(time.Minute); a.Nonce = strings.Repeat("cd", 16) })) - if a := AnswerHandOver(following, handle, late); !strings.Contains(a.Refused, "lease is handed away") || len(given) != before { - t.Fatalf("after the lease write an ask signed by the controller was answered %+v", a) - } - trusted = m.Signer - if a := AnswerHandOver(following, handle, late); a.Refused != "" || len(given) != before+1 { - t.Fatalf("while the controller holds the lease its ask was answered %+v", a) + opened := (&natsLink{node: m.Node}).check() + opened.Now, opened.Nonces = func() time.Time { return now }, &Nonces{} + late := signedAsk(t, key, with(func(a *HandOverAsk) { a.Nonce = strings.Repeat("cd", 16) })) + if a := AnswerHandOver(opened, handle, late); !strings.Contains(a.Refused, "not signed by the mesh") || len(given) != before { + t.Fatalf("a link opened on the mesh answered an ask signed by the controller's key: %+v", a) } // A nonce is forgotten once its ask has expired, so the set does not grow for ever. now = now.Add(HandOverGood + time.Second) @@ -192,23 +186,17 @@ func TestAHandOverIsAnsweredOnlyWhenItCarriesAReply(t *testing.T) { // handingLink is a quiet link that hears hand-overs, keeping the handler it was given. type handingLink struct { *quietLink - given chan HandOverFunc - signer chan func() ed25519.PublicKey + given chan HandOverFunc } func (l handingLink) HearHandOvers(h HandOverFunc) { l.given <- h } -// FollowSigner keeps what the link is told to verify the controller's asks with. -func (l handingLink) FollowSigner(f func() ed25519.PublicKey) { l.signer <- f } - // **The link is given the queue's hand-over** when it is served: what the controller asks reaches the engine's // judgement, and nothing else. func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { m, _ := aMember(t) - l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1), signer: make(chan func() ed25519.PublicKey, 1)} + l := handingLink{quietLink: newQuietLink(), given: make(chan HandOverFunc, 1)} q := &Session{Membership: m, Timeout: time.Second} - // The link verifies the controller's asks with the key the engine trusts now (ADR 0563 §7). - q.ControllerKey = func() ed25519.PublicKey { return nil } q.HandOver = func(path, by string) (string, error) { return "the queue's: " + path + " by " + by, nil } ctx, stop := context.WithCancel(context.Background()) served := make(chan error, 1) @@ -221,14 +209,6 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { case <-time.After(5 * time.Second): t.Fatal("the link was never given the queue's hand-over") } - select { - case f := <-l.signer: - if f() != nil { - t.Fatal("the link was given another key than the queue's") - } - case <-time.After(5 * time.Second): - t.Fatal("the link was never given the key the engine trusts now") - } stop() <-served // 774 (ADR 0563 §1): a call of the node seat reaches its verb with its caller as the subject proves it, unproven on @@ -241,15 +221,13 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { return map[string]any{"module": module, "digest": digest, "diff": diff}, nil }, Compose: func(module string) (any, error) { return map[string]any{"composed": true, "retried": module}, nil }, - // ADR 0565 §6, §15: the node's key beside the controller's mirror's, the mirror's from its writer's test. + // ADR 0565 §6, §15: the node's key of the engine's state packages. Packages: func(write bool) (any, error) { key := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false}]}`) - mirror := []byte(`{"node":"n1","packages":[{"package":"curl","modules":["backup","web"],"absent":false},` + - `{"package":"nano","modules":["web"],"absent":true}]}`) if !write { - return PackagesAnswer("n1", key, mirror, nil, nil), nil + return PackagesAnswer("n1", key, nil), nil } - v := PackagesAnswer("n1", nil, nil, nil, errors.New("not granted")) + v := PackagesAnswer("n1", nil, nil) wrote := true v.Wrote = &wrote return v, nil @@ -265,11 +243,10 @@ func TestALinkThatHearsHandOversIsGivenTheQueues(t *testing.T) { // The recovery verb compose (issue 774) makes a pass now, with a module's apply tried again when named. {"compose", EngineToolSubject("compose", "n1"), `"composed":true,"retried":""`}, {"compose", EngineToolSubject("compose", "n1") + " " + `{"module":"route-proxy"}`, `"retried":"route-proxy"`}, - // The read verb packages (ADR 0565 §6) names both sources and their difference; with write, what it wrote, and a - // key the engine has not written, or a mirror it may not read, is said and not an error. - {"packages", EngineToolSubject("packages", "n1"), `"differences":[{"package":"nano","absent":true,"engine":[],"mirror":["web"]}],"agree":false`}, + // The read verb packages (ADR 0565 §6) names its source; with write, what it wrote, and a key the engine has not + // written is said and not an error. {"packages", EngineToolSubject("packages", "n1"), `"source":"mesh-host.packages","key":{"node":"n1"`}, - {"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"mirror":null,"mirror_error":"not granted","differences":[],"agree":false,"wrote":true`}, + {"packages", EngineToolSubject("packages", "n1") + " " + `{"write":true}`, `"key":null,"wrote":true`}, {"packages", EngineToolSubject("packages", "n1") + " " + `{"write":"true"}`, `"note":"the engine has written no key for this node yet`}, {"unserved", "mesh.seat.node-engine.call.unserved.n1.x", `"error":"the node seat node-engine serves no verb \"unserved\""`}, } { diff --git a/internal/link/have.go b/internal/link/have.go index 020d06a..6f8040a 100644 --- a/internal/link/have.go +++ b/internal/link/have.go @@ -756,9 +756,8 @@ func (q *Session) NoteCandidates(ctx context.Context, answers map[string]deliver q.writeHaves(ctx, bus) } -// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn or -// delivery.ComposesController) and why: written at each start of its agent, so a node returning to the controller's -// compositions says so where every have is read (issue 1126). +// NoteComposes says, in the node's own key, whose compositions the node applies (delivery.ComposesOwn) and why: written +// at each start of its agent, where every have is read (issue 1126). func (q *Session) NoteComposes(ctx context.Context, composes, why string) { q.init() s := &q.haves diff --git a/internal/link/have_test.go b/internal/link/have_test.go index 86f5a95..f092ed2 100644 --- a/internal/link/have_test.go +++ b/internal/link/have_test.go @@ -24,7 +24,7 @@ type haveLink struct { hmu sync.Mutex kv map[string]haveEntry sets []string // keys written, in order - events []string // "report" and "have ", in the order they happened + events []string // "have ", in the order they happened failing bool binds int setTries int @@ -40,16 +40,6 @@ type haveEntry struct { func newHaveLink() *haveLink { return &haveLink{quietLink: newQuietLink(), kv: map[string]haveEntry{}} } -func (l *haveLink) Report(ctx context.Context, node string, body []byte) error { - err := l.quietLink.Report(ctx, node, body) - if err == nil { - l.hmu.Lock() - l.events = append(l.events, "report") - l.hmu.Unlock() - } - return err -} - func (l *haveLink) Health(context.Context, string, []byte) error { return nil } func (l *haveLink) HaveBind(ctx context.Context) (HaveBucket, error) { @@ -241,9 +231,8 @@ func TestAModulesHealthFollowsTheLook(t *testing.T) { } } -// 1132/AC6: the report is sent first and carries no new key; the node's key holds its windows, busy, strays, filters -// and profile. -func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { +// 1132/AC6: the node's key holds its windows, busy, strays, filters and profile. +func TestTheNodeKeyCarriesTheNodesFacts(t *testing.T) { l := newHaveLink() q, _ := aHaveQueue(l) rep := appliedModules() @@ -253,13 +242,6 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { rep.Filters = []Filter{{Where: "inet mesh", Owner: "mesh"}} rep.Profile = map[string]any{"cores": 8} q.SayFacts(context.Background(), rep) - if len(l.events) < 2 || l.events[0] != "report" { - t.Fatalf("the report was not first: %v", l.events) - } - raw, _ := json.Marshal(l.said()[0]) - if strings.Contains(string(raw), "Modules") || strings.Contains(string(raw), "\"modules\"") { - t.Errorf("the report carries the module facts: %s", raw) - } got, err := delivery.DecodeHaveNode(l.kv["n1.node"].value) if err != nil { t.Fatal(err) @@ -310,16 +292,13 @@ func TestTheReportIsSentFirstAndTheNodeKeyCarriesTheNodesFacts(t *testing.T) { } } -// 1132/AC7: a write the bucket refuses is said in the log; the report went out, and the next apply writes again. -func TestAFailedHaveWriteIsSaidAndNeverBlocksTheReport(t *testing.T) { +// 1132/AC7: a write the bucket refuses is said in the log, and the next apply writes again. +func TestAFailedHaveWriteIsSaidAndWrittenAgain(t *testing.T) { l := newHaveLink() l.failing = true q, said := aHaveQueue(l) if !q.SayFacts(context.Background(), appliedModules()) { - t.Fatal("the report did not go out") - } - if len(l.said()) != 1 { - t.Fatalf("reports sent: %d", len(l.said())) + t.Fatal("the facts were not said with a link open") } if !strings.Contains(strings.Join(*said, "\n"), "n1.a") || !strings.Contains(strings.Join(*said, "\n"), "refused") { t.Errorf("the failure is not in the log: %v", *said) diff --git a/internal/link/hearing_nats.go b/internal/link/hearing_nats.go index 06cedf2..93209a9 100644 --- a/internal/link/hearing_nats.go +++ b/internal/link/hearing_nats.go @@ -8,7 +8,6 @@ import ( "os" "regexp" "strings" - "sync" "time" "github.com/nats-io/nats.go" @@ -18,11 +17,6 @@ import ( // The host's link on the bus: one connection, and what is asked of this node over it. The node hears no declaration // (novox/hq ADR 0539): it watches its wants and its compositions, and says its facts. -// EnrolSubject is where a joining machine asks. One subject for every node, because a machine -// enrolling has no name the mesh has agreed to yet — which is why its authority to publish here is -// the whole of what its enrolment user may do. -const EnrolSubject = "mesh.control.enrol" - // AskReportSubject is where the mesh asks this node to say its facts again: its own, on core NATS and off any stream. // The answer is the facts event on its report subject — nobody's inbox is answered. func AskReportSubject(node string) string { return "mesh.node." + node + ".ask.report" } @@ -39,14 +33,11 @@ func natsURL(address string) string { // natsLink is this node's connection as a JetStream subscription. type natsLink struct { - // mu guards signerNow (FollowSigner). - mu sync.Mutex - signerNow func() ed25519.PublicKey - conn *nats.Conn - js nats.JetStreamContext - asking *nats.Subscription - asked chan struct{} - handing *nats.Subscription + conn *nats.Conn + js nats.JetStreamContext + asking *nats.Subscription + asked chan struct{} + handing *nats.Subscription // searching hears the controller's terminal asking for a fresh setuid search (novox/hq issue 361). searching *nats.Subscription // judging hears the controller asking for a read-only judgement of this node's directories (issue 358). @@ -117,7 +108,9 @@ func dialNats(ctx context.Context, m Membership, timeout time.Duration) (Link, e } l := &natsLink{ - conn: conn, js: js, node: m.Node, user: user, signer: m.Signer, + // No signer: the asks these subjects carry were signed by the controller, which ended by its unassignment + // (issue 1126), so a link opened on the mesh believes none of them (ADR 0563 §7, ADR 0550 §2). + conn: conn, js: js, node: m.Node, user: user, lost: make(chan error, 1), asked: make(chan struct{}, 1), refused: make(chan string, 64), @@ -163,19 +156,9 @@ func (l *natsLink) hearAsks() { } } -// FollowSigner has every signed ask heard on this link verified with the key the engine trusts at that ask, the -// controller's while it holds its lease and none from its hand-over (novox/hq ADR 0563 §7, ADR 0550 §2). -func (l *natsLink) FollowSigner(f func() ed25519.PublicKey) { - l.mu.Lock() - l.signerNow = f - l.mu.Unlock() -} - -// check is how a signed ask is believed on this link: the static membership signer until FollowSigner names the trust. +// check is how a signed ask is believed on this link: by the link's signer, which a link opened on the mesh has none of. func (l *natsLink) check() HandOverCheck { - l.mu.Lock() - defer l.mu.Unlock() - return HandOverCheck{Signer: l.signer, SignerNow: l.signerNow, Node: l.node} + return HandOverCheck{Signer: l.signer, Node: l.node} } // HearHandOvers listens for the controller asking this node to hand a directory used as found to the mesh @@ -323,10 +306,6 @@ func (l *natsLink) Close() { } } -func (l *natsLink) Report(ctx context.Context, node string, body []byte) error { - return OverNATS{Conn: l.conn, JS: l.js}.Report(ctx, node, body) -} - func (l *natsLink) Health(ctx context.Context, node string, body []byte) error { return OverNATS{Conn: l.conn, JS: l.js}.Health(ctx, node, body) } diff --git a/internal/link/messages.go b/internal/link/messages.go index 7a64191..531a1ab 100644 --- a/internal/link/messages.go +++ b/internal/link/messages.go @@ -1,8 +1,6 @@ package link import ( - "strconv" - "strings" "time" "git.novox.be/novox/mesh-sdk/go/delivery" @@ -47,13 +45,6 @@ type Report struct { // machine saying what is true of it, not asking for anything. Carried []int `json:"carried,omitempty"` - // Writer, and Order — the node-engine's own epoch and sequence — say who wrote these facts and where they stand - // among everything it wrote (novox/hq ADR 0227 §2, ADR 0539 §9): the epoch is one per start of the node-engine, - // the sequence one higher for every facts event, so the controller refuses older facts by this order. On the - // wire as three top-level keys. - Writer string `json:"writer,omitempty"` - Order - // Held is what this adopted node found and is keeping as it was until its module is taken // (novox/hq ADR 0100). Without it an adopted node reads as converged. Held []Held `json:"held,omitempty"` @@ -142,10 +133,6 @@ type Report struct { // that does not load is a machine filtering nothing while its unit reports success. Outward []string `json:"outward,omitempty"` - // Rekey is this node taking a found tunnel's key as its overlay key after enrolment (novox/hq - // ADR 0105). Not an account of the machine: a report carrying one says nothing else. - Rekey *Rekey `json:"rekey,omitempty"` - // Rollbacks is what this machine's witnesses decided about a core build that was not healthy // in bound (novox/hq to-be 45 §8, ADR 0227 rule 8): the node-engine's launcher about the // engine, the engine about the controller and the node tools. **Said on every report while it @@ -488,9 +475,8 @@ const WitnessContract = 1 // The core components a witness judges (to-be 45 §8), as a rollback names them. const ( - ComponentEngine = "node-engine" - ComponentController = "controller" - ComponentNodeTools = "node-tools" + ComponentEngine = "node-engine" + ComponentNodeTools = "node-tools" ) // What a witness concluded, as a rollback says it. @@ -506,7 +492,7 @@ const ( // RestoreFailed is a restoration that was attempted and did not complete. RestoreFailed = "restore-failed" // Unwitnessed is a build whose health could not be judged at all within the bound — the - // witness could not ask (no grant, no lease bucket) — so it is neither proved nor rolled back. + // witness could not ask (no grant, the bus away) — so it is neither proved nor rolled back. Unwitnessed = "unwitnessed" // Halted is the node-engine's launcher giving up: the build it would go back to fails too, so // the fault is the machine's and not a build's. @@ -531,26 +517,6 @@ type Rollback struct { At time.Time `json:"at"` } -// Order is where a writer's saying stands among everything it said (novox/hq ADR 0227 §2): its **epoch**, and its -// **sequence** within it. A node-engine's epoch is the time it started, so a later start orders after an earlier one -// with nothing kept on disk. -type Order struct { - Epoch int64 `json:"epoch,omitempty"` - Sequence int64 `json:"sequence,omitempty"` -} - -// Words is an order as a person reads it in a log line. Not String: Report embeds Order, and a -// Stringer promoted onto every report would print each one as its order alone. -func (o Order) Words() string { - switch { - case o.Epoch > 0: - return "epoch " + strconv.FormatInt(o.Epoch, 10) + ", sequence " + strconv.FormatInt(o.Sequence, 10) - case o.Sequence > 0: - return "sequence " + strconv.FormatInt(o.Sequence, 10) + ", no epoch" - } - return "no order" -} - // CarriedTunnel is this node's account of the tunnel it took over. State is one of the Carried // states below; Note is what the host did about it, when it did something. type CarriedTunnel struct { @@ -573,36 +539,6 @@ const ( CarriedDown = "down" ) -// Rekey is this node saying it took a found tunnel's key as its overlay key after enrolling -// (novox/hq ADR 0105): the path for a node that enrolled before the mesh knew to take a tunnel -// over, since re-enrolling would rotate every key it holds. Signed with the identity key over -// RekeyProof, so a report forged on a stolen broker account cannot move this node's overlay key. -type Rekey struct { - // Previous is the overlay key this node held until now, as the mesh records it; the mesh - // refuses a rekey naming another, which is how a replayed one is refused. - Previous string `json:"previous"` - OverlayKey string `json:"overlay_key"` - Tunnel *Tunnel `json:"tunnel"` - Proof []byte `json:"proof"` -} - -// RekeyProof is what a node signs when it rekeys — the node, the key it leaves, the key it takes -// and the tunnel it took it from — so a proof cannot be moved to another node or another tunnel. -// Byte for byte the mesh's own (mesh-controller internal/link RekeyProof). -func RekeyProof(node, previous, key string, tunnel *Tunnel) []byte { - var t Tunnel - if tunnel != nil { - t = *tunnel - } - peers := make([]string, 0, len(t.Peers)) - for _, p := range t.Peers { - peers = append(peers, p.PublicKey+"@"+p.Address) - } - return []byte("novox-mesh-rekey\x00" + node + "\x00" + previous + "\x00" + key + "\x00" + - t.Interface + "\x00" + t.Unit + "\x00" + t.Config + "\x00" + strconv.Itoa(t.Port) + "\x00" + - t.Address + "\x00" + t.Range + "\x00" + t.PublicKey + "\x00" + strings.Join(peers, ",")) -} - // Held is one file or container found on an adopted node and kept as it was. type Held struct { ID string `json:"id"` diff --git a/internal/link/messages_test.go b/internal/link/messages_test.go index b27f39b..7ea51d2 100644 --- a/internal/link/messages_test.go +++ b/internal/link/messages_test.go @@ -12,9 +12,6 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { value any expect []string }{ - // novox/hq ADR 0539 §9: the node's facts carry the node-engine's own writer, epoch and sequence. - {Report{Node: "n", Writer: Writer, Order: Order{Epoch: 1, Sequence: 2}}, - []string{"node", "writer", "epoch", "sequence"}}, {Report{Node: "n", Applied: []string{"a"}, Failed: map[string]string{"k": "v"}, Refused: "r"}, []string{"node", "applied", "failed", "refused"}}, // novox/hq ADR 0100: what an adopted node holds, the firewall it was found with, and what @@ -27,7 +24,7 @@ func TestTheWireFormatIsExactlyTheseFieldNames(t *testing.T) { []string{"protocol", "address", "port", "by", "published", "container-port"}}, // novox/hq to-be 45 §8: a witness's verdicts, said on every report while they stand, and the // witness contract this host keeps. - {Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentController}}, Witness: WitnessContract}, + {Report{Node: "n", Rollbacks: []Rollback{{Component: ComponentNodeTools}}, Witness: WitnessContract}, []string{"node", "rollbacks", "witness"}}, {Rollback{Component: ComponentNodeTools, From: "sha256:b", To: "sha256:a", Outcome: RolledBack, Why: "w"}, []string{"component", "from", "to", "outcome", "why", "at"}}, diff --git a/internal/link/run.go b/internal/link/run.go index 968d504..2e8b08d 100644 --- a/internal/link/run.go +++ b/internal/link/run.go @@ -3,7 +3,6 @@ package link import ( "context" "crypto/ed25519" - "encoding/json" "errors" "fmt" "time" @@ -199,12 +198,6 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann } // And the controller asking this node to hand a directory used as found to the mesh (novox/hq issue // 356), answered on the link the question came over. - // Every signed ask is verified with the key trusted at that ask, none after the lease write (ADR 0563 §7). - if f, ok := link.(interface { - FollowSigner(func() ed25519.PublicKey) - }); ok && queue.ControllerKey != nil { - f.FollowSigner(queue.ControllerKey) - } if h, ok := link.(HandOvers); ok { h.HearHandOvers(queue.HandOver) } @@ -290,41 +283,3 @@ func serve(ctx context.Context, link Link, m Membership, queue *Session, say Ann } } } - -// publishReport tells the mesh what this node did, and says whether the broker took it. -// Publish sends one report on this node's own connection and returns: the one-shot path for a -// report a command makes rather than the running host — a rekey (novox/hq ADR 0105). The same -// account, the same pinned certificate and the same exchange as the running host's reports. -func Publish(ctx context.Context, m Membership, report Report, timeout time.Duration) error { - link, err := Open(ctx, m, timeout) - if err != nil { - return err - } - defer link.Close() - var said string - if !publishReport(ctx, link, m, report, func(s string) { said = s }, timeout) { - return errors.New(said) - } - return nil -} - -func publishReport(ctx context.Context, bus Bus, m Membership, report Report, - say Announce, timeout time.Duration) bool { - report.Node = m.Node - body, err := json.Marshal(report) - if err != nil { - say("cannot encode this node's own report: " + err.Error()) - return false - } - publish, cancel := context.WithTimeout(ctx, timeout) - defer cancel() - - // Said rather than swallowed. A report that fails to publish leaves the mesh believing this - // node never answered, while the node believes it did — and the two would go on disagreeing - // with nothing anywhere saying so. That shape of fault is the one this project keeps finding. - if err := bus.Report(publish, m.Node, body); err != nil { - say(fmt.Sprintf("applied, and could not tell the mesh: %v", err)) - return false - } - return true -} diff --git a/internal/link/session.go b/internal/link/session.go index 832348e..3bcd467 100644 --- a/internal/link/session.go +++ b/internal/link/session.go @@ -2,7 +2,6 @@ package link import ( "context" - "crypto/ed25519" "encoding/json" "errors" "fmt" @@ -14,16 +13,11 @@ import ( "git.novox.be/novox/mesh-sdk/go/health" ) -// Writer is what the node-engine calls itself on the facts it says (novox/hq ADR 0227 §2, ADR 0539 §9): the program -// that wrote them, beside its epoch and sequence. -const Writer = "node-engine" - -// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what the controller asks -// of this machine over it, and what the machine says on it unasked: its health, its haves and its facts. +// Session is the node-engine's hold on the bus (novox/hq ADR 0539 §9): the link open now, what is asked of this +// machine over it, and what the machine says on it unasked: its health and its haves. // -// **It applies nothing and orders nothing.** A module is applied by the node agent from its want, and the facts are an -// event of the node, ordered by the node-engine's own epoch and sequence. Nothing here receives, orders or reconciles -// a whole declaration. +// **It applies nothing and orders nothing.** A module is applied by the node agent from its want. Nothing here +// receives, orders or reconciles a whole declaration. type Session struct { // Fallback is the engine's fall back to its old login its health report carries now (Rotator.Fallback); nil says none. Fallback func() *delivery.LoginFallback @@ -54,9 +48,6 @@ type Session struct { // MoveSecret moves one value sealed to this machine into the vault, asked by the controller during the move of // the secrets (design 57's V3, issue 676). Nil answers every such ask with a refusal. MoveSecret MoveSecretFunc - // ControllerKey is the controller's key as the engine trusts it now: nil once its lease is handed away (novox/hq - // ADR 0563 §7). Each link verifies every signed ask with it, read per ask; nil ControllerKey keeps the membership's. - ControllerKey func() ed25519.PublicKey // Acts is the node seat node-engine this engine holds (novox/hq ADR 0563): served on every link, and its asks to the // operator asked and heard over it. Nil serves no seat. Acts *HandActs @@ -88,12 +79,8 @@ type Session struct { haveAgainPending bool haveAgainTries int - // saying is held while a facts event is made and said, so they leave in the order they are made. + // saying is held while a facts event is noted, so the haves are written in the order the facts are made. saying sync.Mutex - // epoch is this process's own: the time it started, in milliseconds, so a later process orders after an earlier one - // without anything kept on disk; sequence is one higher for every facts event it says (ADR 0227 §2). - epoch int64 - sequence int64 // haves is what this node-engine writes beside its facts (novox/hq ADR 0429 §4, issue 1132). haves haveState @@ -137,7 +124,6 @@ func (q *Session) init() { if q.Say == nil { q.Say = func(string) {} } - q.epoch = time.Now().UnixMilli() q.changed = make(chan struct{}) q.owed = make(chan struct{}, 1) }) @@ -160,8 +146,10 @@ func (q *Session) oweHealth() { } } -// SayFacts says this node's facts on the link open now, in the node-engine's own order, and whether the bus took them. -// A facts event made with no link open is not kept: the next link says the facts as they are then. +// SayFacts says this node's facts in its haves on the link open now, in the order they are made, and whether a link +// was open to say them on. A facts event made with no link open is not kept: the next link says the facts as they are +// then. They are said in the haves alone: the CONTROL stream they were also published on was the controller's, which +// ended by its unassignment (issue 1126). A write the bucket refuses is said and written again (issue 1132). func (q *Session) SayFacts(ctx context.Context, r Report) bool { q.init() q.saying.Lock() @@ -172,14 +160,7 @@ func (q *Session) SayFacts(ctx context.Context, r Report) bool { if bus == nil { return false } - q.sequence++ r.Node = q.Membership.Node - r.Writer = Writer - r.Order = Order{Epoch: q.epoch, Sequence: q.sequence} - if !publishReport(context.WithoutCancel(ctx), bus, q.Membership, r, q.Say, q.Timeout) { - return false - } - // After the facts, which are sent first and unchanged; a write the bucket refuses never undoes them (issue 1132). if q.noteApply(r) { q.writeHaves(ctx, bus) } @@ -309,23 +290,6 @@ func (q *Session) AskTool(ctx context.Context, module, tool string) (health.Answ return ReadToolAnswer(reply) } -// AskController asks the controller a line mesh-cli was given on this machine, on this node's own subject, over -// the link open now (novox/hq ADR 0272 §3), and returns its answer as it came. No link open is said, not waited for. -func (q *Session) AskController(ctx context.Context, body []byte) ([]byte, error) { - q.init() - q.mu.Lock() - bus := q.bus - q.mu.Unlock() - if bus == nil { - return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) - } - asker, ok := bus.(ToolBus) - if !ok { - return nil, fmt.Errorf("%w: the link to the bus open now cannot ask the controller", ErrNotAsked) - } - return asker.Ask(ctx, CLISubject(q.Membership.Node), body) -} - // ErrNotAsked is a request the link knows it never delivered: no link open, or nobody listening on its subject. Any // other failure of a request may come after it was delivered, so what came of it is not known (novox/hq ADR 0272). var ErrNotAsked = errors.New("not asked") diff --git a/internal/link/session_test.go b/internal/link/session_test.go index 331efb0..7fc1fdd 100644 --- a/internal/link/session_test.go +++ b/internal/link/session_test.go @@ -3,7 +3,6 @@ package link import ( "context" "crypto/ed25519" - "encoding/json" "errors" "fmt" "os" @@ -15,42 +14,16 @@ import ( "github.com/nats-io/nats.go" ) -// quietLink is a link with no broker behind it. A report is refused once the context it is published on is cancelled, -// as the bus's is ("reporting: context canceled"), or while refusing. +// quietLink is a link with no broker behind it. type quietLink struct { - mu sync.Mutex - reports []Report - refusing bool - lost chan error + lost chan error } func newQuietLink() *quietLink { return &quietLink{lost: make(chan error, 1)} } -func (l *quietLink) Report(ctx context.Context, _ string, body []byte) error { - if err := ctx.Err(); err != nil { - return err - } - l.mu.Lock() - defer l.mu.Unlock() - if l.refusing { - return errors.New("refused") - } - var r Report - if err := json.Unmarshal(body, &r); err != nil { - return err - } - l.reports = append(l.reports, r) - return nil -} func (l *quietLink) Lost() <-chan error { return l.lost } func (l *quietLink) Close() {} -func (l *quietLink) said() []Report { - l.mu.Lock() - defer l.mu.Unlock() - return append([]Report(nil), l.reports...) -} - func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { t.Helper() public, private, err := ed25519.GenerateKey(nil) @@ -60,40 +33,6 @@ func aMember(t *testing.T) (Membership, ed25519.PrivateKey) { return Membership{Node: "n1", Signer: public}, private } -// R6-A AC7: the facts event carries the node-engine's own writer, epoch and sequence (novox/hq ADR 0227 §2, ADR 0539 -// §9): one epoch per start, a sequence one higher for every event, so the controller can refuse older facts. -func TestTheFactsEventCarriesTheNodeEnginesWriterEpochAndSequence(t *testing.T) { - m, _ := aMember(t) - l := newQuietLink() - s := &Session{Membership: m, Timeout: time.Second} - s.Attach(context.Background(), l) - - if !s.SayFacts(context.Background(), Report{Firewall: "none"}) || !s.SayFacts(context.Background(), Report{}) { - t.Fatal("the facts were not taken") - } - got := l.said() - if len(got) != 2 { - t.Fatalf("%d facts events were said", len(got)) - } - for _, r := range got { - if r.Writer != Writer || r.Node != "n1" || r.Epoch <= 0 { - t.Fatalf("the facts do not say who wrote them: %+v", r) - } - } - if got[0].Epoch != got[1].Epoch || got[1].Sequence != got[0].Sequence+1 || got[0].Sequence <= 0 { - t.Fatalf("the order is not one epoch with a rising sequence: %+v then %+v", got[0].Order, got[1].Order) - } - - // A later start orders after this one with nothing kept on disk. - time.Sleep(2 * time.Millisecond) - later := &Session{Membership: m, Timeout: time.Second} - later.Attach(context.Background(), l) - later.SayFacts(context.Background(), Report{}) - if last := l.said()[2]; last.Epoch <= got[1].Epoch { - t.Fatalf("a later start has epoch %d, not above %d", last.Epoch, got[1].Epoch) - } -} - // Facts made with no link open are not kept, and nothing is said: the next link says them as they are then. func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { m, _ := aMember(t) @@ -104,7 +43,7 @@ func TestFactsWithNoLinkOpenAreNotSaid(t *testing.T) { } s.Attach(context.Background(), l) s.Detach(l) - if s.Linked() || s.SayFacts(context.Background(), Report{}) || len(l.said()) != 0 { + if s.Linked() || s.SayFacts(context.Background(), Report{}) { t.Fatal("facts were said on a link that was let go") } } @@ -140,16 +79,6 @@ func aBus(t *testing.T) (*nats.Conn, nats.JetStreamContext) { if err != nil { t.Fatal(err) } - want := &nats.StreamConfig{Name: "CONTROL", Subjects: []string{"mesh.control.*.report", "mesh.control.enrol"}, - Retention: nats.WorkQueuePolicy} - if _, err := js.StreamInfo(want.Name); err != nil { - if _, err := js.AddStream(want); err != nil { - t.Fatal(err) - } - } - if err := js.PurgeStream(want.Name); err != nil { - t.Fatal(err) - } return conn, js } diff --git a/internal/link/testdata/witness-grants.conf b/internal/link/testdata/witness-grants.conf index ccc5311..78be114 100644 --- a/internal/link/testdata/witness-grants.conf +++ b/internal/link/testdata/witness-grants.conf @@ -3,16 +3,15 @@ # docker run -d --rm --name w -p 14224:4222 -v $PWD/internal/link/testdata:/c:ro nats:2.11-alpine -js -c /c/witness-grants.conf # MESH_TEST_NATS_GRANTS=nats://127.0.0.1:14224 go test ./internal/link/ -run TestNatsWitness # -# `node.anchor` is a machine's host with exactly the two subjects the witness asks added to what a -# host already has (its inbox); `node.bare` is a host without them. `runtime` stands in for the tool -# runtime, `admin` for the controller writing the lease. +# `node.anchor` is a machine's host with exactly the subject the witness asks added to what a host +# already has (its inbox); `node.bare` is a host without it. `runtime` stands in for the tool runtime. jetstream: enabled accounts { MESH { jetstream: enabled users = [ { user: "node.anchor", password: "a", permissions: { - publish: { allow: ["$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder", "$SRV.PING.node-tools.anchor"] } + publish: { allow: ["$SRV.PING.node-tools.anchor"] } subscribe: { allow: ["_INBOX.node.anchor.>"] } } } { user: "node.bare", password: "b", permissions: { @@ -23,7 +22,6 @@ accounts { subscribe: { allow: ["$SRV.PING.node-tools.>"] } allow_responses: { max: 1, ttl: "1m" } } } - { user: "admin", password: "x" } ] } } diff --git a/internal/link/witnessing.go b/internal/link/witnessing.go index 4b1f8ad..ed364d9 100644 --- a/internal/link/witnessing.go +++ b/internal/link/witnessing.go @@ -10,14 +10,11 @@ import ( "github.com/nats-io/nats.go" ) -// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the controller and the -// node tools it placed on this machine, by what the bus says about them rather than by what they -// say about themselves. +// What a witness asks the bus (novox/hq to-be 45 §8): the node-engine judging the node tools it placed on this +// machine, by what the bus says about them rather than by what they say about themselves. // -// **Two questions, both read-only, both on the host's own link.** The controller's lease key, read -// by JetStream's direct get — one subject, no stream information, nothing written; and this -// machine's tool runtime, asked the NATS services protocol's PING by its name and this machine's — -// so only this machine's runtime can answer it. +// **One question, read-only, on the host's own link**: this machine's tool runtime, asked the NATS services protocol's +// PING by its name and this machine's, so only this machine's runtime can answer it. // ErrCannotAsk is a question the bus did not let this host put, or did not answer: no grant for it, // no such bucket, the bus slow. **It says nothing about the build being judged**, so a witness @@ -30,20 +27,11 @@ var ErrNoAnswer = errors.New("nothing answered") // Asker is what a witness asks through. The link open now implements it; nil while there is none. type Asker interface { - // ReadKey is a key-value bucket's current value for a key. Found false is nobody holding it — - // absent, deleted or expired; an error wrapping ErrCannotAsk is no reading at all. - ReadKey(ctx context.Context, bucket, key string) (value []byte, found bool, err error) // Ping asks the service `service`'s instance `id` whether it is there. Nil is an answer; // ErrNoAnswer is none in time; ErrCannotAsk is no asking. Ping(ctx context.Context, service, id string) error } -// DirectGetSubject is the one subject a host asks a bucket's key on: JetStream's direct get of the -// bucket's stream, for the key's own subject. What a host's grant names exactly — no wildcard. -func DirectGetSubject(bucket, key string) string { - return "$JS.API.DIRECT.GET.KV_" + bucket + ".$KV." + bucket + "." + key -} - // PingSubject is the services protocol's PING of one instance of one service. func PingSubject(service, id string) string { return "$SRV.PING." + service + "." + id } @@ -58,36 +46,6 @@ func (q *Session) Asker() Asker { return nil } -func (l *natsLink) ReadKey(ctx context.Context, bucket, key string) ([]byte, bool, error) { - subject := DirectGetSubject(bucket, key) - before := l.conn.LastError() - msg, err := l.conn.RequestWithContext(ctx, subject, nil) - switch { - case errors.Is(err, nats.ErrNoResponders): - return nil, false, fmt.Errorf("%w: the bus has no bucket %s that answers a direct get", ErrCannotAsk, bucket) - case err != nil: - return nil, false, fmt.Errorf("%w: reading %s from %s: %v%s", ErrCannotAsk, key, bucket, err, l.refusal(subject, before)) - } - if msg.Header != nil { - switch status := msg.Header.Get("Status"); status { - case "": - case "404": - return nil, false, nil - default: - return nil, false, fmt.Errorf("%w: reading %s from %s: the bus answered %s %s", ErrCannotAsk, key, - bucket, status, msg.Header.Get("Description")) - } - switch msg.Header.Get("KV-Operation") { - case "DEL", "PURGE": - return nil, false, nil - } - } - if len(msg.Data) == 0 { - return nil, false, nil - } - return msg.Data, true, nil -} - func (l *natsLink) Ping(ctx context.Context, service, id string) error { subject := PingSubject(service, id) before := l.conn.LastError() diff --git a/internal/link/witnessing_nats_test.go b/internal/link/witnessing_nats_test.go index 98c29e1..9c2c006 100644 --- a/internal/link/witnessing_nats_test.go +++ b/internal/link/witnessing_nats_test.go @@ -2,7 +2,6 @@ package link import ( "context" - "encoding/json" "errors" "os" "testing" @@ -12,7 +11,7 @@ import ( ) // What a witness asks, asked of a real server with the grants the mesh composes (novox/hq to-be 45 §8): -// whether the direct get and the PING work as the two subjects named, and whether a host without them +// whether the PING works as the subject named, and whether a host without them // is told apart — "cannot ask", never "the build did not answer". See testdata/witness-grants.conf. func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { url := os.Getenv("MESH_TEST_NATS_GRANTS") @@ -28,63 +27,14 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { t.Cleanup(conn.Close) return conn } - admin := dial("admin", "x", "_INBOX.admin") - js, err := admin.JetStream() - if err != nil { - t.Fatal(err) - } - _ = js.DeleteKeyValue("mesh-controller_lease") - kv, err := js.CreateKeyValue(&nats.KeyValueConfig{Bucket: "mesh-controller_lease", TTL: 15 * time.Second, History: 1}) - if err != nil { - t.Fatal(err) - } - anchor := &natsLink{conn: dial("node.anchor", "a", "_INBOX.node.anchor")} bare := &natsLink{conn: dial("node.bare", "b", "_INBOX.node.bare")} ask := func() (context.Context, context.CancelFunc) { return context.WithTimeout(context.Background(), 2*time.Second) } - // Nobody holds it. - ctx, cancel := ask() - if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found { - t.Fatalf("an empty lease read as %v, %v", found, err) - } - cancel() - // Held. - if _, err := kv.Put("holder", []byte(`{"instance":"controller@anchor pid 1 since now","epoch":3}`)); err != nil { - t.Fatal(err) - } - ctx, cancel = ask() - value, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder") - cancel() - if err != nil || !found { - t.Fatalf("a held lease read as %v, %v", found, err) - } - var holder struct { - Instance string `json:"instance"` - } - if json.Unmarshal(value, &holder); holder.Instance == "" { - t.Fatalf("the lease's value is %s", value) - } - // Given back. - if err := kv.Delete("holder"); err != nil { - t.Fatal(err) - } - ctx, cancel = ask() - if _, found, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); err != nil || found { - t.Fatalf("a lease given back read as %v, %v", found, err) - } - cancel() - // A host without the grant cannot ask — which is not "nobody holds it". - ctx, cancel = ask() - if _, _, err := bare.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) { - t.Fatalf("a host without the grant read the lease as %v, want it unable to ask", err) - } - cancel() - // No runtime on the bus: not there to answer. - ctx, cancel = ask() + ctx, cancel := ask() if err := anchor.Ping(ctx, "node-tools", "anchor"); !errors.Is(err, ErrNoAnswer) { t.Fatalf("a PING nobody serves is %v, want no answer", err) } @@ -112,14 +62,4 @@ func TestNatsWitnessAsksWhatItIsGranted(t *testing.T) { t.Fatalf("a host without the grant heard %v, want it unable to ask", err) } cancel() - - // No lease bucket at all — a controller from before the lease: cannot ask, not "nobody holds it". - if err := js.DeleteKeyValue("mesh-controller_lease"); err != nil { - t.Fatal(err) - } - ctx, cancel = ask() - if _, _, err := anchor.ReadKey(ctx, "mesh-controller_lease", "holder"); !errors.Is(err, ErrCannotAsk) { - t.Fatalf("a bus with no lease bucket read as %v, want unable to ask", err) - } - cancel() } diff --git a/internal/meshcli/ask.go b/internal/meshcli/ask.go deleted file mode 100644 index eb281b5..0000000 --- a/internal/meshcli/ask.go +++ /dev/null @@ -1,92 +0,0 @@ -package meshcli - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "time" - - "git.novox.be/novox/mesh-sdk/go/cli" - "github.com/novox/mesh-host/internal/link" -) - -// The controller answers a line as it answers every call (novox/hq ADR 0272, review): in an envelope, `result` or -// `error`, and within its window for an answer — so a line still running is answered `running` with its call, and -// followed by the same account on the same node until it ends. Its side holds the same field names (mesh-controller -// internal/link, the mesh-cli field names' test). - -// ErrNotAsked is the link's: a request it knows it never delivered. -var ErrNotAsked = link.ErrNotAsked - -// AskFunc asks the controller one request on this node's subject and returns its reply. -type AskFunc func(ctx context.Context, body []byte) ([]byte, error) - -// followEvery is how often a line still running is asked about again. -var followEvery = 2 * time.Second - -// envelope is a call's answer: its result, or its error. -type envelope struct { - Result json.RawMessage `json:"result"` - Error string `json:"error"` -} - -// runningResult is what a call still running answers in its result. -type runningResult struct { - Running bool `json:"running"` - Call string `json:"call"` -} - -// AskTheController asks the line and follows it to its answer, within ctx. What it says is one of three things, -// never confused: what the line printed; that nothing ran, and why (the request was never taken); or that what came -// of it is not known — it may have run — naming the call to look at when one is known. -func AskTheController(ctx context.Context, ask AskFunc, asked Asked) cli.Answer { - body, _ := json.Marshal(asked) - reply, err := ask(ctx, body) - if err != nil { - // **Only a request the link knows it never asked ran nothing**: no link open, nobody listening. Any other - // failure may come after the request was delivered, and the line may have run (review of ADR 0272). - if errors.Is(err, ErrNotAsked) { - return cli.Refusal("the controller was not reached, so nothing ran: " + err.Error()) - } - return cli.NotKnown("", "no answer from the controller ("+err.Error()+")") - } - call := "" - for { - var env envelope - if err := json.Unmarshal(reply, &env); err != nil { - return cli.NotKnown(call, "the controller's answer could not be read") - } - if env.Error != "" { - return cli.NotKnown(call, "the controller answered "+env.Error) - } - var r runningResult - _ = json.Unmarshal(env.Result, &r) - if !r.Running { - var a cli.Answer - if err := json.Unmarshal(env.Result, &a); err != nil { - return cli.NotKnown(call, "the controller's answer could not be read") - } - return a - } - if r.Call != "" { - call = r.Call - } - if call == "" { - return cli.NotKnown("", "the controller says the line is running and names no call to follow") - } - // Asked again until it ends, through a link that may drop and come back: a failed follow is tried - // again, and only ctx ending stops it. - for { - select { - case <-ctx.Done(): - return cli.NotKnown(call, fmt.Sprintf("no answer within the time mesh-cli waits (%s)", cli.AskWithin)) - case <-time.After(followEvery): - } - body, _ := json.Marshal(Asked{Follow: call, Account: asked.Account, UID: asked.UID, Session: asked.Session}) - if reply, err = ask(ctx, body); err == nil { - break - } - } - } -} diff --git a/internal/meshcli/ask_test.go b/internal/meshcli/ask_test.go deleted file mode 100644 index 3291f65..0000000 --- a/internal/meshcli/ask_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package meshcli - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strings" - "sync" - "testing" - "time" -) - -// controllerSays is a controller answering each request in turn from a script, and recording what it was asked. -type controllerSays struct { - mu sync.Mutex - script []func(Asked) ([]byte, error) - asked []Asked -} - -func (c *controllerSays) ask(_ context.Context, body []byte) ([]byte, error) { - c.mu.Lock() - defer c.mu.Unlock() - var a Asked - _ = json.Unmarshal(body, &a) - c.asked = append(c.asked, a) - if len(c.script) == 0 { - return nil, context.DeadlineExceeded - } - next := c.script[0] - if len(c.script) > 1 { - c.script = c.script[1:] - } - return next(a) -} - -func result(v any) func(Asked) ([]byte, error) { - return func(Asked) ([]byte, error) { return json.Marshal(map[string]any{"result": v}) } -} - -func fastFollow(t *testing.T) { - was := followEvery - followEvery = time.Millisecond - t.Cleanup(func() { followEvery = was }) -} - -// A line still running when the controller first answers is followed, by its call, to what it printed. -func TestALineStillRunningIsFollowedToItsAnswer(t *testing.T) { - fastFollow(t) - running := result(map[string]any{"running": true, "call": "call-9"}) - c := &controllerSays{script: []func(Asked) ([]byte, error){running, running, running, - result(map[string]any{"stdout": []byte("pushed\n"), "exit": 0, "terminal": true})}} - a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"push", "laptop"}, Account: "op", UID: 1000}) - if string(a.Stdout) != "pushed\n" || !a.Terminal || a.Refused != "" || a.Unknown != "" { - t.Fatalf("followed to %+v", a) - } - for _, f := range c.asked[1:] { - if f.Follow != "call-9" || f.Account != "op" || len(f.Line) != 0 { - t.Fatalf("a follow asked %+v", f) - } - } -} - -// **No answer in time is never "nothing ran"** (review of ADR 0272): the line was asked and may have run, and the -// call to look at is named when it is known. -func TestNoAnswerInTimeSaysTheLineMayHaveRun(t *testing.T) { - fastFollow(t) - c := &controllerSays{} - a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"push"}, Account: "op"}) - if a.Refused != "" || !strings.Contains(a.Unknown, "may have run") || a.Exit == 0 { - t.Fatalf("a request with no answer reads %+v", a) - } - c = &controllerSays{script: []func(Asked) ([]byte, error){result(map[string]any{"running": true, "call": "call-3"}), - func(Asked) ([]byte, error) { return nil, context.DeadlineExceeded }}} - ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) - defer cancel() - a = AskTheController(ctx, c.ask, Asked{Line: []string{"push"}, Account: "op"}) - if a.Refused != "" || !strings.Contains(a.Unknown, "call-3") || !strings.Contains(a.Unknown, "may have run") { - t.Fatalf("a running line never answered reads %+v", a) - } -} - -// A request the bus never carried to a controller ran nothing, and says so. -func TestARequestNobodyTookSaysNothingRan(t *testing.T) { - c := &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) { - return nil, fmt.Errorf("%w: nothing on this machine answers mesh.control.laptop.cli", ErrNotAsked) - }}} - a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"status"}, Account: "op"}) - if !strings.Contains(a.Refused, "nothing ran") || a.Unknown != "" { - t.Fatalf("a request nobody took reads %+v", a) - } - c = &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) { - return nil, fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked) - }}} - if a := AskTheController(context.Background(), c.ask, Asked{Line: []string{"status"}}); !strings.Contains(a.Refused, "nothing ran") { - t.Fatalf("no link reads %+v", a) - } -} - -// **Any error once the request may have been delivered is not known, never "nothing ran"** (review of ADR 0272): -// only a request the link says it never asked — no link open, nobody listening — ran nothing. -func TestOnlyARequestNeverAskedSaysNothingRan(t *testing.T) { - for _, c := range []struct { - err error - nothing bool - }{ - {fmt.Errorf("%w: this machine has no link to the bus open now", ErrNotAsked), true}, - {fmt.Errorf("%w: nothing on this machine answers mesh.control.laptop.cli", ErrNotAsked), true}, - {errors.New("nats: connection closed"), false}, - {errors.New("mesh.control.laptop.cli: permissions violation"), false}, - {context.DeadlineExceeded, false}, - } { - err := c.err - ctl := &controllerSays{script: []func(Asked) ([]byte, error){func(Asked) ([]byte, error) { return nil, err }}} - a := AskTheController(context.Background(), ctl.ask, Asked{Line: []string{"push"}, Account: "op"}) - if c.nothing && (!strings.Contains(a.Refused, "nothing ran") || a.Unknown != "") { - t.Errorf("%v reads %+v; it was never asked", err, a) - } - if !c.nothing && (a.Refused != "" || !strings.Contains(a.Unknown, "may have run")) { - t.Errorf("%v reads %+v; it may have been asked", err, a) - } - } -} diff --git a/internal/meshcli/deadline_test.go b/internal/meshcli/deadline_test.go deleted file mode 100644 index 05c4d88..0000000 --- a/internal/meshcli/deadline_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package meshcli - -import ( - "context" - "net" - "strings" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/cli" -) - -// **A connection that says nothing is bounded from the moment it is accepted** (review of ADR 0272): its deadline is -// set before any lookup, so it is answered and closed within readWithin, holds no slot after, and asks nothing. -func TestAConnectionThatSaysNothingIsAnsweredWithinItsDeadline(t *testing.T) { - was := readWithin - readWithin = 200 * time.Millisecond - t.Cleanup(func() { readWithin = was }) - asked := false - socket := serving(t, func(context.Context, Asked) cli.Answer { - asked = true - return cli.Answer{} - }) - conn, err := net.Dial("unix", socket) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) - a, err := cli.ReadAnswer(conn) - if err != nil { - t.Fatalf("a silent connection was not answered within its deadline: %v", err) - } - if asked || a.Refused == "" || !strings.Contains(a.Refused, "nothing was asked") { - t.Fatalf("a silent connection was asked (%v) or answered %+v", asked, a) - } - // Its slot was given back: the account's next line is not busy. - b, err := cli.Ask(context.Background(), socket, []string{"status"}) - if err != nil || strings.Contains(b.Refused, "busy") || !asked { - t.Fatalf("the next line after a silent one reads %+v (%v), asked %v", b, err, asked) - } -} diff --git a/internal/meshcli/meshcli.go b/internal/meshcli/meshcli.go deleted file mode 100644 index 31a4b50..0000000 --- a/internal/meshcli/meshcli.go +++ /dev/null @@ -1,339 +0,0 @@ -// Package meshcli is the node-engine's side of `mesh-cli` (novox/hq ADR 0272): a socket on this machine that any -// account may connect to, the account read from the kernel, and the line asked of the controller. -// -// **The node-engine vouches; it does not decide.** It adds what only it can know — which account on this machine -// is asking, from the kernel's peer credentials, never from anything the program says — and sends the request out -// over its own link, on its own node's subject. The controller judges whether the line runs as its terminal, as an -// ordinary call, or not at all (ADR 0272 §4). Nothing received here is run or applied on the machine. -// -// **No listening control surface** (novox/hq ADR 0004, ADR 0272 §3): the socket has no network address, so -// nothing on another machine reaches it, and nothing sent to it acts on this one. -package meshcli - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "net" - "os" - "os/user" - "path/filepath" - "strconv" - "strings" - "sync" - "time" - - "git.novox.be/novox/mesh-sdk/go/cli" -) - -// Asked is what the node-engine sends the controller on its own node's subject: the line, and the account the -// kernel says is asking. The controller's side holds the same field names (a test on each side). -type Asked struct { - Line []string `json:"line,omitempty"` - // Account is the login of the user id the kernel gave, from this machine's user database. - Account string `json:"account"` - // UID is the user id itself, so a renamed or duplicated login is still the same id. - UID uint32 `json:"uid"` - // Follow is a call a line runs as, asked about again by the account that asked it (AskTheController). - Follow string `json:"follow,omitempty"` - // Session is the login session the asking process runs in — its systemd scope, `session-.scope` under the - // account's own slice — as the kernel's cgroup says it; empty for anything else: a system service, a user unit, - // a process another account's session started. Only a login session is a person at a terminal (novox/hq ADR - // 0272): the controller makes nothing else the terminal. - Session string `json:"session,omitempty"` - // Stdin is what mesh-cli's standard input held, at most cli.MaxStdin: carried to the controller with the line - // and nowhere else. The engine says it in no journal, keeps it in no state, and never sends it again: a follow - // carries none. The controller gives it to a line that runs as its terminal alone. - Stdin []byte `json:"stdin,omitempty"` -} - -// Forward asks the controller and follows the line to what came of it (AskTheController): what it printed, that -// nothing ran and why, or that it is not known. -type Forward func(ctx context.Context, asked Asked) cli.Answer - -// readWithin bounds how long a connection may take to say its line: a program that connects and says nothing -// holds nothing for long. A variable so a test need not wait. -var readWithin = 30 * time.Second - -// refusedWithin bounds how long the engine waits for the request of a line it refuses before it is asked: a client -// that has connected writes its request within microseconds, so a refused connection holds nothing for long. Never -// longer than readWithin. A variable so a test can say. -var refusedWithin = 1 * time.Second - -// refusalsWaiting bounds how many refusals wait for their request at the same time, on the whole engine; drainCap -// bounds how much of a refused request is read, and it is read into a small buffer and dropped. The socket is open -// to every account, so a refusal must cost the engine little however many connections ask (review of ADR 0272). -// Variables so a test can say. -var ( - refusalsWaiting = make(chan struct{}, 32) - drainCap = 4 << 10 -) - -// drain reads and drops what a refused client wrote, up to the end of its request line, the end of the stream, the -// connection's deadline or drainCap bytes, whichever comes first. Nothing is kept. -func drain(r io.Reader) { - var buf [512]byte - for read := 0; read < drainCap; { - n, err := r.Read(buf[:min(len(buf), drainCap-read)]) - read += n - if bytes.IndexByte(buf[:n], '\n') >= 0 || err != nil { - return - } - } -} - -// atOnce bounds how many lines are asked at the same time, and perAccount how many of them one account's: a person -// types one at a time, so more is a program on the machine. One over either bound is answered busy at once — the -// engine never stops accepting for it, so one account cannot hold the socket from every other (review of ADR 0272). -var ( - atOnce = 8 - perAccount = 2 -) - -// slots counts the lines being asked, in all and per user id. -type slots struct { - mu sync.Mutex - all int - byUID map[uint32]int -} - -func (s *slots) take(uid uint32) bool { - s.mu.Lock() - defer s.mu.Unlock() - if s.all >= atOnce || s.byUID[uid] >= perAccount { - return false - } - s.all++ - s.byUID[uid]++ - return true -} - -func (s *slots) give(uid uint32) { - s.mu.Lock() - defer s.mu.Unlock() - s.all-- - if s.byUID[uid]--; s.byUID[uid] <= 0 { - delete(s.byUID, uid) - } -} - -// peerOf reads the connecting process's credentials from the kernel: its user id and process id. A variable so a -// test can say what the kernel would; it is the only source of the account. -var peerOf = peerCredentials - -// sessionOf reads the login session a process runs in, from the kernel. A variable so a test can say. -var sessionOf = func(pid int32, uid uint32) string { - raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) - if err != nil { - return "" - } - return userSessionIn(string(raw), uid) -} - -// sessionsDir is where logind keeps a record of each session it holds. A variable so a test can say. -var sessionsDir = "/run/systemd/sessions" - -// userSessionIn is the login session a cgroup names, when logind's own record of it says it is a person's: this -// account's (UID), of class `user` — not a greeter's or the user manager's — and of this very scope. The cgroup -// alone names a scope; only the record says whose session it is. No record, or one that does not say so, is no -// session. -func userSessionIn(cgroup string, uid uint32) string { - scope := loginSessionIn(cgroup, uid) - id, ok := strings.CutPrefix(strings.TrimSuffix(scope, ".scope"), "session-") - if !ok || id == "" || strings.ContainsAny(id, "/.") { - return "" - } - raw, err := os.ReadFile(filepath.Join(sessionsDir, id)) - if err != nil { - return "" - } - // **The record is systemd's private format**, not an interface it promises: `KEY=value` lines logind writes for - // itself. Only UID, CLASS and SCOPE are read, each at its first occurrence — logind writes each once, and a later - // line saying otherwise is not the record's word. A systemd that changes the format, renames a key or stops - // writing the file fails closed here: no session is read, so nobody is the terminal until this reads it again. - record := map[string]string{} - for _, line := range strings.Split(string(raw), "\n") { - k, v, ok := strings.Cut(line, "=") - if !ok { - continue - } - if _, said := record[k]; !said { - record[k] = v - } - } - if record["UID"] != strconv.FormatUint(uint64(uid), 10) || record["CLASS"] != "user" || record["SCOPE"] != scope { - return "" - } - return scope -} - -// loginSessionIn is the login session scope a process's cgroup names, for its own account: the unified hierarchy's -// line, `0::/user.slice/user-.slice/session-.scope`, and nothing below or beside it. Anything else is no -// login session — a service under system.slice, a unit under the account's user manager (user@.service), a -// scope of another account's. -func loginSessionIn(cgroup string, uid uint32) string { - for _, line := range strings.Split(cgroup, "\n") { - path, ok := strings.CutPrefix(line, "0::") - if !ok { - continue - } - prefix := fmt.Sprintf("/user.slice/user-%d.slice/", uid) - scope, ok := strings.CutPrefix(path, prefix) - if !ok || strings.Contains(scope, "/") || !strings.HasPrefix(scope, "session-") || !strings.HasSuffix(scope, ".scope") { - return "" - } - return scope - } - return "" -} - -// sameNamespaces refuses a peer whose mount or user namespace is not the engine's. A variable so a test can say. -var sameNamespaces = namespacesOf - -// lookupAccount names a user id from this machine's user database. -var lookupAccount = func(uid uint32) (string, error) { - u, err := user.LookupId(strconv.FormatUint(uint64(uid), 10)) - if err != nil { - return "", err - } - return u.Username, nil -} - -// Serve listens on socket until ctx ends, and answers each connection: one request, one answer. -// -// The directory is made 0755 and the socket 0666: every account on the machine may ask, because the account is -// what is vouched for and the controller decides what it may do. A file at the socket's path that is not a socket -// is never removed: the engine says so and serves nothing. -func Serve(ctx context.Context, socket string, forward Forward, say func(string)) error { - dir := filepath.Dir(socket) - if err := os.MkdirAll(dir, 0o755); err != nil { - return fmt.Errorf("mesh-cli's socket directory: %w", err) - } - // The directory is the engine's: a link, or one another account owns, could put the socket where that - // account replaces it with its own. Said, and nothing served. - if problem := notOwnDirectory(dir); problem != "" { - return fmt.Errorf("mesh-cli is not served on this machine: %s", problem) - } - if err := os.Chmod(dir, 0o755); err != nil { - return fmt.Errorf("mesh-cli's socket directory: %w", err) - } - if info, err := os.Lstat(socket); err == nil { - if info.Mode()&os.ModeSocket == 0 { - return fmt.Errorf("%s is there and is not a socket, and the node-engine did not put it there; "+ - "mesh-cli is not served on this machine until it is moved", socket) - } - // A socket left by the engine before this one: nothing listens on it any more. - if err := os.Remove(socket); err != nil { - return fmt.Errorf("the socket a previous engine left at %s: %w", socket, err) - } - } - l, err := net.ListenUnix("unix", &net.UnixAddr{Name: socket, Net: "unix"}) - if err != nil { - return fmt.Errorf("listening for mesh-cli at %s: %w", socket, err) - } - if err := os.Chmod(socket, 0o666); err != nil { - l.Close() - return fmt.Errorf("opening mesh-cli's socket to every account: %w", err) - } - go func() { - <-ctx.Done() - l.Close() - }() - held := &slots{byUID: map[uint32]int{}} - var wg sync.WaitGroup - defer wg.Wait() - for { - conn, err := l.AcceptUnix() - if err != nil { - if ctx.Err() != nil { - return nil - } - var ne net.Error - if errors.As(err, &ne) && ne.Timeout() { - continue - } - return fmt.Errorf("mesh-cli's socket: %w", err) - } - // Bounded from the first byte: nothing below may wait on a connection for longer than this before the - // line is read, whatever the peer does (review of ADR 0272). - readBy := time.Now().Add(readWithin) - _ = conn.SetDeadline(readBy) - wg.Add(1) - go func() { - defer wg.Done() - answer(ctx, conn, readBy, held, forward, say) - }() - } -} - -// answer serves one connection; readBy is the deadline set when it was accepted, for reading its line. -func answer(ctx context.Context, conn *net.UnixConn, readBy time.Time, held *slots, forward Forward, say func(string)) { - defer conn.Close() - reply := func(a cli.Answer) { - _ = conn.SetWriteDeadline(time.Now().Add(readWithin)) - if err := cli.WriteAnswer(conn, a); err != nil && say != nil { - say(fmt.Sprintf("mesh-cli: the answer could not be written back: %v", err)) - } - } - // A refusal is written only after the request it answers is read (novox/hq issue 434). The client writes its - // request first and reads the answer after; an engine that answers and closes before that write has landed - // makes the write fail with a broken pipe, and the client never reads the refusal waiting for it — under load a - // line beyond its account's bound read as an error instead of busy. The wait ends at refusedWithin, or earlier - // at the deadline set when the connection was accepted; what was written is drained and never asked. Only - // refusalsWaiting refusals wait at once: over that the refusal is written at once, as before, and only then can a - // client still find its write broken. - refuse := func(why string) { - select { - case refusalsWaiting <- struct{}{}: - by := time.Now().Add(refusedWithin) - if readBy.Before(by) { - by = readBy - } - _ = conn.SetReadDeadline(by) - drain(conn) - <-refusalsWaiting - default: - } - reply(cli.Refusal(why)) - } - uid, pid, err := peerOf(conn) - if err != nil { - // Never sent as anyone: an account the kernel did not give is no account at all. - refuse("the node-engine could not read which account is asking, so nothing was asked: " + err.Error()) - return - } - // The user id is this machine's account only in this machine's namespaces: a container running with the - // operator's id shares the number and nothing else (review of ADR 0272). - if err := sameNamespaces(pid); err != nil { - refuse("the asking process does not run in this machine's own namespaces, so it is not this " + - "machine's account and nothing was asked: " + err.Error()) - return - } - // And the login session, now: judged as the peer was when it connected, before it can exec or exit while its - // line is read (review of ADR 0272). - session := sessionOf(pid, uid) - // Looked up before a slot is taken, so a refused account never holds one while it is refused. - account, err := lookupAccount(uid) - if err != nil || account == "" { - refuse(fmt.Sprintf("user id %d is not in this machine's user database, so nothing was asked", uid)) - return - } - if !held.take(uid) { - refuse(fmt.Sprintf("busy: user id %d has %d lines being asked already, or the machine %d; ask "+ - "again when one has ended. Nothing was asked", uid, perAccount, atOnce)) - return - } - defer held.give(uid) - req, err := cli.ReadRequest(conn) - if err != nil { - reply(cli.Refusal("the request could not be read, so nothing was asked: " + err.Error())) - return - } - // From here the wait is the controller's: as long as mesh-cli waits, and no longer. - _ = conn.SetDeadline(time.Now().Add(cli.AskWithin)) - asking, cancel := context.WithTimeout(ctx, cli.AskWithin-30*time.Second) - defer cancel() - reply(forward(asking, Asked{Line: req.Line, Account: account, UID: uid, Session: session, Stdin: req.Stdin})) -} diff --git a/internal/meshcli/meshcli_test.go b/internal/meshcli/meshcli_test.go deleted file mode 100644 index 4b83f51..0000000 --- a/internal/meshcli/meshcli_test.go +++ /dev/null @@ -1,534 +0,0 @@ -package meshcli - -import ( - "context" - "encoding/json" - "errors" - "io" - "net" - "os" - "os/user" - "path/filepath" - "sort" - "strings" - "sync" - "testing" - "time" - - "git.novox.be/novox/mesh-sdk/go/cli" -) - -// serving starts Serve on a socket of the test's own, and stops it when the test ends. -func serving(t *testing.T, forward Forward) string { - t.Helper() - socket := filepath.Join(t.TempDir(), "run", "cli.sock") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { done <- Serve(ctx, socket, forward, func(string) {}) }() - t.Cleanup(func() { - cancel() - if err := <-done; err != nil { - t.Errorf("Serve ended with %v", err) - } - }) - for i := 0; i < 200; i++ { - if _, err := os.Stat(socket); err == nil { - return socket - } - time.Sleep(5 * time.Millisecond) - } - t.Fatal("the socket never appeared") - return "" -} - -// The account sent is the one the kernel gives for the connecting process, and the answer comes back as it was. -func TestTheAccountAskingIsTheKernelsAndTheAnswerComesBack(t *testing.T) { - var got Asked - socket := serving(t, func(_ context.Context, asked Asked) cli.Answer { - got = asked - return cli.Answer{Stdout: []byte("done\n"), Exit: 0, Terminal: true, Why: "the terminal"} - }) - a, err := cli.Ask(context.Background(), socket, []string{"settings", "show", "zsh"}) - if err != nil { - t.Fatal(err) - } - if string(a.Stdout) != "done\n" || !a.Terminal || a.Refused != "" { - t.Fatalf("the answer came back as %+v", a) - } - me, err := user.Current() - if err != nil { - t.Skipf("this machine cannot name the test's account: %v", err) - } - if int(got.UID) != os.Getuid() || got.Account != me.Username { - t.Fatalf("asked as %d %q, and the kernel says %d %q", got.UID, got.Account, os.Getuid(), me.Username) - } - if strings.Join(got.Line, " ") != "settings show zsh" { - t.Fatalf("the line was sent as %q", got.Line) - } - if info, err := os.Stat(socket); err != nil || info.Mode().Perm() != 0o666 { - t.Fatalf("the socket is not open to every account: %v %v", info.Mode(), err) - } -} - -// Credentials the kernel did not give are no account: refused, and nothing is asked. -func TestAConnectionWhoseAccountCannotBeReadIsNeverAskedAsAnyone(t *testing.T) { - before := peerOf - peerOf = func(*net.UnixConn) (uint32, int32, error) { return 0, 0, errors.New("no credentials") } - t.Cleanup(func() { peerOf = before }) - asked := false - socket := serving(t, func(context.Context, Asked) cli.Answer { - asked = true - return cli.Answer{} - }) - a, err := cli.Ask(context.Background(), socket, []string{"status"}) - if err != nil { - t.Fatal(err) - } - if asked || a.Refused == "" || a.Exit == 0 { - t.Fatalf("a connection with no account was asked (%v) or not refused: %+v", asked, a) - } -} - -// A request over the bound is refused and never asked. -func TestARequestOverTheBoundIsRefused(t *testing.T) { - asked := false - socket := serving(t, func(context.Context, Asked) cli.Answer { - asked = true - return cli.Answer{} - }) - a, err := cli.Ask(context.Background(), socket, []string{strings.Repeat("x", cli.MaxRequest)}) - if err != nil { - t.Fatal(err) - } - if asked || a.Refused == "" { - t.Fatalf("a request over the bound was asked (%v): %+v", asked, a) - } -} - -// A file at the socket's path that is not a socket is never removed. -func TestAFileThatIsNotASocketIsKept(t *testing.T) { - dir := t.TempDir() - socket := filepath.Join(dir, "cli.sock") - if err := os.WriteFile(socket, []byte("mine"), 0o644); err != nil { - t.Fatal(err) - } - err := Serve(context.Background(), socket, nil, nil) - if err == nil || !strings.Contains(err.Error(), "not a socket") { - t.Fatalf("a file at the socket's path was not refused: %v", err) - } - if body, _ := os.ReadFile(socket); string(body) != "mine" { - t.Fatal("the file at the socket's path was touched") - } -} - -// The request the controller reads holds these field names; the controller's side holds the same list -// (mesh-controller, the mesh-cli request's test). -func TestTheRequestToTheControllerKeepsItsFieldNames(t *testing.T) { - body, err := json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1}) - if err != nil { - t.Fatal(err) - } - body, _ = json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1, Session: "session-3.scope"}) - if got := keysOf(t, body); got != "account line session uid" { - t.Fatalf("the request's fields are %q", got) - } - body, _ = json.Marshal(Asked{Line: []string{"status"}, Account: "a", UID: 1}) - if got := keysOf(t, body); got != "account line uid" { - t.Fatalf("the request's fields are %q", got) - } - body, _ = json.Marshal(Asked{Line: []string{"secret"}, Account: "a", UID: 1, Stdin: []byte("x")}) - if got := keysOf(t, body); got != "account line stdin uid" { - t.Fatalf("a request with standard input has the fields %q", got) - } - body, _ = json.Marshal(Asked{Follow: "call-1", Account: "a", UID: 1}) - if got := keysOf(t, body); got != "account follow uid" { - t.Fatalf("a follow's fields are %q", got) - } - // What the controller answers as a line's result, and as a running line's: the same names its side holds. - body, err = json.Marshal(cli.Answer{Stdout: []byte("o"), Stderr: []byte("e"), Exit: 1, Terminal: true, Why: "w", - Refused: "r", Cut: true}) - if err != nil { - t.Fatal(err) - } - if got := keysOf(t, body); got != "cut exit refused stderr stdout terminal why" { - t.Fatalf("the answer's fields are %q", got) - } - body, _ = json.Marshal(runningResult{Running: true, Call: "call-1"}) - if got := keysOf(t, body); got != "call running" { - t.Fatalf("a running answer's fields are %q", got) - } -} - -func keysOf(t *testing.T, body []byte) string { - t.Helper() - var m map[string]any - if err := json.Unmarshal(body, &m); err != nil { - t.Fatal(err) - } - var keys []string - for k := range m { - keys = append(keys, k) - } - sort.Strings(keys) - return strings.Join(keys, " ") -} - -// holdALine asks one line and returns once the engine is asking it, so it holds its account's slot. A first line -// that ends or never arrives fails the test at once, with what came of it, instead of hanging the suite (novox/hq -// issue 434). -func holdALine(t *testing.T, socket string, started <-chan struct{}) { - t.Helper() - first := make(chan error, 1) - go func() { - _, err := cli.Ask(context.Background(), socket, []string{"push"}) - first <- err - }() - select { - case <-started: - case err := <-first: - t.Fatalf("the first line ended before it was asked: %v", err) - case <-time.After(5 * time.Second): - t.Fatal("the first line was never asked") - } -} - -// **One account cannot hold the socket** (review of ADR 0272): its lines beyond its own bound are answered busy at -// once, nothing is asked, and the engine goes on accepting. -func TestOneAccountsLinesBeyondItsBoundAreAnsweredBusy(t *testing.T) { - was := perAccount - perAccount = 1 - t.Cleanup(func() { perAccount = was }) - release := make(chan struct{}) - started := make(chan struct{}, 2) - socket := serving(t, func(context.Context, Asked) cli.Answer { - started <- struct{}{} - <-release - return cli.Answer{} - }) - // Released before the engine is stopped: cleanups run last first, and the engine waits for its lines. - t.Cleanup(func() { close(release) }) - holdALine(t, socket, started) - type asked struct { - a cli.Answer - err error - } - done := make(chan asked, 1) - go func() { - a, err := cli.Ask(context.Background(), socket, []string{"status"}) - done <- asked{a, err} - }() - select { - case got := <-done: - // An error is a busy line the client never read (novox/hq issue 434), not an answer. - if got.err != nil || !strings.Contains(got.a.Refused, "busy") { - t.Fatalf("a second line from the same account reads %+v (%v)", got.a, got.err) - } - case <-time.After(5 * time.Second): - t.Fatal("a second line from the same account waited instead of being answered busy") - } - if len(started) != 0 { - t.Fatal("the busy line was asked") - } -} - -// **A refusal waits for the request it answers** (novox/hq issue 434): the client writes its line first and reads -// the answer after. An engine that answered busy and closed before that write landed broke the client's write, and -// the refusal was never read — under the whole suite's load the busy line read as an error. Here the client -// connects and holds its line back: no answer may come before the line, and the line, once written, is answered -// busy. Ordered by the client's own write, so it holds however slow the machine is. -func TestARefusalWaitsForTheRequestItAnswers(t *testing.T) { - was, wasWithin := perAccount, refusedWithin - perAccount, refusedWithin = 1, time.Minute - t.Cleanup(func() { perAccount, refusedWithin = was, wasWithin }) - release := make(chan struct{}) - started := make(chan struct{}, 2) - socket := serving(t, func(context.Context, Asked) cli.Answer { - started <- struct{}{} - <-release - return cli.Answer{} - }) - t.Cleanup(func() { close(release) }) - holdALine(t, socket, started) - conn, err := net.Dial("unix", socket) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - // Nothing is written yet: an engine that answers now answers before the request. The wait is far inside - // refusedWithin, so an engine that waits is still waiting when it ends. - _ = conn.SetReadDeadline(time.Now().Add(200 * time.Millisecond)) - var b [1]byte - if n, err := conn.Read(b[:]); n > 0 || !errors.Is(err, os.ErrDeadlineExceeded) { - t.Fatalf("the engine answered before the request was written (%d bytes, %v)", n, err) - } - _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) - if err := cli.WriteRequest(conn, cli.Request{Line: []string{"status"}}); err != nil { - t.Fatalf("the request could not be written: %v", err) - } - a, err := cli.ReadAnswer(conn) - if err != nil || !strings.Contains(a.Refused, "busy") { - t.Fatalf("a line written after the engine chose busy reads %+v (%v)", a, err) - } - if len(started) != 0 { - t.Fatal("the busy line was asked") - } -} - -// **Refusals that wait are bounded** (novox/hq issue 434, review of ADR 0272): with refusalsWaiting already full, a -// further refused line is answered at once, without waiting for its request. -func TestARefusalOverTheWaitingBoundIsAnsweredAtOnce(t *testing.T) { - was, wasWithin, wasWaiting := perAccount, refusedWithin, refusalsWaiting - perAccount, refusedWithin = 1, time.Minute - refusalsWaiting = make(chan struct{}, 1) - refusalsWaiting <- struct{}{} - t.Cleanup(func() { perAccount, refusedWithin, refusalsWaiting = was, wasWithin, wasWaiting }) - release := make(chan struct{}) - started := make(chan struct{}, 2) - socket := serving(t, func(context.Context, Asked) cli.Answer { - started <- struct{}{} - <-release - return cli.Answer{} - }) - t.Cleanup(func() { close(release) }) - holdALine(t, socket, started) - conn, err := net.Dial("unix", socket) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - // Nothing is written: a refusal that waited would wait refusedWithin, a minute, for it. - _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) - a, err := cli.ReadAnswer(conn) - if err != nil || !strings.Contains(a.Refused, "busy") { - t.Fatalf("a refusal over the waiting bound reads %+v (%v)", a, err) - } -} - -// countingReader is an endless request with no line end, counting what was read of it. -type countingReader struct{ read int } - -func (c *countingReader) Read(p []byte) (int, error) { - for i := range p { - p[i] = 'x' - } - c.read += len(p) - return len(p), nil -} - -// **A refused request is drained, not kept** (novox/hq issue 434): what a refused client writes is read up to -// drainCap bytes and no further, however much it sends. -func TestARefusedRequestIsDrainedNoFurtherThanItsCap(t *testing.T) { - r := &countingReader{} - drain(r) - if r.read != drainCap { - t.Fatalf("drained %d bytes of an endless request; the cap is %d", r.read, drainCap) - } - after := &countingReader{} - drain(io.MultiReader(strings.NewReader("{\"line\":[\"status\"]}\n"), after)) - if after.read != 0 { - t.Fatalf("draining read %d bytes past the end of the request line", after.read) - } -} - -// **A process in another namespace is not this machine's account** (review of ADR 0272): a container running with -// the operator's user id shares the id and not the machine; refused, and nothing is asked. -func TestAPeerInAnotherNamespaceIsRefused(t *testing.T) { - before := sameNamespaces - sameNamespaces = func(pid int32) error { return errors.New("its mount namespace is not the engine's") } - t.Cleanup(func() { sameNamespaces = before }) - asked := false - socket := serving(t, func(context.Context, Asked) cli.Answer { - asked = true - return cli.Answer{} - }) - a, err := cli.Ask(context.Background(), socket, []string{"status"}) - if err != nil { - t.Fatal(err) - } - if asked || !strings.Contains(a.Refused, "namespace") { - t.Fatalf("a peer in another namespace was asked (%v): %+v", asked, a) - } -} - -// **The session a line comes from is the kernel's word, and only a login session counts** (review of ADR 0272): a -// service running as the operator — the tool runner, a user unit — is not a person at a terminal. The engine reads -// the peer's cgroup and sends the login session's scope, or nothing; the controller makes no other terminal. -func TestTheLoginSessionAskingIsReadFromTheKernel(t *testing.T) { - for _, c := range []struct{ cgroup, session string }{ - {"0::/user.slice/user-1000.slice/session-3.scope\n", "session-3.scope"}, - {"0::/user.slice/user-1000.slice/session-c1.scope\n", "session-c1.scope"}, - {"0::/user.slice/user-1000.slice/user@1000.service/app.slice/tmux.scope\n", ""}, - {"0::/system.slice/node-tools.service\n", ""}, - {"0::/user.slice/user-1001.slice/session-3.scope\n", ""}, // another account's session - {"12:pids:/user.slice\n0::/user.slice/user-1000.slice/session-9.scope/sub\n", ""}, - // Below a session scope, in a scope of its own: only the check that the scope is the account slice's direct - // child refuses it. - {"0::/user.slice/user-1000.slice/session-9.scope/x.scope\n", ""}, - {"", ""}, - } { - if got := loginSessionIn(c.cgroup, 1000); got != c.session { - t.Errorf("%q read as %q, want %q", c.cgroup, got, c.session) - } - } - // And it is sent: the test process's own, whatever it is, as the kernel says it. - var got Asked - socket := serving(t, func(_ context.Context, asked Asked) cli.Answer { - got = asked - return cli.Answer{} - }) - if _, err := cli.Ask(context.Background(), socket, []string{"status"}); err != nil { - t.Fatal(err) - } - raw, _ := os.ReadFile("/proc/self/cgroup") - if want := userSessionIn(string(raw), uint32(os.Getuid())); got.Session != want { - t.Fatalf("sent session %q, the kernel says %q", got.Session, want) - } -} - -// **A login session is a person's: systemd's own record says so** (review of ADR 0272): the scope the cgroup names -// must be a session logind keeps for this very account, of class `user` — not a greeter's, not the user manager's, -// not one whose record names another account — and read at accept, with the namespaces, before anything else. -func TestALoginSessionIsAUsersByItsOwnRecord(t *testing.T) { - dir := t.TempDir() - was := sessionsDir - sessionsDir = dir - t.Cleanup(func() { sessionsDir = was }) - write := func(id, body string) { - if err := os.WriteFile(filepath.Join(dir, id), []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - write("9", "UID=1000\nUSER=op\nCLASS=user\nSCOPE=session-9.scope\nACTIVE=1\n") - write("3", "UID=1000\nCLASS=greeter\nSCOPE=session-3.scope\n") - write("5", "UID=1000\nCLASS=user\nSCOPE=session-6.scope\n") - // The account the record names is not the slice's: only the record's UID check refuses it. - write("8", "UID=1001\nCLASS=user\nSCOPE=session-8.scope\n") - // A name with a dot, held as a record that would pass: only the check of the id's characters refuses it. - write("a.b", "UID=1000\nCLASS=user\nSCOPE=session-a.b.scope\n") - // A key said twice: the first is the record's word (systemd writes each once); a later one is not read. - write("11", "UID=1001\nCLASS=user\nSCOPE=session-11.scope\nUID=1000\n") - write("12", "UID=1000\nCLASS=user\nSCOPE=session-12.scope\nCLASS=greeter\nUID=1001\n") - for _, c := range []struct { - cgroup string - uid uint32 - session string - }{ - {"0::/user.slice/user-1000.slice/session-9.scope\n", 1000, "session-9.scope"}, - {"0::/user.slice/user-1000.slice/x.scope\n", 1000, ""}, - {"0::/user.slice/user-1000.slice/session-9.scope\n", 1001, ""}, // asked as another uid than the slice's - {"0::/user.slice/user-1000.slice/session-8.scope\n", 1000, ""}, // the record names uid 1001 - {"0::/user.slice/user-1000.slice/session-3.scope\n", 1000, ""}, // a greeter's - {"0::/user.slice/user-1000.slice/session-5.scope\n", 1000, ""}, // a record naming another scope - {"0::/user.slice/user-1000.slice/session-7.scope\n", 1000, ""}, // no record at all - {"0::/user.slice/user-1000.slice/session-a.b.scope\n", 1000, ""}, - {"0::/user.slice/user-1000.slice/session-...scope\n", 1000, ""}, - {"0::/user.slice/user-1000.slice/session-11.scope\n", 1000, ""}, // first UID is 1001 - {"0::/user.slice/user-1000.slice/session-12.scope\n", 1000, "session-12.scope"}, // first UID and CLASS stand - } { - if got := userSessionIn(c.cgroup, c.uid); got != c.session { - t.Errorf("%q as uid %d read as %q, want %q", c.cgroup, c.uid, got, c.session) - } - } -} - -// **The session is read at accept, with the namespaces, before the line** (review of ADR 0272): the peer is judged as -// it was when it connected. The client sends its line only once the session has been read; were the session read -// after the line — in the forward, say — the line would never be sent and the test fails. -func TestTheSessionIsReadBeforeTheLine(t *testing.T) { - var mu sync.Mutex - var order []string - note := func(s string) { mu.Lock(); order = append(order, s); mu.Unlock() } - read := make(chan struct{}, 1) - beforeNS, beforeSession := sameNamespaces, sessionOf - sameNamespaces = func(int32) error { note("namespaces"); return nil } - sessionOf = func(int32, uint32) string { note("session"); read <- struct{}{}; return "session-1.scope" } - t.Cleanup(func() { sameNamespaces, sessionOf = beforeNS, beforeSession }) - var got Asked - socket := serving(t, func(_ context.Context, asked Asked) cli.Answer { - note("forward") - got = asked - return cli.Answer{} - }) - conn, err := net.Dial("unix", socket) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - select { - case <-read: - case <-time.After(3 * time.Second): - t.Fatal("the session was not read before the line was sent") - } - note("line sent") - if err := cli.WriteRequest(conn, cli.Request{Line: []string{"status"}}); err != nil { - t.Fatal(err) - } - _ = conn.SetDeadline(time.Now().Add(5 * time.Second)) - if _, err := cli.ReadAnswer(conn); err != nil { - t.Fatal(err) - } - mu.Lock() - defer mu.Unlock() - if strings.Join(order, ",") != "namespaces,session,line sent,forward" || got.Session != "session-1.scope" { - t.Fatalf("read in the order %v, sent %q", order, got.Session) - } -} - -// mesh-cli's standard input reaches the controller with its line, whole, and only with the line: a follow of the -// same call carries none, and the engine says it nowhere (novox/hq ADR 0259 §10, ADR 0272). -func TestStandardInputGoesToTheControllerWithTheLineAndNowhereElse(t *testing.T) { - secret := []byte("123456789:the-bot-token-given-at-the-terminal\n") - var said []string - var mu sync.Mutex - var asked [][]byte - ask := func(_ context.Context, body []byte) ([]byte, error) { - mu.Lock() - defer mu.Unlock() - asked = append(asked, append([]byte(nil), body...)) - if len(asked) == 1 { - return []byte(`{"result":{"running":true,"call":"call-7"}}`), nil - } - return []byte(`{"result":{"exit":0,"terminal":true}}`), nil - } - was := followEvery - followEvery = time.Millisecond - t.Cleanup(func() { followEvery = was }) - socket := filepath.Join(t.TempDir(), "run", "cli.sock") - ctx, cancel := context.WithCancel(context.Background()) - done := make(chan error, 1) - go func() { - done <- Serve(ctx, socket, func(ctx context.Context, a Asked) cli.Answer { return AskTheController(ctx, ask, a) }, - func(s string) { mu.Lock(); said = append(said, s); mu.Unlock() }) - }() - defer func() { cancel(); <-done }() - for i := 0; i < 200; i++ { - if _, err := os.Stat(socket); err == nil { - break - } - time.Sleep(5 * time.Millisecond) - } - a, err := cli.AskWith(context.Background(), socket, cli.Request{Line: []string{"secret", "accept", "anchor", - "telegram", "telegram-token", "--from", "-"}, Stdin: secret}) - if err != nil || a.Exit != 0 { - t.Fatalf("%v %+v", err, a) - } - mu.Lock() - defer mu.Unlock() - if len(asked) != 2 { - t.Fatalf("asked %d times, want the line and one follow", len(asked)) - } - var first, follow Asked - _ = json.Unmarshal(asked[0], &first) - _ = json.Unmarshal(asked[1], &follow) - if string(first.Stdin) != string(secret) { - t.Fatalf("the controller was not given the standard input whole: %q", first.Stdin) - } - if len(follow.Stdin) != 0 || strings.Contains(string(asked[1]), "stdin") { - t.Fatalf("a follow carried the standard input again: %s", asked[1]) - } - for _, s := range said { - if strings.Contains(s, "bot-token") { - t.Fatalf("the engine said the standard input: %s", s) - } - } -} diff --git a/internal/meshcli/peer.go b/internal/meshcli/peer.go deleted file mode 100644 index 88397cd..0000000 --- a/internal/meshcli/peer.go +++ /dev/null @@ -1,19 +0,0 @@ -package meshcli - -import "net" - -// The kernel's word on who is at the other end of a socket, shared with the node-engine's other local socket, the -// write of a filled file (internal/filling, novox/hq ADR 0349 §4): one reading of it, so the two never judge a peer -// differently. - -// PeerCredentials is the user id and process id of the process at the other end, as the kernel recorded them when -// it connected (SO_PEERCRED). Nothing the process says is read. -func PeerCredentials(conn *net.UnixConn) (uint32, int32, error) { return peerCredentials(conn) } - -// SameNamespaces refuses a process whose mount or user namespace is not the engine's: a user id is this machine's -// account only in this machine's namespaces. -func SameNamespaces(pid int32) error { return namespacesOf(pid) } - -// NotOwnDirectory says what is wrong with a socket's directory, or nothing: it must be a directory, not a link, -// owned by the account the engine runs as. -func NotOwnDirectory(dir string) string { return notOwnDirectory(dir) } diff --git a/internal/ownresync/ownresync.go b/internal/ownresync/ownresync.go deleted file mode 100644 index dbae480..0000000 --- a/internal/ownresync/ownresync.go +++ /dev/null @@ -1,319 +0,0 @@ -// Package ownresync hands the vault the value of this machine's own secret that runs, where the vault holds another -// (novox/hq issue 1126, cause C4). -// -// The vault's `move-in` never replaces a value it holds, so its first import of an own secret stays fixed; where the -// controller later made that secret anew (a rotation at start, a re-issue), the new value runs and the vault keeps the -// old one. The node's shadow composer sees it: composing from the vault's value, it opens both seals with this -// machine's key and its answer's shown body says delivery.MaskedAnotherValue. After each shadow answer this step asks the -// vault's `move-in` with replace_running: true and what runs, for each own secret so masked, once per running value, and -// says what it asked and what the vault answered. It is move-in's, the verb this node may ask already, so no seat and no -// bus grant changes for it. -// -// **Only while this node applies the controller's compositions**, and only for a module whose composition that runs is -// the one its want names: then the controller's value is what runs, and what runs is the truth the vault lacks. Once the -// node composes its own, the vault's value is the source and nothing is handed. **The switch** is the vault's -// (`own_take_switch`, a person's act): while it is off the vault refuses every ask, and this step says so once per value. -// The vault also refuses any value it changed in the last 15 minutes, so a re-issue the node has not applied yet is -// never put back. -package ownresync - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "sort" - "sync" - - "git.novox.be/novox/mesh-sdk/go/catalogue" - "git.novox.be/novox/mesh-sdk/go/delivery" - - "github.com/novox/mesh-host/internal/selfcompose" -) - -// Verb is the vault's verb this step asks, with ReplaceRunning true in both steps. -const Verb = "move-in" - -// ReplaceRunning is move-in's argument that makes it take the running value over the one the vault holds. -const ReplaceRunning = "replace_running" - -// Env is what the step reads and asks. -type Env struct { - Node string - // Composes is whose compositions the node applies now: delivery.ComposesController, delivery.ComposesOwn, or "" - // while its agent has not decided. The step hands nothing but for ComposesController. - Composes func() string - // Running is what the node runs now (agent.Everything). - Running func() []delivery.Composed - // Get reads one key's latest value; nil and no error when there is none. - Get func(ctx context.Context, bucket, key string) ([]byte, error) - // Unseal opens a value sealed to this machine; Seal seals a value to a raw X25519 key in base64 (identity.Seal). - Unseal func(sealed string) ([]byte, error) - Seal func(to string, value []byte) (string, error) - // Ask asks one of the vault's verbs in this engine's own name (link.Session.AskVault). - Ask func(ctx context.Context, verb string, args map[string]any) (json.RawMessage, error) - Say func(string) -} - -// Outcome is what the step did for one own secret: asked or not, and what was said. -type Outcome struct { - Module string - Name string - Asked bool - Said string - // Retry says a read or an ask the bus did not answer stopped it: no answer, kept as none, and the step is to be - // taken again after a bound (issue 1126, 2026-10-28: "want.shanks.brother could not be read ... no responders"). - Retry bool -} - -// Resync is the step of one node agent's run. -type Resync struct { - env Env - - mu sync.Mutex - // done is, by module/name, the running value's fingerprint the vault was asked with or why it was not: each is - // asked, and each reason said, once. - done map[string]string -} - -// New is the step for one node. -func New(env Env) *Resync { - if env.Say == nil { - env.Say = func(string) {} - } - return &Resync{env: env, done: map[string]string{}} -} - -// Take reads the node's shadow answers and hands the vault what runs of each own secret they say another value of. -func (r *Resync) Take(ctx context.Context, answers map[string]delivery.CandidateAnswer) []Outcome { - seen := map[delivery.OwnValueDiffers]bool{} - var drift []delivery.OwnValueDiffers - proposals := make([]string, 0, len(answers)) - for g := range answers { - proposals = append(proposals, g) - } - sort.Strings(proposals) - for _, g := range proposals { - for _, o := range delivery.OwnValuesDiffer(answers[g]) { - if !seen[o] { - seen[o] = true - drift = append(drift, o) - } - } - } - if len(drift) == 0 { - return nil - } - composes := "" - if r.env.Composes != nil { - composes = r.env.Composes() - } - var out []Outcome - for _, o := range drift { - res := r.one(ctx, composes, o) - if res.Said != "" || res.Retry { - out = append(out, res) - } - } - return out -} - -// one is the step for one own secret: Said is empty when it was done before for the same value and reason. -func (r *Resync) one(ctx context.Context, composes string, o delivery.OwnValueDiffers) Outcome { - res := Outcome{Module: o.Module, Name: o.Name} - key := o.Module + "/" + o.Name - if composes == "" { - return res // the agent has not decided whose compositions it applies: taken again once it has - } - if composes != delivery.ComposesController { - return r.said(key, "not:composes", res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+ - "nothing is handed: this node composes its own (%q), so the vault's value is the source", key, composes)) - } - sealed, why, failed := r.running(ctx, o) - if failed != nil { - // The bus did not answer: no reason to keep, and taken again (Retry); said once per reason. - res.Retry = true - return r.said(key, "retry:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+ - "nothing is handed yet: %s; asked again", key, why)) - } - if why != "" { - return r.said(key, "not:"+why, res, fmt.Sprintf("%s holds another value in the vault than runs here, and "+ - "nothing is handed: %s", key, why)) - } - sum := sha256.Sum256([]byte(sealed)) - fp := hex.EncodeToString(sum[:]) - r.mu.Lock() - asked := r.done[key] == fp - r.mu.Unlock() - if asked { - return res - } - value, err := r.env.Unseal(sealed) - if err != nil { - return r.said(key, "not:unseal:"+fp, res, fmt.Sprintf("%s: what runs does not open with this machine's key, so "+ - "nothing is handed: %v", key, err)) - } - defer wipe(value) - said, err := r.hand(ctx, o, value) - if err != nil { - if ctx.Err() != nil { - return res // the run ended: asked again in the next - } - if selfcompose.BusFailed(err) { - // The vault did not answer on the bus: no word of the vault's, so not kept as asked (Retry). - res.Retry = true - return r.said(key, "retry:ask", res, fmt.Sprintf("%s: the vault's %s with replace_running did not answer, "+ - "and is asked again: %v", key, Verb, err)) - } - said = fmt.Sprintf("%s: the vault's %s with replace_running refused: %v", key, Verb, err) - } - // Once per running value, whatever the vault answered: a refusal is the vault's word, said once, and a new value - // that runs asks again. - r.mu.Lock() - r.done[key] = fp - r.mu.Unlock() - res.Asked, res.Said = true, said - r.env.Say(said) - return res -} - -// said says why once per reason, and keeps the reason as done. -func (r *Resync) said(key, reason string, res Outcome, why string) Outcome { - r.mu.Lock() - again := r.done[key] == reason - r.done[key] = reason - r.mu.Unlock() - if again { - return res - } - res.Said = why - r.env.Say(why) - return res -} - -// running is the sealed value of o in the composition that runs, when that composition is the one the module's want -// names; or why it is not handed, with the error of a read the bus did not answer, which is no answer. -func (r *Resync) running(ctx context.Context, o delivery.OwnValueDiffers) (sealed, why string, failed error) { - var comp *delivery.Composed - if r.env.Running != nil { - for _, c := range r.env.Running() { - if c.Module == o.Module { - c := c - comp = &c - break - } - } - } - if comp == nil { - return "", o.Module + " runs no composition here", nil - } - wantKey, err := delivery.WantKey(r.env.Node, o.Module) - if err != nil { - return "", err.Error(), nil - } - raw, err := r.env.Get(ctx, delivery.AssignerWantsBucket, wantKey) - if err != nil { - return "", fmt.Sprintf("its want %s could not be read (%v)", wantKey, err), err - } - if raw == nil { - return "", fmt.Sprintf("it has no want %s", wantKey), nil - } - want, err := delivery.DecodeWant(raw) - if err != nil { - return "", fmt.Sprintf("its want %s is not one: %v", wantKey, err), nil - } - if want.Self || want.Removed || want.Composed == "" { - return "", fmt.Sprintf("its want %s names no composition of the controller's", wantKey), nil - } - // What runs is the composition the want names, as the module's have says: applied and running. - haveKey, err := delivery.HaveKey(r.env.Node, o.Module) - if err != nil { - return "", err.Error(), nil - } - raw, err = r.env.Get(ctx, delivery.HaveBucket, haveKey) - if err != nil { - return "", fmt.Sprintf("its have %s could not be read (%v)", haveKey, err), err - } - if raw == nil { - return "", fmt.Sprintf("it has no have %s", haveKey), nil - } - have, err := delivery.DecodeHave(raw) - if err != nil { - return "", fmt.Sprintf("its have %s is not one: %v", haveKey, err), nil - } - if have.Composed != want.Composed || have.Phase != delivery.PhaseRunning { - return "", fmt.Sprintf("what runs (%s, %s) is not yet the composition its want names (%s)", have.Composed, - have.Phase, want.Composed), nil - } - // On a node the resource's id carries its module's name (`only-office.needs-jwt`); delivery.OwnNeedName reads it. - id := o.Module + "." + catalogue.NeedID(o.Name) - for _, res := range comp.Resources { - var body struct { - ID string `json:"id"` - Sealed any `json:"sealed"` - } - if json.Unmarshal(res, &body) != nil { - continue - } - if name, own := delivery.OwnNeedName(o.Module, body.ID); !own || name != o.Name { - continue - } - if s, ok := body.Sealed.(string); ok && s != "" { - return s, "", nil - } - return "", fmt.Sprintf("its resource %s carries no value sealed to this machine", id), nil - } - return "", fmt.Sprintf("what runs holds no resource %s", id), nil -} - -// hand asks the vault's two steps with value, and checks what the vault holds after by its fingerprint. -func (r *Resync) hand(ctx context.Context, o delivery.OwnValueDiffers, value []byte) (string, error) { - raw, err := r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "module": o.Module, "name": o.Name, - "why": "this node's shadow answer says the vault holds another value than runs here"}) - if err != nil { - return "", err - } - var opened struct { - Call string `json:"call"` - SealTo string `json:"seal_to"` - Secret string `json:"secret"` - } - if err := json.Unmarshal(raw, &opened); err != nil || opened.Call == "" || opened.SealTo == "" { - return "", fmt.Errorf("the vault answered no call and no key to seal to") - } - sealed, err := r.env.Seal(opened.SealTo, value) - if err != nil { - return "", fmt.Errorf("the vault's key for the call is not a sealing key: %w", err) - } - raw, err = r.env.Ask(ctx, Verb, map[string]any{ReplaceRunning: true, "call": opened.Call, "sealed": sealed}) - if err != nil { - return "", err - } - var kept struct { - Secret string `json:"secret"` - From int `json:"from_generation"` - To int `json:"to_generation"` - Unchanged bool `json:"unchanged"` - Fingerprint string `json:"fingerprint"` - Said string `json:"said"` - } - if err := json.Unmarshal(raw, &kept); err != nil || kept.Secret == "" { - return "", fmt.Errorf("the vault answered %s with no secret", opened.Secret) - } - sum := sha256.Sum256(value) - if kept.Fingerprint != "sha256:"+hex.EncodeToString(sum[:]) { - return "", fmt.Errorf("the vault says it holds %s at generation %d, but not the value handed in", kept.Secret, kept.To) - } - if kept.Unchanged { - return fmt.Sprintf("handed the vault what runs of %s: it held that value already (generation %d)", kept.Secret, kept.To), nil - } - return fmt.Sprintf("handed the vault what runs of %s: generation %d replaced by %d, the value that runs", kept.Secret, - kept.From, kept.To), nil -} - -func wipe(b []byte) { - for i := range b { - b[i] = 0 - } -} diff --git a/internal/ownresync/ownresync_test.go b/internal/ownresync/ownresync_test.go deleted file mode 100644 index d4351c2..0000000 --- a/internal/ownresync/ownresync_test.go +++ /dev/null @@ -1,117 +0,0 @@ -package ownresync - -import ( - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "errors" - "strings" - "testing" - - "git.novox.be/novox/mesh-sdk/go/delivery" -) - -// Issue 1126, cause C4: after a shadow answer saying the vault holds another value of an own secret than runs, the node -// hands the vault what runs, in the verb's two steps, once per running value; and only while it applies the -// controller's compositions and runs the one its want names. -func TestTheNodeHandsTheVaultWhatRunsOfADriftedOwnSecret(t *testing.T) { - const running = "the-value-that-runs" - digest := "sha256:" + strings.Repeat("a", 64) - fp := "src2:" + strings.Repeat("b", 64) - sum := sha256.Sum256([]byte(running)) - answers := map[string]delivery.CandidateAnswer{"g": {Proposal: "p-1", Revision: 1, Verdict: delivery.CandidateDiffers, - Differs: []delivery.ModuleDiffers{{Module: "only-office", Why: "the resource differs", Bodies: []delivery.ResourceBodies{ - {ID: "only-office.needs-jwt", Composed: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`, Running: `{"sealed":"` + delivery.MaskedAnotherValue + `"}`}, - {ID: "only-office.needs-salt", Composed: `{"sealed":""}`}}}}}} - for name, row := range map[string]struct { - composes string - runs string // the have's composition digest - refuse string // the vault's refusal - asks int // the vault's verb asked, both passes - says string - // Issue 1126, 2026-10-28: a read or an ask the bus did not answer is no answer: taken again (Retry), never kept. - wantRead string // the want's read: "" answers it, "fails" the bus does not answer, "absent" no key - retry bool - }{ - "applies the controller's, runs its want": {delivery.ComposesController, digest, "", 2, "generation 1 replaced by 2", "", false}, - "composes its own": {delivery.ComposesOwn, digest, "", 0, "composes its own", "", false}, - "not decided yet": {"", digest, "", 0, "", "", false}, - "runs not yet what its want names": {delivery.ComposesController, "sha256:" + strings.Repeat("c", 64), "", 0, "not yet the composition", "", false}, - "the vault refuses": {delivery.ComposesController, digest, "switched off", 1, "switched off", "", false}, - "the vault does not answer": {delivery.ComposesController, digest, "nats: no responders available for request", 2, "asked again", "", true}, - "the want's read does not answer": {delivery.ComposesController, digest, "", 0, "asked again", "fails", true}, - "it has no want": {delivery.ComposesController, digest, "", 0, "it has no want", "absent", false}, - } { - var asked []map[string]any - var said []string - want, _ := delivery.EncodeWant(delivery.Want{FP: fp, Composed: digest}) - have, _ := delivery.EncodeHave(delivery.Have{FP: fp, Composed: row.runs, Phase: delivery.PhaseRunning, Health: delivery.HealthOK}) - r := New(Env{Node: "novox", Composes: func() string { return row.composes }, - Running: func() []delivery.Composed { - return []delivery.Composed{{Node: "novox", Module: "only-office", FP: fp, Resources: []json.RawMessage{ - // As novox runs it (its shadow answer, 2026-10-28): the id carries the module's name. - json.RawMessage(`{"id":"only-office.needs-jwt","module":"only-office","type":"file",` + - `"path":"/var/lib/only-office/jwt.secret","sealed":"box-of-the-running-value"}`)}}} - }, - Get: func(_ context.Context, bucket, key string) ([]byte, error) { - switch { - case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "fails": - return nil, errors.New("the bucket mesh-delivery-assigner_wants: nats: no responders available for request") - case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office" && row.wantRead == "absent": - return nil, nil - case bucket == delivery.AssignerWantsBucket && key == "want.novox.only-office": - return want, nil - case bucket == delivery.HaveBucket && key == "novox.only-office": - return have, nil - } - return nil, nil - }, - Unseal: func(sealed string) ([]byte, error) { - if sealed != "box-of-the-running-value" { - return nil, errors.New("not this machine's") - } - return []byte(running), nil - }, - Seal: func(to string, value []byte) (string, error) { return to + "(" + string(value) + ")", nil }, - Ask: func(_ context.Context, verb string, args map[string]any) (json.RawMessage, error) { - asked = append(asked, args) - if verb != "move-in" || args["replace_running"] != true { - t.Errorf("%s: asked %s with %v, not move-in with replace_running", name, verb, args) - } - if row.refuse != "" { - return nil, errors.New(row.refuse) - } - if args["call"] == nil { - if args["module"] != "only-office" || args["name"] != "jwt" || args["node"] != nil { - t.Errorf("%s: the first step names %v", name, args) - } - return json.RawMessage(`{"call":"c1","seal_to":"key","secret":"own/novox/only-office/jwt"}`), nil - } - if args["sealed"] != "key("+running+")" { - t.Errorf("%s: handed %v, not what runs sealed to the call's key", name, args["sealed"]) - } - return json.RawMessage(`{"secret":"own/novox/only-office/jwt","from_generation":1,"to_generation":2,` + - `"fingerprint":"sha256:` + hex.EncodeToString(sum[:]) + `"}`), nil - }, - Say: func(s string) { said = append(said, s) }}) - first := r.Take(context.Background(), answers) - again := r.Take(context.Background(), answers) - if len(asked) != row.asks { - t.Errorf("%s: the vault asked %d times, want %d: %v", name, len(asked), row.asks, asked) - } - if retry := len(first) == 1 && first[0].Retry; retry != row.retry { - t.Errorf("%s: taken again %v, want %v: %+v", name, retry, row.retry, first) - } - if row.retry && (len(again) != 1 || !again[0].Retry || again[0].Said != "") || !row.retry && len(again) != 0 { - t.Errorf("%s: the same answer did or said something again: %+v", name, again) - } - all := strings.Join(said, "\n") - if row.says == "" && (len(first) != 0 || all != "") || row.says != "" && (len(first) != 1 || !strings.Contains(all, row.says)) { - t.Errorf("%s: did %+v and said %q, want %q", name, first, all, row.says) - } - if strings.Contains(all, running) { - t.Errorf("%s: the value is in what the node said", name) - } - } -} diff --git a/internal/selfcompose/packages.go b/internal/selfcompose/packages.go index b06f61e..38a3a11 100644 --- a/internal/selfcompose/packages.go +++ b/internal/selfcompose/packages.go @@ -15,10 +15,10 @@ import ( // **The packages this node's compositions declare, kept by its engine** (novox/hq ADR 0565 §6, §15; issue 1953). The // composer keeps the package resources of each composition it handed, and after a pass that handed something new (or // the run's first) it works out the node's key of the engine's state packages from them (delivery.DeclaredPackagesOf, -// the controller mirror's layout and order) and writes it only when the value differs from what the key holds (ADR 0289 +// in the layout and order the controller's mirror had) and writes it only when the value differs from what the key holds (ADR 0289 // §1): no timer, and no composing of another node. A module held keeps the packages of what was handed of it, as it keeps -// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it beside the -// controller's mirror until the lease write, and with write writes it again (ADR 0565 §15). +// running what it runs. pacman reads the key as mesh-host.packages; the node seat's verb packages answers it, and with +// write writes it again (ADR 0565 §15). // PackagesStore is this node's key of the engine's state packages (link.EnginePackages). type PackagesStore interface { diff --git a/internal/selfcompose/selfcompose_test.go b/internal/selfcompose/selfcompose_test.go index 802b4d4..d14cde2 100644 --- a/internal/selfcompose/selfcompose_test.go +++ b/internal/selfcompose/selfcompose_test.go @@ -1057,7 +1057,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { go func() { sh.Keep(sctx, func(int) time.Duration { return 5 * time.Millisecond }); close(kept) }() time.Sleep(30 * time.Millisecond) sb.missing(delivery.PlacementCandidatesBucket, false) - sb.made(7) + sb.made(6) rec, _ := delivery.EncodeHaveNode(delivery.HaveNode{Record: &delivery.NodeRecord{Node: "n1", TunnelKey: "k", SealingKey: "sk", Capabilities: []string{"systemd"}}}) tp, _ := delivery.EncodePlacement(trunk) sb.put(delivery.PlacementBucket, "placement.n1", tp, 1) @@ -1321,7 +1321,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { nctx, ncancel := context.WithCancel(ctx) defer ncancel() go func() { _ = nsh.Run(nctx) }() - nb.made(7) + nb.made(6) nb.put(delivery.PlacementBucket, "placement.n1", tp, 1) nb.put(delivery.HaveBucket, "n1.node", rec, 1) for name, m := range mods { @@ -1335,19 +1335,19 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { oldRaw, _ := delivery.EncodePlacementCandidate(delivery.PlacementCandidate{Proposal: "older", Placement: &trunk}) oldKey, _ := delivery.PlacementCandidateKey("older", "n1") nb.put(delivery.PlacementCandidatesBucket, oldKey, oldRaw, 2) - for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, ResolvedBucket, delivery.BuilderBuildsBucket} { + for _, bucket := range []string{delivery.PlacementCandidatesBucket, delivery.PlacementBucket, delivery.HaveBucket, delivery.ModulesBucket, delivery.BuilderBuildsBucket} { nb.send(bucket, agent.Update{Caught: true}) } select { case a := <-nanswered: - if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, delivery.AssignerWantsBucket) || g.Proposal != "same" { - t.Errorf("a shadow waiting for its wants answered %+v", g) + if g := a["same"]; g.Verdict != delivery.CandidateRefused || !strings.Contains(g.Refused, ResolvedBucket) || g.Proposal != "same" { + t.Errorf("a shadow waiting for its values answered %+v", g) } case <-time.After(5 * time.Second): t.Fatal("a shadow not ready after its wait answered nothing") } nmu.Lock() - if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+delivery.AssignerWantsBucket) { + if !strings.Contains(strings.Join(nsaid, "\n"), "this node answers no placement candidate yet: the node has not read "+ResolvedBucket) { t.Errorf("a shadow not ready said %q", nsaid) } nmu.Unlock() @@ -1355,7 +1355,7 @@ func TestTheNodeComposesItsWantsFromTheBus(t *testing.T) { scancel() <-kept setFor(t, &slowAfter, time.Nanosecond) // every answer is slow: said, and noted in words that do not move with the time - nb.send(delivery.AssignerWantsBucket, agent.Update{Caught: true}) + nb.send(ResolvedBucket, agent.Update{Caught: true}) select { case a := <-nanswered: if g := a["same"]; g.Verdict != delivery.CandidateSame || !strings.Contains(g.Note, "this answer took over") { diff --git a/internal/selfcompose/shadow.go b/internal/selfcompose/shadow.go index 254699f..bf6c878 100644 --- a/internal/selfcompose/shadow.go +++ b/internal/selfcompose/shadow.go @@ -32,8 +32,6 @@ type Shadow struct { mu sync.Mutex candidates map[string]map[string]candidateKey // by proposal, by node answers map[string]delivery.CandidateAnswer - // wants are the node's wants by module (agent.NodeModule for its own); a change of one answers again. - wants map[string]delivery.Want // ran is, by module, the digest the module's have says runs: what runs, where the agent's list holds only the copy // from the declaration kept before the switch (issue 1126, probe #532 on shanks). ran map[string]string @@ -64,7 +62,7 @@ type answeredAt struct { } // inputWords name the shadow's inputs, in Run's order, as an answer says which of them woke it. -var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves", "wants"} +var inputWords = [...]string{"the candidate", "placement", "records", "modules", "values", "haves"} // wokenBy says what moved between the answer a proposal stands at (had, when done) and the one about to be composed (at): // its first answer, a retry, its candidate's revisions, or the inputs that changed, by their words. @@ -99,7 +97,7 @@ type candidateKey struct { // NewShadow is the shadow composer of one node; answer writes the node's answers into its own have key. func NewShadow(env Env, answer func(ctx context.Context, answers map[string]delivery.CandidateAnswer)) *Shadow { return &Shadow{c: New(env), answer: answer, candidates: map[string]map[string]candidateKey{}, - answers: map[string]delivery.CandidateAnswer{}, wants: map[string]delivery.Want{}, ran: map[string]string{}} + answers: map[string]delivery.CandidateAnswer{}, ran: map[string]string{}} } // Keep runs the shadow until ctx ends, beginning again after backoff(try) whenever it stops: an input it cannot watch @@ -141,7 +139,6 @@ func (s *Shadow) Run(ctx context.Context) error { defer cancel() s.mu.Lock() s.candidates = map[string]map[string]candidateKey{} - s.wants = map[string]delivery.Want{} s.ran = map[string]string{} s.answered = map[string]answeredAt{} s.moved = [len(inputWords)]uint64{} @@ -159,8 +156,6 @@ func (s *Shadow) Run(ctx context.Context) error { {ResolvedBucket, node + ".>", (*Composer).takeValue, nil}, // What this node runs moved: an apply ends in its module's have. {delivery.HaveBucket, node + ".>", nil, nil}, - // Its wants. - {delivery.AssignerWantsBucket, "want." + node + ".>", nil, nil}, } type tagged struct { in int @@ -225,8 +220,6 @@ func (s *Shadow) Run(ctx context.Context) error { case t.in == 5: folded := c.takeModuleHave(t.u) changed = s.takeRan(t.u) || folded - case t.in == 6: - changed = s.takeWant(t.u) default: changed = c.takeChanged(ins[t.in], t.u) } @@ -685,33 +678,6 @@ func sameAnswer(a, b delivery.CandidateAnswer) bool { return errA == nil && errB == nil && string(x) == string(y) } -// takeWant takes one of the node's wants, and says whether it changed. -func (s *Shadow) takeWant(u agent.Update) bool { - module := "" - var w delivery.Want - if _, err := delivery.ParseWantNodeKey(u.Key); err == nil { - module = agent.NodeModule - } else if node, m, err := delivery.ParseWantKey(u.Key); err == nil && node == s.c.env.Node { - module = m - if !u.Deleted { - if w, err = delivery.DecodeWant(u.Value); err != nil || w.Removed || w.FP == "" { - u.Deleted = true - } - } - } else { - return false - } - s.mu.Lock() - defer s.mu.Unlock() - old, had := s.wants[module] - if u.Deleted { - delete(s.wants, module) - return had - } - s.wants[module] = w - return !had || old.FP != w.FP -} - // sameValue says whether two sealed values open, with this node's key, to the same value; unknown when either does not // open (or the node holds no opener). Only the comparison leaves: never a value (issue 1126, the shadow shows its work). func (c *Composer) sameValue(a, b string) (same, known bool) { diff --git a/internal/userlist/userlist.go b/internal/userlist/userlist.go index 668f473..b818729 100644 --- a/internal/userlist/userlist.go +++ b/internal/userlist/userlist.go @@ -13,23 +13,26 @@ // or is deleted is said, the key last heard stands, and the list loaded stays; with no key ever heard nothing is // written. The list a composition carries is never written. // -// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition in the node's -// composition bucket, verified as the agent verified it before anything of it is read: what tells the bus node it is -// one, and the file's place. The have of that module says what was written, which list it follows and this node's key, -// which the grants part seals to. +// **Where it writes.** The file is the user-list resource (`bus-users`) of the bus module's composition as this node runs +// it (Env.Running: the agent's own record of what runs, its own composition or the declaration kept before the switch), +// never one read from a composition bucket: the controller that wrote `composed-` ended (issue 1126), and the bus +// module on the bus node runs from what the agent holds. That is what tells the bus node it is one, and the file's +// place. The have of that module says what was written, which list it follows and this node's key, which the grants +// part seals to. // -// **It watches, it never polls**, and looks again after every apply of the bus module and when asked (Again). It reads no -// want and writes none, moves no fingerprint, and touches nothing of the bus module but its user list. +// **It watches, it never polls**: the grants part's key is watched, and what runs is looked at again when the agent says +// it changed, after every apply of the bus module and when asked (Again). It reads no want and writes none, moves no +// fingerprint, and touches nothing of the bus module but its user list. package userlist import ( "context" - "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "regexp" + "sort" "strings" "sync" "time" @@ -49,10 +52,10 @@ type Written = delivery.UserListWritten // Env is everything the follower needs of the node. type Env struct { - Node string - Composed agent.Watcher - // Trusted is the keys a composition may be signed by: one any other key signed declares no file. - Trusted func() []ed25519.PublicKey + Node string + // Running is the compositions this node runs now, as its agent holds them (agent.Agent.Everything): where the user + // list's file is is read from these alone (issue 1126). Nil runs nothing. + Running func() []delivery.Composed // OnDisk is what the resource's file holds on the machine now; nil when there is none. OnDisk func(resource json.RawMessage) ([]byte, error) // Write writes the user list's file resource of a module's composition alone, with body as its content, when it @@ -79,19 +82,18 @@ type Env struct { } type source struct { - key string module string - rev uint64 resource json.RawMessage - wanted string } -// Follower follows the node's compositions for the bus's user list. +// Follower follows the grants part's user list into the file the bus module this node runs declares. type Follower struct { env Env kick chan struct{} - mu sync.Mutex - srcs map[string]source // by composition key + // looks asks a look at what runs (RunsChanged): written only when the file it names moved. + looks chan struct{} + mu sync.Mutex + srcs map[string]source // by module, of what runs now // cur is the source last taken, and written whether its write went through. cur source written bool @@ -116,7 +118,8 @@ func New(env Env) *Follower { if env.Backoff == nil { env.Backoff = func(int) time.Duration { return time.Minute } } - f := &Follower{env: env, kick: make(chan struct{}, 1), srcs: map[string]source{}, said: map[string]bool{}} + f := &Follower{env: env, kick: make(chan struct{}, 1), looks: make(chan struct{}, 1), srcs: map[string]source{}, + said: map[string]bool{}} if env.Grants != nil && env.Kept != nil { if raw, rev := env.Kept(); raw != nil { if g, err := delivery.DecodeGrantsUserList(raw); err == nil { @@ -127,8 +130,8 @@ func New(env Env) *Follower { return f } -// Again asks the follower to look at the file again, though no composition changed: after an apply of the bus module, -// and when a person asks. It never blocks. +// Again asks the follower to look at what runs and the file again: when what runs changed, after an apply of the bus +// module, and when a person asks. It never blocks. func (f *Follower) Again() { select { case f.kick <- struct{}{}: @@ -136,21 +139,25 @@ func (f *Follower) Again() { } } -// State is the module whose list the follower writes ("" when no composition of this node carries one) and what it -// last did. +// RunsChanged asks the follower to look at what this node runs again: the list is written again only when the file the +// bus module declares moved. It never blocks. +func (f *Follower) RunsChanged() { + select { + case f.looks <- struct{}{}: + default: + } +} + +// State is the module whose list the follower writes ("" when nothing this node runs carries one) and what it last did. func (f *Follower) State() (string, Written) { f.mu.Lock() defer f.mu.Unlock() return f.mod, f.state } -// Run follows the compositions until ctx ends. +// Run follows the grants part's list until ctx ends. func (f *Follower) Run(ctx context.Context) error { - updates, err := f.env.Composed.Watch(ctx, f.env.Node+".>") - if err != nil { - return fmt.Errorf("the user list cannot be followed: %w", err) - } - caught := false + f.RunsChanged() // the first look at what runs, at once // The grants part's key, on the node whose compositions carry the list: the bus node alone is granted it (mesh-sdk // buscompose.GrantsUserListFollows). Until its watch caught up nothing is written, so a new follower (one per link) // never writes a list older than the key; a watch refused or ended is tried again on the backoff, and meanwhile the @@ -185,17 +192,11 @@ func (f *Follower) Run(ctx context.Context) error { return nil case <-f.kick: again = true + f.look() + case <-f.looks: + f.look() case <-retry: retry = nil - case u, ok := <-updates: - if !ok { - return fmt.Errorf("the watch of the compositions ended") - } - if u.Caught { - caught = true - } else { - f.take(u) - } case u, ok := <-grants: if !ok { failed("its watch ended") @@ -210,9 +211,6 @@ func (f *Follower) Run(ctx context.Context) error { } f.takeGrants(u) } - if !caught { - continue - } open() if opened && !grantsCaught && grants != nil { continue // the switch is not heard yet: nothing is written before it is @@ -226,11 +224,37 @@ func (f *Follower) Run(ctx context.Context) error { } } -// busNode says some composition of this node carries the bus's user list. +// busNode says something this node runs carries the bus's user list, or did. func (f *Follower) busNode() bool { f.mu.Lock() defer f.mu.Unlock() - return f.cur.key != "" || len(f.srcs) > 0 + return f.cur.module != "" || len(f.srcs) > 0 +} + +// look reads what this node runs now for the bus's user list: each module whose composition carries it. +func (f *Follower) look() { + var running []delivery.Composed + if f.env.Running != nil { + running = f.env.Running() + } + srcs := map[string]source{} + for _, c := range running { + if c.Node != "" && c.Node != f.env.Node { + continue + } + raw, _, _, ok := userList(c.Resources) + if !ok { + continue + } + module := c.Module + if module == "" { + module = agent.NodeModule + } + srcs[module] = source{module: module, resource: raw} + } + f.mu.Lock() + f.srcs = srcs + f.mu.Unlock() } // takeGrants keeps the grants part's key as heard. A key deleted, or a value it cannot read, is said and changes nothing: @@ -272,51 +296,6 @@ func (f *Follower) key() string { return f.env.Key() } -// take keeps a composition key's change when it carries the bus's user list, and forgets a key deleted. -func (f *Follower) take(u agent.Update) { - node, module, _, isNode, err := delivery.ParseComposedKey(u.Key) - if err != nil || node != f.env.Node || isNode { - return - } - f.mu.Lock() - defer f.mu.Unlock() - if u.Deleted { - delete(f.srcs, u.Key) - return - } - c, err := delivery.DecodeComposed(u.Value) - if err != nil { - delete(f.srcs, u.Key) - return - } - var trusted []ed25519.PublicKey - if f.env.Trusted != nil { - trusted = f.env.Trusted() - } - if _, err := delivery.VerifyComposed(c, trusted); err != nil { - // The agent says it is refused; a refused composition is never a source of the list. - delete(f.srcs, u.Key) - return - } - if c.Node != f.env.Node || c.Module != module { - // Signed, and for another node or module than its key says: a re-put of another composition is no source. - delete(f.srcs, u.Key) - f.sayOnceLocked(fmt.Sprintf("the composition under %s is %s's of %s, and is no source of the bus's user list", - u.Key, c.Node, c.Module)) - return - } - raw, content, digest, ok := userList(c.Resources) - if !ok { - delete(f.srcs, u.Key) - return - } - wanted := digest - if wanted == "" { - wanted = Digest([]byte(content)) - } - f.srcs[u.Key] = source{key: u.Key, module: module, rev: u.Revision, resource: raw, wanted: wanted} -} - // settle writes the grants part's list: when its key moved, or when asked. func (f *Follower) settle(ctx context.Context, again bool) { f.settleGrants(ctx, again) } @@ -337,18 +316,26 @@ func (f *Follower) settleGrants(ctx context.Context, again bool) { g = *f.grants } rev := f.grantsRev + // What runs now: the module written before while it still carries the list, else the first by name; what was + // written before when nothing runs that carries it now (a look between two applies). file := f.cur - for _, s := range f.srcs { - if s.rev > file.rev { - file = s + if s, ok := f.srcs[f.cur.module]; ok { + file = s + } else if len(f.srcs) > 0 { + names := make([]string, 0, len(f.srcs)) + for m := range f.srcs { + names = append(names, m) } + sort.Strings(names) + file = f.srcs[names[0]] } - if file.key == "" { + if file.module == "" { f.mu.Unlock() - return // no composition of this node declares the file: not the bus node + return // nothing this node runs declares the file: not the bus node } + moved := file.module != f.cur.module || pathOf(file.resource) != pathOf(f.cur.resource) f.cur = file - if !again && f.state.Source != "" && f.state.Revision == rev && f.written { + if !again && !moved && f.state.Source != "" && f.state.Revision == rev && f.written { f.mu.Unlock() return } @@ -522,6 +509,15 @@ func userList(resources []json.RawMessage) (json.RawMessage, string, string, boo return nil, "", "", false } +// pathOf is where a file resource says its file is; "" when it says none. +func pathOf(raw json.RawMessage) string { + var head struct { + Path string `json:"path"` + } + _ = json.Unmarshal(raw, &head) + return head.Path +} + // Digest is content's digest as the have says it, `sha256:`. func Digest(content []byte) string { sum := sha256.Sum256(content) diff --git a/internal/userlist/userlist_test.go b/internal/userlist/userlist_test.go index 33153da..a891016 100644 --- a/internal/userlist/userlist_test.go +++ b/internal/userlist/userlist_test.go @@ -2,7 +2,6 @@ package userlist import ( "context" - "crypto/ed25519" "encoding/json" "errors" "strings" @@ -17,17 +16,6 @@ import ( const node = "novox" -func key() (ed25519.PublicKey, ed25519.PrivateKey) { - pub, k, err := ed25519.GenerateKey(nil) - if err != nil { - panic(err) - } - return pub, k -} - -var controllerPub, controllerKey = key() -var _, strangerKey = key() - func fpOf(c string) string { return delivery.SourcePrefix2 + strings.Repeat(c, 64) } // feed is a watch the test writes to: the keys there before the watch, Caught, then each put. watching is closed once @@ -69,19 +57,14 @@ func list(users ...string) string { return b.String() } -// comp is a composition of module at fp holding the user list content (none when empty), signed by signer; by digest -// when byDigest, as the controller sends a file over 64 KiB; for another node or module than its key when asked. -func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p put) (string, []byte) { - t.Helper() - owner := module - if p.asModule != "" { - owner = p.asModule - } - server, _ := json.Marshal(map[string]any{"id": owner + ".server-conf", "type": "file", "module": owner, +// comp is a composition of module at fp as this node runs it, holding the user list content (none when empty); by +// digest when byDigest, as a file over 64 KiB is given; for another node than this one when asked. +func comp(module, fp, content string, p put) delivery.Composed { + server, _ := json.Marshal(map[string]any{"id": module + ".server-conf", "type": "file", "module": module, "path": "/var/lib/nats-module/conf/nats.conf", "content": "include accounts.conf\n"}) rs := []json.RawMessage{server} if content != "" { - r := map[string]any{"id": owner + ".bus-users", "type": "file", "module": owner, + r := map[string]any{"id": module + ".bus-users", "type": "file", "module": module, "path": "/var/lib/nats-module/conf/accounts.conf", "content": content, "mode": "0600"} if p.byDigest { delete(r, "content") @@ -94,22 +77,7 @@ func comp(t *testing.T, module, fp, content string, signer ed25519.PrivateKey, p if p.asNode != "" { c.Node = p.asNode } - if p.asModule != "" { - c.Module = p.asModule - } - k, err := delivery.ComposedKey(node, module, fp) - if err != nil { - t.Fatal(err) - } - signed, err := delivery.SignComposed(c, signer) - if err != nil { - t.Fatal(err) - } - raw, err := delivery.EncodeComposed(signed) - if err != nil { - t.Fatal(err) - } - return k, raw + return c } // gput is the grants part's key as the bus node hears it at the watch's start (ADR 0547 clauses 7, 8): which list to @@ -125,12 +93,12 @@ type gput struct { type put struct { module, fp, content string - stranger, deleted bool byDigest bool - asNode, asModule string + asNode string } -// Defends novox/hq ADR 0547 §8: the bus node writes the newest signed list alone, only when it changed, and reports it. +// Defends novox/hq ADR 0547 §8: the bus node writes the grants part's list alone, only when it changed, and reports it, +// into the file the bus module it runs declares (issue 1126, B17: what runs, never a composition bucket). func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { one, two, three := list("controller", "a"), list("controller", "a", "b"), list("controller", "a", "b", "c") five := list("controller", "a", "b", "c", "d") @@ -152,13 +120,14 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { // wantSource and wantFollows are what the last have says it took the list from and follows. wantSource, wantFollows string }{ - // Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module's - // composition declares; the list a composition carries is never written, whatever becomes of the grants key. - {name: "the grants part's list is written into the file a composition declares", + // Issue 1126, A6 (ADR 0547 clauses 7, 8, 11): the grants part's list alone, written into the file the bus module + // this node runs declares; the list a composition carries is never written, whatever becomes of the grants key. + // B17: the file's place is read from what runs, as the agent holds it, never from a composition bucket. + {name: "the grants part's list is written into the file the bus module it runs declares", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}, wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, - {name: "a key that says follow the controller still writes the grants part's list", - before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{follow: delivery.FollowController, content: three}}, + {name: "a bus module it runs by digest still names the file", + before: []put{{module: "nats", fp: fpOf("a"), content: two, byDigest: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}, wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, {name: "a newer composition's list is not written", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, @@ -167,8 +136,8 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { wantWrites: []string{three}, wantUsers: 4, wantSource: delivery.GrantsUserListSource, wantFollows: delivery.FollowGrants}, {name: "a composition without the list is ignored", before: []put{{module: "web", fp: fpOf("a"), content: ""}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}}, - {name: "a composition signed by a stranger declares no file", - before: []put{{module: "nats", fp: fpOf("a"), content: two, stranger: true}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}}, + {name: "another node's composition declares no file", + before: []put{{module: "nats", fp: fpOf("a"), content: two, asNode: "ace"}}, grants: []gput{{follow: delivery.FollowGrants, content: three}}}, {name: "an unreadable grants key with none heard before writes nothing, never the composition's list", before: []put{{module: "nats", fp: fpOf("a"), content: two}}, grants: []gput{{unreadable: true}}, onDisk: two, wantErr: "key is not heard", wantRefused: true, wantSource: delivery.GrantsUserListSource}, @@ -201,9 +170,8 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { onDisk: two, wantErr: "sealed to another key", wantRefused: true, wantSource: delivery.GrantsUserListSource}, } { t.Run(c.name, func(t *testing.T) { - f := &feed{ch: make(chan agent.Update, 64), watching: make(chan struct{})} - objects := map[string]string{} var mu sync.Mutex + var running []delivery.Composed var writes []string var reports []Written reported := make(chan struct{}, 64) @@ -257,7 +225,12 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { gf.first = append(gf.first, u) } } - fl := New(Env{Node: node, Composed: f, Grants: gf, + fl := New(Env{Node: node, Grants: gf, + Running: func() []delivery.Composed { + mu.Lock() + defer mu.Unlock() + return append([]delivery.Composed(nil), running...) + }, Kept: func() ([]byte, uint64) { return keptRaw, 9 }, Keep: func([]byte, uint64) {}, Backoff: func(int) time.Duration { return time.Hour }, @@ -271,8 +244,7 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { out, err := seal.Open(b, nodeKey.PrivateKey) return []byte(out), err }, - Key: func() string { return nodeKey.PublicKey }, - Trusted: func() []ed25519.PublicKey { return []ed25519.PublicKey{controllerPub} }, + Key: func() string { return nodeKey.PublicKey }, OnDisk: func(json.RawMessage) ([]byte, error) { if c.onDisk == "" { return nil, nil @@ -296,30 +268,14 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { }, Now: func() time.Time { return time.Date(2026, 10, 25, 21, 50, 0, 0, time.UTC) }, }) - update := func(p put) agent.Update { - f.rev++ - signer := controllerKey - if p.stranger { - signer = strangerKey - } - k, raw := comp(t, p.module, p.fp, p.content, signer, p) - if p.byDigest { - objects[Digest([]byte(p.content))] = p.content - } - if p.deleted { - return agent.Update{Key: k, Revision: f.rev, Deleted: true} - } - return agent.Update{Key: k, Value: raw, Revision: f.rev} - } for _, p := range c.before { - f.first = append(f.first, update(p)) + running = append(running, comp(p.module, p.fp, p.content, p)) } ctx, cancel := context.WithCancel(context.Background()) defer cancel() done := make(chan error, 1) go func() { done <- fl.Run(ctx) }() - <-f.watching - // A later put after the first list is written: the first write waits for the grants part's key to catch up. + // What runs changes after the first list is written: the first write waits for the grants part's key. if len(c.after) > 0 { for until := time.Now().Add(5 * time.Second); time.Now().Before(until); time.Sleep(5 * time.Millisecond) { mu.Lock() @@ -331,7 +287,10 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { } } for _, p := range c.after { - f.ch <- update(p) + mu.Lock() + running = []delivery.Composed{comp(p.module, p.fp, p.content, p)} + mu.Unlock() + fl.RunsChanged() } if len(grantsAfter) > 0 { for until := time.Now().Add(5 * time.Second); ; time.Sleep(5 * time.Millisecond) { @@ -384,23 +343,18 @@ func TestTheBusNodeWritesTheGrantsPartsUserListAlone(t *testing.T) { if c.wantUnwatched != (st.Unwatched != "") { t.Fatalf("the have says unwatched %q", st.Unwatched) } - // While the grants part's list is followed, an apply of the controller's composition writes not its list. - if _, raw := comp(t, "nats", fpOf("a"), two, controllerKey, put{}); c.wantFollows != "" { - cmp, err := delivery.DecodeComposed(raw) - if err != nil { - t.Fatal(err) - } + // While the grants part's list is followed, an apply of the bus module's composition writes not its list. + if cmp := comp("nats", fpOf("a"), two, put{}); c.wantFollows != "" { held, err := fl.Hold(cmp) if err != nil { t.Fatal(err) } - _, _, _, hasList := userList(held.Resources) - if want := c.wantFollows == delivery.FollowController; hasList != want || len(held.Resources) < 1 { - t.Fatalf("an apply while following %s holds the controller's list: %v", c.wantFollows, hasList) + if _, _, _, hasList := userList(held.Resources); hasList || len(held.Resources) < 1 { + t.Fatalf("an apply while following %s holds the composition's list: %v", c.wantFollows, hasList) } // A list on the machine that cannot be read stops the apply; it is never left out to be removed. if c.wantFollows == delivery.FollowGrants { - readErr := New(Env{Node: node, Composed: f, OnDisk: func(json.RawMessage) ([]byte, error) { + readErr := New(Env{Node: node, OnDisk: func(json.RawMessage) ([]byte, error) { return nil, errors.New("permission denied") }}) readErr.grants = &delivery.GrantsUserList{Follow: delivery.FollowGrants} diff --git a/internal/witness/contract.go b/internal/witness/contract.go index e3f2321..35e8b0b 100644 --- a/internal/witness/contract.go +++ b/internal/witness/contract.go @@ -1,155 +1,54 @@ -// Package witness is the node-engine watching a core build it placed — the controller on the control -// node, the node tools on every machine — and restoring the build before it when the new one is not -// healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the component being replaced is never the -// only witness of its successor). +// Package witness is the node-engine watching a core build it placed — the node tools on every machine — and +// restoring the build before it when the new one is not healthy in bound (novox/hq to-be 45 §8, ADR 0227 rule 8: the +// component being replaced is never the only witness of its successor). // // **The contract is this file.** What the engine reads to call a build healthy, from where, in what -// shape, within which bound — said once, here, and held by contract_test.go. The controller's side -// writes what is read here (mesh-controller internal/lease Holder); a change on either side is a -// change to this file and its test. +// shape, within which bound — said once, here, and held by contract_test.go. package witness import ( - "encoding/json" - "fmt" - "strings" "time" "github.com/novox/mesh-host/internal/link" ) -// The two witnesses, as a process declares which watches it (`witness`). A process that says none is -// placed as ever; one that says nothing is judged by its name's default (Default). +// The witnesses, as a process declares which watches it (`witness`). A process that says none is placed as ever; one +// that says nothing is judged by its name's default (Default). const ( - // ByLease: healthy when the controller this machine started holds the controller's lease. - ByLease = "lease" // ByPing: healthy when this machine's runtime answers the services protocol's PING. ByPing = "ping" // ByNone: not judged. ByNone = "none" ) -// The controller's lease, as the host reads it (novox/hq ADR 0229): the key `holder` in the bucket -// `mesh-controller_lease`, whose keys live fifteen seconds unless renewed, renewed every five. -const ( - LeaseBucket = "mesh-controller_lease" - LeaseKey = "holder" - // LeaseAge is the bucket's age for its key. A holder whose last renewal is older than this is - // not holding it, whatever the key still says. - LeaseAge = 15 * time.Second - // Skew is how far the controller's clock and this host's may disagree about when it took the - // lease. One machine, one clock, in practice: a margin, not a tolerance. - Skew = 2 * time.Second -) - // Bounds (to-be 45 §8). A build is given Within of time the witness could ask in; asked every Every. const ( - // ControllerWithin: the controller holds the lease within sixty seconds of starting. - ControllerWithin = 60 * time.Second // NodeToolsWithin: the runtime is announced and answering within sixty seconds of starting, // each PING answered within PingWithin. NodeToolsWithin = 60 * time.Second PingWithin = 5 * time.Second Every = 5 * time.Second // GiveUp is how long a witness goes on when it cannot ask at all before it says the build is - // unwitnessed: the grant missing, the bucket missing, the bus away the whole time. + // unwitnessed: the grant missing, the bus away the whole time. GiveUp = 30 * time.Minute ) -// ControllerLease is the lease's value as the controller writes it — mesh-controller internal/lease -// Holder, field for field by its JSON names. Only what the witness reads is required; a field the -// controller adds later is ignored here. -type ControllerLease struct { - // Instance names one controller process: "controller@ pid since ". - Instance string `json:"instance"` - // Host is the machine it runs on, as its own hostname says. - Host string `json:"host,omitempty"` - // Build is the controller's build as it knows it; not read here — the toolchain stamps no - // version, so it says "development build" — and a bundle's identity is its digest, which the - // host already knows. - Build string `json:"build,omitempty"` - Epoch uint64 `json:"epoch,omitempty"` - // Taken is when this instance took the key, Renewed when it last renewed it, by its own clock. - Taken time.Time `json:"taken"` - Renewed time.Time `json:"renewed"` -} - -// ParseLease reads the key's value. -func ParseLease(value []byte) (ControllerLease, error) { - var l ControllerLease - if err := json.Unmarshal(value, &l); err != nil { - return ControllerLease{}, fmt.Errorf("the lease's holder is not the controller's lease value: %w", err) - } - return l, nil -} - -// HeldBySince says whether the lease is held by a controller on machine `host` that took it at or -// after `since` — the controller this machine started then, and not the one before it — and is -// held now. Why says what it saw when it is not. -func (l ControllerLease) HeldBySince(host string, since, now time.Time) (bool, string) { - switch { - case l.Instance == "": - return false, "the lease names no holder" - case host != "" && !sameMachine(l.Host, host): - return false, fmt.Sprintf("the lease is held by %s, on %s and not this machine", l.Instance, l.Host) - case l.Taken.Before(since.Add(-Skew)): - return false, fmt.Sprintf("the lease is held by %s, taken %s — before the new build started at %s", - l.Instance, l.Taken.UTC().Format(time.RFC3339), since.UTC().Format(time.RFC3339)) - case now.Sub(latest(l.Taken, l.Renewed)) > LeaseAge: - return false, fmt.Sprintf("the lease names %s and was last renewed %s ago, past its %s", - l.Instance, now.Sub(latest(l.Taken, l.Renewed)).Round(time.Second), LeaseAge) - } - return true, fmt.Sprintf("%s holds the lease, epoch %d", l.Instance, l.Epoch) -} - -// sameMachine compares two hostnames as names, so a short name and its fully qualified form agree. -func sameMachine(a, b string) bool { - short := func(s string) string { - s = strings.ToLower(strings.TrimSpace(s)) - if i := strings.IndexByte(s, '.'); i > 0 { - s = s[:i] - } - return s - } - return short(a) == short(b) -} - -func latest(a, b time.Time) time.Time { - if b.After(a) { - return b - } - return a -} - // NodeToolsService is the runtime's name on the services protocol, and its instance is this // machine's node name: what `$SRV.PING..` asks, so only this machine's runtime can // answer (mesh-tools node-tools internal/runtime, announce.Service{Name: module, ID: node}). const NodeToolsService = "node-tools" -// Default is the witness a process is judged by when it names none: the two core processes the mesh -// composes, by the names it composes them under, and nothing else. +// Default is the witness a process is judged by when it names none: the core process the mesh composes on every +// machine, by the name it composes it under, and nothing else. func Default(process string) string { - switch process { - case "mesh-controller": - return ByLease - case NodeToolsService: + if process == NodeToolsService { return ByPing } return ByNone } // Within is the bound for a witness. -func Within(by string) time.Duration { - if by == ByLease { - return ControllerWithin - } - return NodeToolsWithin -} +func Within(string) time.Duration { return NodeToolsWithin } // Component names what a witness judges, as a rollback says it. -func Component(by string) string { - if by == ByLease { - return link.ComponentController - } - return link.ComponentNodeTools -} +func Component(string) string { return link.ComponentNodeTools } diff --git a/internal/witness/contract_test.go b/internal/witness/contract_test.go index 5168968..2ea5c5c 100644 --- a/internal/witness/contract_test.go +++ b/internal/witness/contract_test.go @@ -9,82 +9,7 @@ import ( "github.com/novox/mesh-host/internal/link" ) -// The contract, held. Each of these is a line the controller's side relies on or writes; a change on -// either side changes this test (novox/hq to-be 45 §8). - -// The lease is read where the controller keeps it (ADR 0229), on the one subject a host's grant names. -func TestTheLeaseIsReadWhereTheControllerKeepsIt(t *testing.T) { - if LeaseBucket != "mesh-controller_lease" || LeaseKey != "holder" { - t.Fatalf("the lease is read from %s/%s; the controller keeps it in mesh-controller_lease/holder", - LeaseBucket, LeaseKey) - } - if got := link.DirectGetSubject(LeaseBucket, LeaseKey); got != "$JS.API.DIRECT.GET.KV_mesh-controller_lease.$KV.mesh-controller_lease.holder" { - t.Fatalf("the host asks %s for the lease; its grant names exactly the direct get of the key", got) - } - if LeaseAge != 15*time.Second { - t.Fatalf("the lease's age is %s; the controller's bucket keeps a key fifteen seconds", LeaseAge) - } -} - -// What the controller writes — mesh-controller internal/lease Holder, by its JSON names — is what is -// read. The value below is the shape that Holder marshals to. -func TestTheLeaseValueIsTheControllersHolder(t *testing.T) { - written := `{"instance":"controller@anchor pid 4242 since 2026-10-06T10:00:00Z","host":"anchor",` + - `"build":"development build","epoch":57,"taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:31Z"}` - l, err := ParseLease([]byte(written)) - if err != nil { - t.Fatal(err) - } - if l.Instance == "" || l.Host != "anchor" || l.Epoch != 57 || l.Taken.IsZero() || l.Renewed.IsZero() { - t.Fatalf("the lease was not read whole: %+v", l) - } - // And a field the controller adds later does not stop it being read. - if _, err := ParseLease([]byte(`{"instance":"i","taken":"2026-10-06T10:00:01Z","renewed":"2026-10-06T10:00:01Z","bundle":"sha256:x"}`)); err != nil { - t.Fatalf("a lease value with a field this host does not know was refused: %v", err) - } -} - -// Healthy is: held now, on this machine, by an instance that took it after the new build started. -func TestTheNewControllerHoldsTheLeaseOnlyWhenItTookItAfterItStarted(t *testing.T) { - started := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - now := started.Add(30 * time.Second) - held := ControllerLease{Instance: "controller@anchor pid 2 since …", Host: "anchor.example", - Taken: started.Add(3 * time.Second), Renewed: now.Add(-2 * time.Second), Epoch: 58} - for _, c := range []struct { - name string - l ControllerLease - want bool - says string - }{ - {"held by the new instance", held, true, "holds the lease"}, - {"nobody", ControllerLease{}, false, "no holder"}, - {"the old instance, taken before the restart", func() ControllerLease { - l := held - l.Taken = started.Add(-time.Hour) - return l - }(), false, "before the new build started"}, - {"another machine's controller", func() ControllerLease { - l := held - l.Host = "home-server" - return l - }(), false, "not this machine"}, - {"taken and no longer renewed", func() ControllerLease { - l := held - l.Renewed = now.Add(-20 * time.Second) - return l - }(), false, "past its"}, - {"taken within the clocks' skew of the start", func() ControllerLease { - l := held - l.Taken = started.Add(-time.Second) - return l - }(), true, "holds the lease"}, - } { - got, why := c.l.HeldBySince("anchor", started, now) - if got != c.want || !strings.Contains(why, c.says) { - t.Errorf("%s: healthy %v (%s), want %v saying %q", c.name, got, why, c.want, c.says) - } - } -} +// The contract, held (novox/hq to-be 45 §8). // The runtime is asked by its own name and this machine's: only this machine's runtime can answer. func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) { @@ -93,23 +18,23 @@ func TestTheRuntimeIsAskedByItsNameAndThisMachines(t *testing.T) { } } -// What is judged by default is the mesh's two core processes, by the names the controller composes -// them under, and nothing else. -func TestOnlyTheCoreProcessesAreJudgedByDefault(t *testing.T) { - for name, want := range map[string]string{"mesh-controller": ByLease, "node-tools": ByPing, "greeter": ByNone} { +// What is judged by default is the mesh's core process on every machine, by the name it is composed under, and +// nothing else: the controller ended (issue 1126), so its name is judged as any other. +func TestOnlyTheCoreProcessIsJudgedByDefault(t *testing.T) { + for name, want := range map[string]string{"mesh-controller": ByNone, "node-tools": ByPing, "greeter": ByNone} { if got := Default(name); got != want { t.Errorf("%s is judged by %q, want %q", name, got, want) } } - if ControllerWithin != 60*time.Second || NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second { - t.Fatal("the bounds are to-be 45 §8's: the lease within sixty seconds, a PING answered within five") + if NodeToolsWithin != 60*time.Second || PingWithin != 5*time.Second { + t.Fatal("the bounds are to-be 45 §8's: the runtime within sixty seconds, a PING answered within five") } } // A verdict on the wire names its component as the controller's condition does. func TestAVerdictNamesItsComponent(t *testing.T) { - raw, _ := json.Marshal(link.Rollback{Component: Component(ByLease), Outcome: link.RolledBack}) - if !strings.Contains(string(raw), `"component":"controller"`) || Component(ByPing) != "node-tools" { + raw, _ := json.Marshal(link.Rollback{Component: Component(ByPing), Outcome: link.RolledBack}) + if !strings.Contains(string(raw), `"component":"node-tools"`) { t.Fatalf("a verdict names its component as %s", raw) } } diff --git a/internal/witness/draining_test.go b/internal/witness/draining_test.go index 4a7d32d..6429891 100644 --- a/internal/witness/draining_test.go +++ b/internal/witness/draining_test.go @@ -85,11 +85,11 @@ func retiredBuilds(t *testing.T, root, name string) []string { // after its build was moved into a 0700 directory, and then deleted, could not run its own verbs. func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) { root := t.TempDir() - name := "mesh-controller" + name := "node-tools" start := time.Date(2026, 10, 7, 2, 0, 0, 0, time.UTC) // Build A placed and running. - p, err := Place(root, name, ByLease, buildA, "", "", start) + p, err := Place(root, name, ByPing, buildA, "", "", start) if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestABuildIsKeptReachableAndUndeletedWhileAProcessRunsFromIt(t *testing.T) } // Build B placed while A still runs: A is moved aside, and reachable at its new path. - place(t, root, name, ByLease, buildB, "", "", start.Add(time.Minute)) + place(t, root, name, ByPing, buildB, "", "", start.Add(time.Minute)) for _, dir := range []string{filepath.Join(root, ".witness"), Dir(root, name)} { info, err := os.Stat(dir) if err != nil { diff --git a/internal/witness/watch.go b/internal/witness/watch.go index 2fe5d2f..bb9d0a3 100644 --- a/internal/witness/watch.go +++ b/internal/witness/watch.go @@ -14,15 +14,14 @@ import ( // concluded (to-be 45 §8). // // **Time counts only while it could ask.** A build's bound is spent only on looks that got an answer -// from the bus — the lease read, or the PING delivered — so a machine whose own link is down, or a +// from the bus — the PING delivered — so a machine whose own link is down, or a // bus that refuses the question, never rolls a build back for the host's own trouble. A look that // could not ask counts towards GiveUp instead, and at GiveUp the build is said to be unwitnessed: // neither proved nor rolled back, and said. type Watcher struct { Root string - // Node is this machine's node name, the instance its runtime answers PING as; Host its hostname, - // as the controller's lease names its machine. - Node, Host string + // Node is this machine's node name, the instance its runtime answers PING as. + Node string // Asker is the link open now, or nil. Asker func() link.Asker Run Runner @@ -69,7 +68,7 @@ func (w *Watcher) look(ctx context.Context, s State) { every = Every } now := w.now() - healthy, why, err := w.judge(ctx, s, now) + healthy, why, err := w.judge(ctx, s) w.hold(func() { // Read again in turn: an apply may have placed another build meanwhile, which starts its own @@ -123,7 +122,7 @@ func (w *Watcher) look(ctx context.Context, s State) { } // judge asks once. An error is a look that could not ask; otherwise healthy, and why not when not. -func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, string, error) { +func (w *Watcher) judge(ctx context.Context, s State) (bool, string, error) { var asker link.Asker if w.Asker != nil { asker = w.Asker() @@ -134,20 +133,6 @@ func (w *Watcher) judge(ctx context.Context, s State, now time.Time) (bool, stri asking, cancel := context.WithTimeout(ctx, PingWithin) defer cancel() switch s.Witness { - case ByLease: - value, found, err := asker.ReadKey(asking, LeaseBucket, LeaseKey) - if err != nil { - return false, "", err - } - if !found { - return false, "nobody holds the controller's lease", nil - } - lease, err := ParseLease(value) - if err != nil { - return false, err.Error(), nil - } - held, why := lease.HeldBySince(w.Host, s.Started, now) - return held, why, nil case ByPing: err := asker.Ping(asking, s.Process, w.Node) switch { diff --git a/internal/witness/watch_test.go b/internal/witness/watch_test.go index 0f3d908..148b6a2 100644 --- a/internal/witness/watch_test.go +++ b/internal/witness/watch_test.go @@ -13,8 +13,8 @@ import ( "github.com/novox/mesh-host/internal/link" ) -// Each rollback path, induced against a real directory: a controller that never takes the lease, a -// runtime that never answers PING, a build declared not reversible; and a healthy update, which +// Each rollback path, induced against a real directory: a runtime that never answers PING, a build +// declared not reversible; and a healthy update, which // deletes nothing before it is proved and the build before it once it is (novox/hq to-be 45 §8). const ( @@ -62,18 +62,10 @@ func kept(root, name string) bool { // asker answers as the build it is told is running would. type asker struct { mu sync.Mutex - lease func() ([]byte, bool, error) ping func() error asked int } -func (a *asker) ReadKey(context.Context, string, string) ([]byte, bool, error) { - a.mu.Lock() - defer a.mu.Unlock() - a.asked++ - return a.lease() -} - func (a *asker) Ping(context.Context, string, string) error { a.mu.Lock() defer a.mu.Unlock() @@ -95,7 +87,7 @@ func (m *machine) run(_ context.Context, name string, args ...string) (string, e } func watcher(root string, a link.Asker, m *machine, clock *time.Time, verdicts *[]link.Rollback) *Watcher { - return &Watcher{Root: root, Node: "anchor", Host: "anchor", + return &Watcher{Root: root, Node: "anchor", Asker: func() link.Asker { if a == nil { return nil @@ -116,45 +108,37 @@ func look(w *Watcher, clock *time.Time, n int) { } } -func leaseHeldBy(instance string, taken time.Time, clock *time.Time) func() ([]byte, bool, error) { - return func() ([]byte, bool, error) { - return []byte(fmt.Sprintf(`{"instance":%q,"host":"anchor","epoch":9,"taken":%q,"renewed":%q}`, - instance, taken.Format(time.RFC3339Nano), clock.Add(-time.Second).Format(time.RFC3339Nano))), true, nil - } -} - -// A controller that starts and never takes the lease: restored to the build before, once, said once. -func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) { +// A runtime that starts and never answers PING: restored to the build before, once, said once. +func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) - place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) - if running(t, root, "mesh-controller") != buildB || !kept(root, "mesh-controller") { - t.Fatal("the new controller is not running with the build before it kept beside it") + place(t, root, "node-tools", ByPing, buildA, "", "", clock) + place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) + if running(t, root, "node-tools") != buildB || !kept(root, "node-tools") { + t.Fatal("the new runtime is not running with the build before it kept beside it") } - // The old instance's lease, taken an hour ago, is all the bus ever shows. - a := &asker{lease: leaseHeldBy("controller@anchor pid 1 since earlier", clock.Add(-time.Hour), &clock)} + a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) - look(w, &clock, int(ControllerWithin/Every)-1) - if len(verdicts) != 0 || running(t, root, "mesh-controller") != buildB { + look(w, &clock, int(NodeToolsWithin/Every)-1) + if len(verdicts) != 0 || running(t, root, "node-tools") != buildB { t.Fatalf("rolled back before its bound: %+v", verdicts) } look(w, &clock, 1) if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].From != buildB || verdicts[0].To != buildA || - verdicts[0].Component != link.ComponentController { - t.Fatalf("the verdict is %+v, want controller rolled back from B to A", verdicts) + verdicts[0].Component != link.ComponentNodeTools { + t.Fatalf("the verdict is %+v, want node-tools rolled back from B to A", verdicts) } - if running(t, root, "mesh-controller") != buildA { - t.Fatalf("the controller running is %s, want the build before", running(t, root, "mesh-controller")) + if running(t, root, "node-tools") != buildA { + t.Fatalf("the runtime running is %s, want the build before", running(t, root, "node-tools")) } - if strings.Join(m.commands, "; ") != "systemctl stop mesh-controller.service; systemctl restart mesh-controller.service" { + if strings.Join(m.commands, "; ") != "systemctl stop node-tools.service; systemctl restart node-tools.service" { t.Fatalf("the machine was asked %v", m.commands) } - if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "before the new build started") { + if !strings.Contains(verdicts[0].Why, "not healthy within 1m0s") || !strings.Contains(verdicts[0].Why, "is on the bus") { t.Fatalf("the verdict does not say why: %s", verdicts[0].Why) } @@ -168,81 +152,58 @@ func TestAControllerThatNeverTakesTheLeaseIsRolledBackOnce(t *testing.T) { t.Fatalf("what every report says is %+v", s) } // The mesh still declares B: refused, A kept running, the verdict still standing. - p := place(t, root, "mesh-controller", ByLease, buildB, buildB, "", clock) - if p.Unpack || running(t, root, "mesh-controller") != buildA || !strings.Contains(p.Detail, "rolled back") { - t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "mesh-controller")) + p := place(t, root, "node-tools", ByPing, buildB, buildB, "", clock) + if p.Unpack || running(t, root, "node-tools") != buildA || !strings.Contains(p.Detail, "rolled back") { + t.Fatalf("a rolled-back build was placed again: %+v, running %s", p, running(t, root, "node-tools")) } if len(Standing(root)) != 1 || len(Trials(root)) != 0 { t.Fatal("refusing the rolled-back build changed what stands or started a trial") } // A newer build is a new trial, and ends what was concluded once it is proved. - place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) - if running(t, root, "mesh-controller") != buildC || len(Trials(root)) != 1 { + place(t, root, "node-tools", ByPing, buildC, buildB, "", clock) + if running(t, root, "node-tools") != buildC || len(Trials(root)) != 1 { t.Fatal("a newer build after a rollback was not placed on trial") } - if err := Proved(root, "mesh-controller"); err != nil { + if err := Proved(root, "node-tools"); err != nil { t.Fatal(err) } - if len(Standing(root)) != 0 || kept(root, "mesh-controller") { + if len(Standing(root)) != 0 || kept(root, "node-tools") { t.Fatal("a newer build proved and the rollback still stands, or the build before it is still kept") } } -// A runtime that never answers PING: restored, once. -func TestARuntimeThatNeverAnswersIsRolledBackOnce(t *testing.T) { - root := t.TempDir() - clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "node-tools", ByPing, buildA, "", "", clock) - place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) - - a := &asker{ping: func() error { return fmt.Errorf("%w: no node-tools on this machine is on the bus", link.ErrNoAnswer) }} - m := &machine{} - var verdicts []link.Rollback - w := watcher(root, a, m, &clock, &verdicts) - look(w, &clock, int(NodeToolsWithin/Every)+20) - - if len(verdicts) != 1 || verdicts[0].Outcome != link.RolledBack || verdicts[0].Component != link.ComponentNodeTools { - t.Fatalf("the verdict is %+v, want node-tools rolled back once", verdicts) - } - if running(t, root, "node-tools") != buildA || kept(root, "node-tools") { - t.Fatal("the runtime running is not the build before, or a copy of it is still kept") - } -} - // A healthy update: nothing deleted before it is proved; the build before it deleted once it is. func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - // The controller placed by a host from before any witness: a record, and no state. - if err := os.MkdirAll(filepath.Join(root, "mesh-controller"), 0o755); err != nil { + // The runtime placed by a host from before any witness: a record, and no state. + if err := os.MkdirAll(filepath.Join(root, "node-tools"), 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(root, "mesh-controller", "build"), []byte(buildA), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(root, "node-tools", "build"), []byte(buildA), 0o644); err != nil { t.Fatal(err) } - place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) - started := clock + place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) - a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} + a := &asker{ping: func() error { return link.ErrNoAnswer }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, 5) - if !kept(root, "mesh-controller") { + if !kept(root, "node-tools") { t.Fatal("the build before was deleted while the new one was still on trial") } // A third build while the second is on trial: the one kept is still the one seen healthy. - place(t, root, "mesh-controller", ByLease, buildC, buildB, "", clock) - if s, _ := Load(root, "mesh-controller"); s.Previous != buildA { + place(t, root, "node-tools", ByPing, buildC, buildB, "", clock) + if s, _ := Load(root, "node-tools"); s.Previous != buildA { t.Fatalf("the build kept to go back to is %s, want A — the last one seen healthy", s.Previous) } - started = clock - a.lease = leaseHeldBy("controller@anchor pid 3 since now", started.Add(4*time.Second), &clock) + a.ping = func() error { return nil } look(w, &clock, 2) if len(verdicts) != 0 || len(m.commands) != 0 { t.Fatalf("a healthy update was judged or acted on: %+v %v", verdicts, m.commands) } - if kept(root, "mesh-controller") || len(Trials(root)) != 0 || running(t, root, "mesh-controller") != buildC { + if kept(root, "node-tools") || len(Trials(root)) != 0 || running(t, root, "node-tools") != buildC { t.Fatal("the build before was not deleted once the new one was proved") } } @@ -252,14 +213,14 @@ func TestAHealthyUpdateDeletesNothingUntilItIsProved(t *testing.T) { func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) - place(t, root, "mesh-controller", ByLease, buildB, buildA, "migration 0073 cannot be undone", clock) + place(t, root, "node-tools", ByPing, buildA, "", "", clock) + place(t, root, "node-tools", ByPing, buildB, buildA, "migration 0073 cannot be undone", clock) - a := &asker{lease: func() ([]byte, bool, error) { return nil, false, nil }} + a := &asker{ping: func() error { return link.ErrNoAnswer }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) - look(w, &clock, int(ControllerWithin/Every)+20) + look(w, &clock, int(NodeToolsWithin/Every)+20) if len(verdicts) != 1 || verdicts[0].Outcome != link.NotReversible || verdicts[0].To != "" { t.Fatalf("the verdict is %+v, want not-reversible with nothing restored", verdicts) @@ -267,9 +228,9 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { if !strings.Contains(verdicts[0].Why, "migration 0073 cannot be undone") { t.Fatalf("the verdict does not say why it may not be rolled back: %s", verdicts[0].Why) } - if len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { + if len(m.commands) != 0 || running(t, root, "node-tools") != buildB { t.Fatalf("the build before was started against the newer data: %v, running %s", m.commands, - running(t, root, "mesh-controller")) + running(t, root, "node-tools")) } if len(Standing(root)) != 1 { t.Fatal("the not-reversible verdict does not stand") @@ -280,24 +241,24 @@ func TestANotReversibleBuildIsNeverRolledBack(t *testing.T) { func TestAWitnessThatCannotAskNeverRollsBack(t *testing.T) { root := t.TempDir() clock := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) - place(t, root, "mesh-controller", ByLease, buildA, "", "", clock) - place(t, root, "mesh-controller", ByLease, buildB, buildA, "", clock) + place(t, root, "node-tools", ByPing, buildA, "", "", clock) + place(t, root, "node-tools", ByPing, buildB, buildA, "", clock) - a := &asker{lease: func() ([]byte, bool, error) { - return nil, false, fmt.Errorf("%w: this host's grant does not name the lease", link.ErrCannotAsk) + a := &asker{ping: func() error { + return fmt.Errorf("%w: this host's grant does not name the PING", link.ErrCannotAsk) }} m := &machine{} var verdicts []link.Rollback w := watcher(root, a, m, &clock, &verdicts) look(w, &clock, int(GiveUp/Every)-1) - if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "mesh-controller") != buildB { + if len(verdicts) != 0 || len(m.commands) != 0 || running(t, root, "node-tools") != buildB { t.Fatalf("a build was judged on questions that were never asked: %+v %v", verdicts, m.commands) } look(w, &clock, 1) if len(verdicts) != 1 || verdicts[0].Outcome != link.Unwitnessed || len(m.commands) != 0 { t.Fatalf("the verdict at GiveUp is %+v, want unwitnessed with nothing done", verdicts) } - if !kept(root, "mesh-controller") { + if !kept(root, "node-tools") { t.Fatal("the build before was deleted though the new one was never seen healthy") }